From 1feb3838b51c0b27b0f3e64f02358ff5ef3111b8 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Thu, 14 Oct 2021 17:38:57 +0300 Subject: [PATCH 001/226] add Ceil, Floor, Log, Round, Sqrt, Not, Equal, Less, Greater --- .../dnn/include/opencv2/dnn/all_layers.hpp | 43 + modules/dnn/src/cuda/activations.cu | 42 + modules/dnn/src/cuda/functors.hpp | 90 ++ modules/dnn/src/cuda/math.hpp | 21 + .../dnn/src/cuda4dnn/kernels/activations.hpp | 18 + .../src/cuda4dnn/primitives/activation.hpp | 332 +++---- modules/dnn/src/init.cpp | 7 + modules/dnn/src/layers/elementwise_layers.cpp | 917 ++++++++---------- modules/dnn/src/layers/scale_layer.cpp | 53 +- modules/dnn/src/onnx/onnx_importer.cpp | 36 + modules/dnn/src/opencl/activations.cl | 36 + modules/dnn/test/test_onnx_importer.cpp | 76 ++ 12 files changed, 952 insertions(+), 719 deletions(-) diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index 9fde7ad4c4..c70ce2b50a 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -600,6 +600,42 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + class CV_EXPORTS CeilLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS FloorLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS LogLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS RoundLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS SqrtLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS NotLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + class CV_EXPORTS ActivationLayerInt8 : public ActivationLayer { public: @@ -665,6 +701,7 @@ CV__DNN_INLINE_NS_BEGIN public: bool hasBias; int axis; + String mode; static Ptr create(const LayerParams& params); }; @@ -689,6 +726,12 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + class CV_EXPORTS CompareLayer : public Layer + { + public: + static Ptr create(const LayerParams& params); + }; + class CV_EXPORTS DataAugmentationLayer : public Layer { public: diff --git a/modules/dnn/src/cuda/activations.cu b/modules/dnn/src/cuda/activations.cu index 599d58852e..c38fa0346f 100644 --- a/modules/dnn/src/cuda/activations.cu +++ b/modules/dnn/src/cuda/activations.cu @@ -128,6 +128,36 @@ void bnll(const Stream& stream, Span output, View input) { generic_op>(stream, output, input); } +template +void ceil(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + +template +void floor(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + +template +void log(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + +template +void rint(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + +template +void sqrt(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + +template +void not_k(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + template void abs(const Stream& stream, Span output, View input) { generic_op>(stream, output, input); @@ -160,6 +190,12 @@ template void sigmoid<__half>(const Stream&, Span<__half>, View<__half>); template void elu<__half>(const Stream&, Span<__half>, View<__half>); template void abs<__half>(const Stream& stream, Span<__half> output, View<__half> input); template void bnll<__half>(const Stream&, Span<__half>, View<__half>); +template void ceil<__half>(const Stream&, Span<__half>, View<__half>); +template void floor<__half>(const Stream&, Span<__half>, View<__half>); +template void log<__half>(const Stream&, Span<__half>, View<__half>); +template void rint<__half>(const Stream&, Span<__half>, View<__half>); +template void sqrt<__half>(const Stream&, Span<__half>, View<__half>); +template void not_k<__half>(const Stream&, Span<__half>, View<__half>); template void power<__half>(const Stream&, Span<__half>, View<__half>, __half, __half, __half); template void exp<__half>(const Stream&, Span<__half>, View<__half>, __half, __half); #endif @@ -174,6 +210,12 @@ template void sigmoid(const Stream&, Span, View); template void elu(const Stream&, Span, View); template void abs(const Stream& stream, Span output, View input); template void bnll(const Stream&, Span, View); +template void ceil(const Stream&, Span, View); +template void floor(const Stream&, Span, View); +template void log(const Stream&, Span, View); +template void rint(const Stream&, Span, View); +template void sqrt(const Stream&, Span, View); +template void not_k(const Stream&, Span, View); template void power(const Stream&, Span, View, float, float, float); template void exp(const Stream&, Span, View, float, float); diff --git a/modules/dnn/src/cuda/functors.hpp b/modules/dnn/src/cuda/functors.hpp index f01a07c77e..04b545acaf 100644 --- a/modules/dnn/src/cuda/functors.hpp +++ b/modules/dnn/src/cuda/functors.hpp @@ -209,6 +209,96 @@ struct BNLLFunctor { } }; +template +struct CeilFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE CeilFunctor() { } + CUDA4DNN_DEVICE CeilFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::ceil; + return ceil(value); + } +}; + +template +struct FloorFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE FloorFunctor() { } + CUDA4DNN_DEVICE FloorFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::floor; + return floor(value); + } +}; + +template +struct LogFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE LogFunctor() { } + CUDA4DNN_DEVICE LogFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::log; + return log(value); + } +}; + +template +struct RintFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE RintFunctor() { } + CUDA4DNN_DEVICE RintFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::rint; + return rint(value); + } +}; + +template +struct SqrtFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE SqrtFunctor() { } + CUDA4DNN_DEVICE SqrtFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::sqrt; + return sqrt(value); + } +}; + +template +struct NotFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE NotFunctor() { } + CUDA4DNN_DEVICE NotFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::floor; + return floor(static_cast(1.) - value); + } +}; + template struct PowerFunctor { struct Params { diff --git a/modules/dnn/src/cuda/math.hpp b/modules/dnn/src/cuda/math.hpp index 273f3fe98e..0da584197d 100644 --- a/modules/dnn/src/cuda/math.hpp +++ b/modules/dnn/src/cuda/math.hpp @@ -119,6 +119,27 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace csl { namespace de template <> inline __device__ __half round(__half value) { return hrint(value); } #endif + template __device__ T floor(T value); + template <> inline __device__ double floor(double value) { return ::floor(value); } + template <> inline __device__ float floor(float value) { return floorf(value); } +#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) + template <> inline __device__ __half floor(__half value) { return hfloor(value); } +#endif + + template __device__ T log(T value); + template <> inline __device__ double log(double value) { return ::log(value); } + template <> inline __device__ float log(float value) { return logf(value); } +#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) + template <> inline __device__ __half log(__half value) { return hlog(value); } +#endif + + template __device__ T rint(T value); + template <> inline __device__ double rint(double value) { return ::rint(value); } + template <> inline __device__ float rint(float value) { return rintf(value); } +#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) + template <> inline __device__ __half rint(__half value) { return hrint(value); } +#endif + template __device__ T ceil(T value); template <> inline __device__ double ceil(double value) { return ::ceil(value); } template <> inline __device__ float ceil(float value) { return ceilf(value); } diff --git a/modules/dnn/src/cuda4dnn/kernels/activations.hpp b/modules/dnn/src/cuda4dnn/kernels/activations.hpp index 0a7c9878fb..0fcf7dab8a 100644 --- a/modules/dnn/src/cuda4dnn/kernels/activations.hpp +++ b/modules/dnn/src/cuda4dnn/kernels/activations.hpp @@ -42,6 +42,24 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { template void bnll(const csl::Stream& stream, csl::Span output, csl::View input); + template + void ceil(const csl::Stream& stream, csl::Span output, csl::View input); + + template + void floor(const csl::Stream& stream, csl::Span output, csl::View input); + + template + void log(const csl::Stream& stream, csl::Span output, csl::View input); + + template + void rint(const csl::Stream& stream, csl::Span output, csl::View input); + + template + void sqrt(const csl::Stream& stream, csl::Span output, csl::View input); + + template + void not_k(const csl::Stream& stream, csl::Span output, csl::View input); + template void power(const csl::Stream& stream, csl::Span output, csl::View input, T exp, T scale, T shift); diff --git a/modules/dnn/src/cuda4dnn/primitives/activation.hpp b/modules/dnn/src/cuda4dnn/primitives/activation.hpp index 84b95927a3..a179db2da5 100644 --- a/modules/dnn/src/cuda4dnn/primitives/activation.hpp +++ b/modules/dnn/src/cuda4dnn/primitives/activation.hpp @@ -18,14 +18,12 @@ namespace cv { namespace dnn { namespace cuda4dnn { - template - class ReLUOp final : public CUDABackendNode { - public: + template class Op, class T> + struct BaseOp : public CUDABackendNode + { + protected: using wrapper_type = GetCUDABackendWrapperType; - ReLUOp(csl::Stream stream_, T slope_) - : stream(std::move(stream_)), slope{ slope_ } { } - void forward( const std::vector>& inputs, const std::vector>& outputs, @@ -39,9 +37,21 @@ namespace cv { namespace dnn { namespace cuda4dnn { auto output_wrapper = outputs[i].dynamicCast(); auto output = output_wrapper->getSpan(); - kernels::relu(stream, output, input, slope); + static_cast*>(this)->calculate(output, input); } } + }; + + template + class ReLUOp final : public BaseOp { + public: + ReLUOp(csl::Stream stream_, T slope_) + : stream(std::move(stream_)), slope{ slope_ } { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::relu(stream, output, input, slope); + } private: csl::Stream stream; @@ -49,28 +59,14 @@ namespace cv { namespace dnn { namespace cuda4dnn { }; template - class ClippedReLUOp final : public CUDABackendNode { + class ClippedReLUOp final : public BaseOp { public: - using wrapper_type = GetCUDABackendWrapperType; - ClippedReLUOp(csl::Stream stream_, T min_, T max_) : stream(std::move(stream_)), min{ min_ }, max{ max_ } { } - void forward( - const std::vector>& inputs, - const std::vector>& outputs, - csl::Workspace& workspace) override + void calculate(csl::TensorSpan output, csl::TensorView input) const { - for (int i = 0; i < inputs.size(); i++) - { - auto input_wrapper = inputs[i].dynamicCast(); - auto input = input_wrapper->getView(); - - auto output_wrapper = outputs[i].dynamicCast(); - auto output = output_wrapper->getSpan(); - - kernels::clipped_relu(stream, output, input, min, max); - } + kernels::clipped_relu(stream, output, input, min, max); } private: @@ -79,35 +75,21 @@ namespace cv { namespace dnn { namespace cuda4dnn { }; template - class ChannelwiseReLUOp final : public CUDABackendNode { + class ChannelwiseReLUOp final : public BaseOp { public: - using wrapper_type = GetCUDABackendWrapperType; - ChannelwiseReLUOp(csl::Stream stream_, const Mat& slope) - : stream(std::move(stream_)) + : stream(std::move(stream_)) { CV_Assert(!slope.empty()); slopeTensor = csl::makeTensorHeader(slope); csl::copyMatToTensor(slope, slopeTensor, stream); } - void forward( - const std::vector>& inputs, - const std::vector>& outputs, - csl::Workspace& workspace) override + void calculate(csl::TensorSpan output, csl::TensorView input) const { - for (int i = 0; i < inputs.size(); i++) - { - auto input_wrapper = inputs[i].dynamicCast(); - auto input = input_wrapper->getView(); - - auto output_wrapper = outputs[i].dynamicCast(); - auto output = output_wrapper->getSpan(); - - CV_Assert(input.get_axis_size(1) == slopeTensor.size()); - std::size_t inner_size = input.size_range(2, input.rank()); - kernels::axiswise_relu(stream, output, input, inner_size, slopeTensor); - } + CV_Assert(input.get_axis_size(1) == slopeTensor.size()); + std::size_t inner_size = input.size_range(2, input.rank()); + kernels::axiswise_relu(stream, output, input, inner_size, slopeTensor); } private: @@ -116,27 +98,13 @@ namespace cv { namespace dnn { namespace cuda4dnn { }; template - class TanHOp final : public CUDABackendNode { + class TanHOp final : public BaseOp { public: - using wrapper_type = GetCUDABackendWrapperType; - TanHOp(csl::Stream stream_) : stream(std::move(stream_)) { } - void forward( - const std::vector>& inputs, - const std::vector>& outputs, - csl::Workspace& workspace) override + void calculate(csl::TensorSpan output, csl::TensorView input) const { - for (int i = 0; i < inputs.size(); i++) - { - auto input_wrapper = inputs[i].dynamicCast(); - auto input = input_wrapper->getView(); - - auto output_wrapper = outputs[i].dynamicCast(); - auto output = output_wrapper->getSpan(); - - kernels::tanh(stream, output, input); - } + kernels::tanh(stream, output, input); } private: @@ -144,27 +112,13 @@ namespace cv { namespace dnn { namespace cuda4dnn { }; template - class SwishOp final : public CUDABackendNode { + class SwishOp final : public BaseOp { public: - using wrapper_type = GetCUDABackendWrapperType; - SwishOp(csl::Stream stream_) : stream(std::move(stream_)) { } - void forward( - const std::vector>& inputs, - const std::vector>& outputs, - csl::Workspace& workspace) override + void calculate(csl::TensorSpan output, csl::TensorView input) const { - for (int i = 0; i < inputs.size(); i++) - { - auto input_wrapper = inputs[i].dynamicCast(); - auto input = input_wrapper->getView(); - - auto output_wrapper = outputs[i].dynamicCast(); - auto output = output_wrapper->getSpan(); - - kernels::swish(stream, output, input); - } + kernels::swish(stream, output, input); } private: @@ -172,27 +126,13 @@ namespace cv { namespace dnn { namespace cuda4dnn { }; template - class MishOp final : public CUDABackendNode { + class MishOp final : public BaseOp { public: - using wrapper_type = GetCUDABackendWrapperType; - MishOp(csl::Stream stream_) : stream(std::move(stream_)) { } - void forward( - const std::vector>& inputs, - const std::vector>& outputs, - csl::Workspace& workspace) override + void calculate(csl::TensorSpan output, csl::TensorView input) const { - for (int i = 0; i < inputs.size(); i++) - { - auto input_wrapper = inputs[i].dynamicCast(); - auto input = input_wrapper->getView(); - - auto output_wrapper = outputs[i].dynamicCast(); - auto output = output_wrapper->getSpan(); - - kernels::mish(stream, output, input); - } + kernels::mish(stream, output, input); } private: @@ -200,27 +140,13 @@ namespace cv { namespace dnn { namespace cuda4dnn { }; template - class SigmoidOp final : public CUDABackendNode { + class SigmoidOp final : public BaseOp { public: - using wrapper_type = GetCUDABackendWrapperType; - SigmoidOp(csl::Stream stream_) : stream(std::move(stream_)) { } - void forward( - const std::vector>& inputs, - const std::vector>& outputs, - csl::Workspace& workspace) override + void calculate(csl::TensorSpan output, csl::TensorView input) const { - for (int i = 0; i < inputs.size(); i++) - { - auto input_wrapper = inputs[i].dynamicCast(); - auto input = input_wrapper->getView(); - - auto output_wrapper = outputs[i].dynamicCast(); - auto output = output_wrapper->getSpan(); - - kernels::sigmoid(stream, output, input); - } + kernels::sigmoid(stream, output, input); } private: @@ -228,27 +154,13 @@ namespace cv { namespace dnn { namespace cuda4dnn { }; template - class ELUOp final : public CUDABackendNode { + class ELUOp final : public BaseOp { public: - using wrapper_type = GetCUDABackendWrapperType; - ELUOp(csl::Stream stream_) : stream(std::move(stream_)) { } - void forward( - const std::vector>& inputs, - const std::vector>& outputs, - csl::Workspace& workspace) override + void calculate(csl::TensorSpan output, csl::TensorView input) const { - for (int i = 0; i < inputs.size(); i++) - { - auto input_wrapper = inputs[i].dynamicCast(); - auto input = input_wrapper->getView(); - - auto output_wrapper = outputs[i].dynamicCast(); - auto output = output_wrapper->getSpan(); - - kernels::elu(stream, output, input); - } + kernels::elu(stream, output, input); } private: @@ -256,27 +168,13 @@ namespace cv { namespace dnn { namespace cuda4dnn { }; template - class AbsValOp final : public CUDABackendNode { + class AbsValOp final : public BaseOp { public: - using wrapper_type = GetCUDABackendWrapperType; - AbsValOp(csl::Stream stream_) : stream(std::move(stream_)) { } - void forward( - const std::vector>& inputs, - const std::vector>& outputs, - csl::Workspace& workspace) override + void calculate(csl::TensorSpan output, csl::TensorView input) const { - for (int i = 0; i < inputs.size(); i++) - { - auto input_wrapper = inputs[i].dynamicCast(); - auto input = input_wrapper->getView(); - - auto output_wrapper = outputs[i].dynamicCast(); - auto output = output_wrapper->getSpan(); - - kernels::abs(stream, output, input); - } + kernels::abs(stream, output, input); } private: @@ -284,27 +182,13 @@ namespace cv { namespace dnn { namespace cuda4dnn { }; template - class BNLLOp final : public CUDABackendNode { + class BNLLOp final : public BaseOp { public: - using wrapper_type = GetCUDABackendWrapperType; - BNLLOp(csl::Stream stream_) : stream(std::move(stream_)) { } - void forward( - const std::vector>& inputs, - const std::vector>& outputs, - csl::Workspace& workspace) override + void calculate(csl::TensorSpan output, csl::TensorView input) const { - for (int i = 0; i < inputs.size(); i++) - { - auto input_wrapper = inputs[i].dynamicCast(); - auto input = input_wrapper->getView(); - - auto output_wrapper = outputs[i].dynamicCast(); - auto output = output_wrapper->getSpan(); - - kernels::bnll(stream, output, input); - } + kernels::bnll(stream, output, input); } private: @@ -312,28 +196,98 @@ namespace cv { namespace dnn { namespace cuda4dnn { }; template - class PowerOp final : public CUDABackendNode { + class CeilOp final : public BaseOp { public: - using wrapper_type = GetCUDABackendWrapperType; + CeilOp(csl::Stream stream_) : stream(std::move(stream_)) { } + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::ceil(stream, output, input); + } + + private: + csl::Stream stream; + }; + + template + class FloorOp final : public BaseOp { + public: + FloorOp(csl::Stream stream_) : stream(std::move(stream_)) { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::floor(stream, output, input); + } + + private: + csl::Stream stream; + }; + + template + class LogOp final : public BaseOp { + public: + LogOp(csl::Stream stream_) : stream(std::move(stream_)) { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::log(stream, output, input); + } + + private: + csl::Stream stream; + }; + + template + class RoundOp final : public BaseOp { + public: + RoundOp(csl::Stream stream_) : stream(std::move(stream_)) { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::rint(stream, output, input); + } + + private: + csl::Stream stream; + }; + + template + class SqrtOp final : public BaseOp { + public: + SqrtOp(csl::Stream stream_) : stream(std::move(stream_)) { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::sqrt(stream, output, input); + } + + private: + csl::Stream stream; + }; + + template + class NotOp final : public BaseOp { + public: + NotOp(csl::Stream stream_) : stream(std::move(stream_)) { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::not_k(stream, output, input); + } + + private: + csl::Stream stream; + }; + + template + class PowerOp final : public BaseOp { + public: PowerOp(csl::Stream stream_, T exp_, T scale_, T shift_) : stream(std::move(stream_)), exp{ exp_ }, scale{ scale_ }, shift{ shift_ } { } - void forward( - const std::vector>& inputs, - const std::vector>& outputs, - csl::Workspace& workspace) override + void calculate(csl::TensorSpan output, csl::TensorView input) const { - for (int i = 0; i < inputs.size(); i++) - { - auto input_wrapper = inputs[i].dynamicCast(); - auto input = input_wrapper->getView(); - - auto output_wrapper = outputs[i].dynamicCast(); - auto output = output_wrapper->getSpan(); - - kernels::power(stream, output, input, exp, scale, shift); - } + kernels::power(stream, output, input, exp, scale, shift); } private: @@ -342,28 +296,14 @@ namespace cv { namespace dnn { namespace cuda4dnn { }; template - class ExpOp final : public CUDABackendNode { + class ExpOp final : public BaseOp { public: - using wrapper_type = GetCUDABackendWrapperType; - ExpOp(csl::Stream stream_, T nScale_, T nShift_) : stream(std::move(stream_)), normScale{ nScale_ }, normShift{ nShift_ } { } - void forward( - const std::vector>& inputs, - const std::vector>& outputs, - csl::Workspace& workspace) override + void calculate(csl::TensorSpan output, csl::TensorView input) const { - for (int i = 0; i < inputs.size(); i++) - { - auto input_wrapper = inputs[i].dynamicCast(); - auto input = input_wrapper->getView(); - - auto output_wrapper = outputs[i].dynamicCast(); - auto output = output_wrapper->getSpan(); - - kernels::exp(stream, output, input, normScale, normShift); - } + kernels::exp(stream, output, input, normScale, normShift); } private: diff --git a/modules/dnn/src/init.cpp b/modules/dnn/src/init.cpp index 123cb170b7..affaa1a7e1 100644 --- a/modules/dnn/src/init.cpp +++ b/modules/dnn/src/init.cpp @@ -111,6 +111,12 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(AbsVal, AbsLayer); CV_DNN_REGISTER_LAYER_CLASS(Power, PowerLayer); CV_DNN_REGISTER_LAYER_CLASS(Exp, ExpLayer); + CV_DNN_REGISTER_LAYER_CLASS(Ceil, CeilLayer); + CV_DNN_REGISTER_LAYER_CLASS(Floor, FloorLayer); + CV_DNN_REGISTER_LAYER_CLASS(Log, LogLayer); + CV_DNN_REGISTER_LAYER_CLASS(Round, RoundLayer); + CV_DNN_REGISTER_LAYER_CLASS(Sqrt, SqrtLayer); + CV_DNN_REGISTER_LAYER_CLASS(Not, NotLayer); CV_DNN_REGISTER_LAYER_CLASS(BatchNorm, BatchNormLayer); CV_DNN_REGISTER_LAYER_CLASS(MaxUnpool, MaxUnpoolLayer); CV_DNN_REGISTER_LAYER_CLASS(Dropout, BlankLayer); @@ -133,6 +139,7 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(Padding, PaddingLayer); CV_DNN_REGISTER_LAYER_CLASS(Proposal, ProposalLayer); CV_DNN_REGISTER_LAYER_CLASS(Scale, ScaleLayer); + CV_DNN_REGISTER_LAYER_CLASS(Compare, CompareLayer); CV_DNN_REGISTER_LAYER_CLASS(DataAugmentation, DataAugmentationLayer); CV_DNN_REGISTER_LAYER_CLASS(Correlation, CorrelationLayer); CV_DNN_REGISTER_LAYER_CLASS(Accum, AccumLayer); diff --git a/modules/dnn/src/layers/elementwise_layers.cpp b/modules/dnn/src/layers/elementwise_layers.cpp index 1b6483a5a0..c95dbbc933 100644 --- a/modules/dnn/src/layers/elementwise_layers.cpp +++ b/modules/dnn/src/layers/elementwise_layers.cpp @@ -49,7 +49,7 @@ #include "../op_vkcom.hpp" #include -#include +#include #ifdef HAVE_OPENCL #include "opencl_kernels_dnn.hpp" @@ -69,6 +69,11 @@ using std::abs; using std::exp; using std::tanh; using std::pow; +using std::ceil; +using std::floor; +using std::log; +using std::sqrt; +using std::round; template class ElementWiseLayer : public Func::Layer @@ -599,18 +604,9 @@ struct ReLU6Functor : public BaseFunctor int64 getFLOPSPerElement() const { return 2; } }; -struct TanHFunctor : public BaseFunctor +template +struct BaseDefaultFunctor : public BaseFunctor { - typedef TanHLayer Layer; - - bool supportBackend(int backendId, int) - { - return backendId == DNN_BACKEND_OPENCV || - backendId == DNN_BACKEND_CUDA || - backendId == DNN_BACKEND_HALIDE || - backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; - } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const { for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) @@ -618,7 +614,7 @@ struct TanHFunctor : public BaseFunctor for( int i = 0; i < len; i++ ) { float x = srcptr[i]; - dstptr[i] = tanh(x); + dstptr[i] = static_cast(this)->calculate(x); } } } @@ -638,19 +634,85 @@ struct TanHFunctor : public BaseFunctor UMat& src = inputs[i]; UMat& dst = outputs[i]; - ocl::Kernel kernel("TanHForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); + ocl::Kernel kernel(ocl_kernel_name, ocl::dnn::activations_oclsrc, buildopt); + kernel.set(0, static_cast(src.total())); kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); + static_cast(this)->setKernelParams(kernel); size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); + CV_Assert(kernel.run(1, &gSize, nullptr, false)); } return true; } #endif + inline void setKernelParams(ocl::Kernel& kernel) const {} + + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + float inpScale = scales[0][0], outScale = scales[1][0]; + int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale * static_cast(i - inpZp); + float y = static_cast(this)->calculate(x); + int quantized = outZp + static_cast(std::round(y/outScale)); + table[i+128] = saturate_cast(quantized); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + return true; + } + +#ifdef HAVE_DNN_IE_NN_BUILDER_2019 + InferenceEngine::Builder::Layer initInfEngineBuilderAPI() + { + CV_Error(Error::StsNotImplemented, ""); + } +#endif // HAVE_DNN_IE_NN_BUILDER_2019 + +#ifdef HAVE_DNN_NGRAPH + std::shared_ptr initNgraphAPI(const std::shared_ptr& node) + { + CV_Error(Error::StsNotImplemented, ""); + } +#endif // HAVE_DNN_NGRAPH + +#ifdef HAVE_VULKAN + std::shared_ptr initVkCom() + { + // TODO: add vkcom implementation + return std::shared_ptr(); + } +#endif // HAVE_VULKAN + +private: + static const char* const ocl_kernel_name; +}; + +struct TanHFunctor : public BaseDefaultFunctor +{ + typedef TanHLayer Layer; + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || + backendId == DNN_BACKEND_CUDA || + backendId == DNN_BACKEND_HALIDE || + backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; + } + + inline float calculate(float x) const + { + return tanh(x); + } + #ifdef HAVE_CUDA Ptr initCUDA(int target, csl::Stream stream) { @@ -680,38 +742,13 @@ struct TanHFunctor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH -#ifdef HAVE_VULKAN - std::shared_ptr initVkCom() - { - // TODO: add vkcom implementation - return std::shared_ptr(); - } -#endif // HAVE_VULKAN - - bool tryQuantize(const std::vector > &scales, - const std::vector > &zeropoints, LayerParams& params) - { - float inpScale = scales[0][0], outScale = scales[1][0]; - int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; - - Mat lookUpTable(1, 256, CV_8S); - int8_t* table = lookUpTable.ptr(); - for (int i = -128; i < 128; i++) - { - float x = inpScale*(i - inpZp); - float y = tanh(x); - int quantized = outZp + (int)std::round(y/outScale); - table[i+128] = saturate_cast(quantized); - } - params.blobs.clear(); - params.blobs.push_back(lookUpTable); - return true; - } - int64 getFLOPSPerElement() const { return 1; } }; -struct SwishFunctor : public BaseFunctor +template<> +const char* const TanHFunctor::BaseDefaultFunctor::ocl_kernel_name = "TanHForward"; + +struct SwishFunctor : public BaseDefaultFunctor { typedef SwishLayer Layer; @@ -719,49 +756,14 @@ struct SwishFunctor : public BaseFunctor { return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA || - backendId == DNN_BACKEND_HALIDE || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH;; + backendId == DNN_BACKEND_HALIDE || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + inline float calculate(float x) const { - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) - { - for( int i = 0; i < len; i++ ) - { - float x = srcptr[i]; - dstptr[i] = x / (1.0f + exp(-x)); - } - } + return x / (1.f + exp(-x)); } -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) - { - std::vector inputs; - std::vector outputs; - - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); - - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; - - ocl::Kernel kernel("SwishForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); - - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); - } - - return true; - } -#endif - #ifdef HAVE_CUDA Ptr initCUDA(int target, csl::Stream stream) { @@ -777,13 +779,6 @@ struct SwishFunctor : public BaseFunctor } #endif // HAVE_HALIDE -#ifdef HAVE_DNN_IE_NN_BUILDER_2019 - InferenceEngine::Builder::Layer initInfEngineBuilderAPI() - { - CV_Error(Error::StsNotImplemented, ""); - } -#endif // HAVE_DNN_IE_NN_BUILDER_2019 - #ifdef HAVE_DNN_NGRAPH std::shared_ptr initNgraphAPI(const std::shared_ptr& node) { @@ -792,38 +787,13 @@ struct SwishFunctor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH -#ifdef HAVE_VULKAN - std::shared_ptr initVkCom() - { - // TODO: add vkcom implementation - return std::shared_ptr(); - } -#endif // HAVE_VULKAN - - bool tryQuantize(const std::vector > &scales, - const std::vector > &zeropoints, LayerParams& params) - { - float inpScale = scales[0][0], outScale = scales[1][0]; - int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; - - Mat lookUpTable(1, 256, CV_8S); - int8_t* table = lookUpTable.ptr(); - for (int i = -128; i < 128; i++) - { - float x = inpScale*(i - inpZp); - float y = x / (1.0f + exp(-x)); - int quantized = outZp + (int)std::round(y/outScale); - table[i+128] = saturate_cast(quantized); - } - params.blobs.clear(); - params.blobs.push_back(lookUpTable); - return true; - } - int64 getFLOPSPerElement() const { return 3; } }; -struct MishFunctor : public BaseFunctor +template<> +const char* const SwishFunctor::BaseDefaultFunctor::ocl_kernel_name = "SwishForward"; + +struct MishFunctor : public BaseDefaultFunctor { typedef MishLayer Layer; @@ -834,53 +804,18 @@ struct MishFunctor : public BaseFunctor backendId == DNN_BACKEND_HALIDE || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + inline float calculate(float x) const { - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) + // Use fast approximation introduced in https://github.com/opencv/opencv/pull/17200 + if (x >= 8.f) { - for( int i = 0; i < len; i++ ) - { - // Use fast approximation introduced in https://github.com/opencv/opencv/pull/17200 - float x = srcptr[i]; - if (x >= 8.f) - dstptr[i] = x; - else - { - float eX = exp(x); - float n = (eX + 2) * eX; - dstptr[i] = (x * n) / (n + 2); - } - } - } - } - -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) - { - std::vector inputs; - std::vector outputs; - - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); - - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; - - ocl::Kernel kernel("MishForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); - - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); + return x; } - return true; + float eX = exp(x); + float n = (eX + 2.f) * eX; + return (x * n) / (n + 2.f); } -#endif #ifdef HAVE_CUDA Ptr initCUDA(int target, csl::Stream stream) @@ -897,13 +832,6 @@ struct MishFunctor : public BaseFunctor } #endif // HAVE_HALIDE -#ifdef HAVE_DNN_IE_NN_BUILDER_2019 - InferenceEngine::Builder::Layer initInfEngineBuilderAPI() - { - CV_Error(Error::StsNotImplemented, ""); - } -#endif // HAVE_DNN_IE_NN_BUILDER_2019 - #ifdef HAVE_DNN_NGRAPH std::shared_ptr initNgraphAPI(const std::shared_ptr& node) { @@ -917,40 +845,13 @@ struct MishFunctor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH -#ifdef HAVE_VULKAN - std::shared_ptr initVkCom() - { - // TODO: add vkcom implementation - return std::shared_ptr(); - } -#endif // HAVE_VULKAN - - bool tryQuantize(const std::vector > &scales, - const std::vector > &zeropoints, LayerParams& params) - { - float inpScale = scales[0][0], outScale = scales[1][0]; - int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; - - Mat lookUpTable(1, 256, CV_8S); - int8_t* table = lookUpTable.ptr(); - for (int i = -128; i < 128; i++) - { - float x = inpScale*(i - inpZp); - float eX = exp(x); - float n = (eX + 2) * eX; - float y = (x * n) / (n + 2); - int quantized = outZp + (int)std::round(y/outScale); - table[i+128] = saturate_cast(quantized); - } - params.blobs.clear(); - params.blobs.push_back(lookUpTable); - return true; - } - int64 getFLOPSPerElement() const { return 3; } }; -struct SigmoidFunctor : public BaseFunctor +template<> +const char* const MishFunctor::BaseDefaultFunctor::ocl_kernel_name = "MishForward"; + +struct SigmoidFunctor : public BaseDefaultFunctor { typedef SigmoidLayer Layer; @@ -962,46 +863,11 @@ struct SigmoidFunctor : public BaseFunctor backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + inline float calculate(float x) const { - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) - { - for( int i = 0; i < len; i++ ) - { - float x = srcptr[i]; - dstptr[i] = 1.f/(1.f + exp(-x)); - } - } + return 1.f / (1.f + exp(-x)); } -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) - { - std::vector inputs; - std::vector outputs; - - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); - - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; - - ocl::Kernel kernel("SigmoidForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); - - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); - } - - return true; - } -#endif - #ifdef HAVE_CUDA Ptr initCUDA(int target, csl::Stream stream) { @@ -1031,38 +897,13 @@ struct SigmoidFunctor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH -#ifdef HAVE_VULKAN - std::shared_ptr initVkCom() - { - // TODO: add vkcom implementation - return std::shared_ptr(); - } -#endif // HAVE_VULKAN - - bool tryQuantize(const std::vector > &scales, - const std::vector > &zeropoints, LayerParams& params) - { - float inpScale = scales[0][0], outScale = scales[1][0]; - int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; - - Mat lookUpTable(1, 256, CV_8S); - int8_t* table = lookUpTable.ptr(); - for (int i = -128; i < 128; i++) - { - float x = inpScale*(i - inpZp); - float y = 1.f/(1.f + exp(-x)); - int quantized = outZp + (int)std::round(y/outScale); - table[i+128] = saturate_cast(quantized); - } - params.blobs.clear(); - params.blobs.push_back(lookUpTable); - return true; - } - int64 getFLOPSPerElement() const { return 3; } }; -struct ELUFunctor : public BaseFunctor +template<> +const char* const SigmoidFunctor::BaseDefaultFunctor::ocl_kernel_name = "SigmoidForward"; + +struct ELUFunctor : public BaseDefaultFunctor { typedef ELULayer Layer; @@ -1074,46 +915,11 @@ struct ELUFunctor : public BaseFunctor backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + inline float calculate(float x) const { - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) - { - for(int i = 0; i < len; i++ ) - { - float x = srcptr[i]; - dstptr[i] = x >= 0.f ? x : exp(x) - 1; - } - } + return x >= 0.f ? x : exp(x) - 1.f; } -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) - { - std::vector inputs; - std::vector outputs; - - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); - - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; - - ocl::Kernel kernel("ELUForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); - - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); - } - - return true; - } -#endif - #ifdef HAVE_CUDA Ptr initCUDA(int target, csl::Stream stream) { @@ -1143,38 +949,13 @@ struct ELUFunctor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH -#ifdef HAVE_VULKAN - std::shared_ptr initVkCom() - { - // TODO: add vkcom implementation - return std::shared_ptr(); - } -#endif // HAVE_VULKAN - - bool tryQuantize(const std::vector > &scales, - const std::vector > &zeropoints, LayerParams& params) - { - float inpScale = scales[0][0], outScale = scales[1][0]; - int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; - - Mat lookUpTable(1, 256, CV_8S); - int8_t* table = lookUpTable.ptr(); - for (int i = -128; i < 128; i++) - { - float x = inpScale*(i - inpZp); - float y = x >= 0.f ? x : exp(x) - 1; - int quantized = outZp + (int)std::round(y/outScale); - table[i+128] = saturate_cast(quantized); - } - params.blobs.clear(); - params.blobs.push_back(lookUpTable); - return true; - } - int64 getFLOPSPerElement() const { return 2; } }; -struct AbsValFunctor : public BaseFunctor +template<> +const char* const ELUFunctor::BaseDefaultFunctor::ocl_kernel_name = "ELUForward"; + +struct AbsValFunctor : public BaseDefaultFunctor { typedef AbsLayer Layer; @@ -1189,46 +970,11 @@ struct AbsValFunctor : public BaseFunctor backendId == DNN_BACKEND_HALIDE; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + inline float calculate(float x) const { - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) - { - for( int i = 0; i < len; i++ ) - { - float x = srcptr[i]; - dstptr[i] = abs(x); - } - } + return abs(x); } -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) - { - std::vector inputs; - std::vector outputs; - - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); - - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; - - ocl::Kernel kernel("AbsValForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); - - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); - } - - return true; - } -#endif - #ifdef HAVE_CUDA Ptr initCUDA(int target, csl::Stream stream) { @@ -1261,38 +1007,13 @@ struct AbsValFunctor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH -#ifdef HAVE_VULKAN - std::shared_ptr initVkCom() - { - // TODO: add vkcom implementation - return std::shared_ptr(); - } -#endif // HAVE_VULKAN - - bool tryQuantize(const std::vector > &scales, - const std::vector > &zeropoints, LayerParams& params) - { - float inpScale = scales[0][0], outScale = scales[1][0]; - int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; - - Mat lookUpTable(1, 256, CV_8S); - int8_t* table = lookUpTable.ptr(); - for (int i = -128; i < 128; i++) - { - float x = inpScale*(i - inpZp); - float y = abs(x); - int quantized = outZp + (int)std::round(y/outScale); - table[i+128] = saturate_cast(quantized); - } - params.blobs.clear(); - params.blobs.push_back(lookUpTable); - return true; - } - int64 getFLOPSPerElement() const { return 1; } }; -struct BNLLFunctor : public BaseFunctor +template<> +const char* const AbsValFunctor::BaseDefaultFunctor::ocl_kernel_name = "AbsValForward"; + +struct BNLLFunctor : public BaseDefaultFunctor { typedef BNLLLayer Layer; @@ -1303,47 +1024,12 @@ struct BNLLFunctor : public BaseFunctor backendId == DNN_BACKEND_HALIDE; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + inline float calculate(float x) const { - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) - { - for( int i = 0; i < len; i++ ) - { - float x = srcptr[i]; - // https://github.com/BVLC/caffe/blame/1.0/src/caffe/layers/bnll_layer.cpp#L17 - dstptr[i] = x > 0 ? x + log(1. + exp(-x)) : log(1. + exp(x)); - } - } + // https://github.com/BVLC/caffe/blame/1.0/src/caffe/layers/bnll_layer.cpp#L17 + return x > 0 ? x + log(1.f + exp(-x)) : log(1.f + exp(x)); } -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) - { - std::vector inputs; - std::vector outputs; - - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); - - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; - - ocl::Kernel kernel("BNLLForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); - - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); - } - - return true; - } -#endif - #ifdef HAVE_CUDA Ptr initCUDA(int target, csl::Stream stream) { @@ -1360,51 +1046,234 @@ struct BNLLFunctor : public BaseFunctor } #endif // HAVE_HALIDE -#ifdef HAVE_DNN_IE_NN_BUILDER_2019 - InferenceEngine::Builder::Layer initInfEngineBuilderAPI() + int64 getFLOPSPerElement() const { return 5; } +}; + +template<> +const char* const BNLLFunctor::BaseDefaultFunctor::ocl_kernel_name = "BNLLForward"; + +struct CeilFunctor : public BaseDefaultFunctor +{ + typedef CeilLayer Layer; + + bool supportBackend(int backendId, int) { - CV_Error(Error::StsNotImplemented, ""); + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE; } -#endif // HAVE_DNN_IE_NN_BUILDER_2019 + + inline float calculate(float x) const + { + return ceil(x); + } + +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream); + } +#endif + +#ifdef HAVE_HALIDE + void attachHalide(const Halide::Expr& input, Halide::Func& top) + { + Halide::Var x("x"), y("y"), c("c"), n("n"); + top(x, y, c, n) = ceil(input); + } +#endif // HAVE_HALIDE + + int64 getFLOPSPerElement() const { return 1; } +}; + +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "CeilForward"; + +struct FloorFunctor : public BaseDefaultFunctor +{ + typedef FloorLayer Layer; + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE; + } + + inline float calculate(float x) const + { + return floor(x); + } + +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream); + } +#endif + +#ifdef HAVE_HALIDE + void attachHalide(const Halide::Expr& input, Halide::Func& top) + { + Halide::Var x("x"), y("y"), c("c"), n("n"); + top(x, y, c, n) = floor(input); + } +#endif // HAVE_HALIDE + + int64 getFLOPSPerElement() const { return 1; } +}; + +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "FloorForward"; + +struct LogFunctor : public BaseDefaultFunctor +{ + typedef LogLayer Layer; + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE; + } + + inline float calculate(float x) const + { + return log(x); + } + +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream); + } +#endif + +#ifdef HAVE_HALIDE + void attachHalide(const Halide::Expr& input, Halide::Func& top) + { + Halide::Var x("x"), y("y"), c("c"), n("n"); + top(x, y, c, n) = log(input); + } +#endif // HAVE_HALIDE + + int64 getFLOPSPerElement() const { return 1; } +}; + +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "LogForward"; + +struct RoundFunctor : public BaseDefaultFunctor +{ + typedef RoundLayer Layer; + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE; + } + + inline float calculate(float x) const + { + // Rounds to even numbers in halfway cases, so 2.5 -> 2, -2.5 -> -2 + int old_rounding_direction = std::fegetround(); + std::fesetround(FE_TONEAREST); + float y = std::nearbyint(x); + std::fesetround(old_rounding_direction); + return y; + } + +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream); + } +#endif + +#ifdef HAVE_HALIDE + void attachHalide(const Halide::Expr& input, Halide::Func& top) + { + Halide::Var x("x"), y("y"), c("c"), n("n"); + top(x, y, c, n) = round(input); + } +#endif // HAVE_HALIDE + + int64 getFLOPSPerElement() const { return 2; } +}; + +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "RoundForward"; + +struct SqrtFunctor : public BaseDefaultFunctor +{ + typedef SqrtLayer Layer; + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE; + } + + inline float calculate(float x) const + { + return sqrt(x); + } + +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream); + } +#endif + +#ifdef HAVE_HALIDE + void attachHalide(const Halide::Expr& input, Halide::Func& top) + { + Halide::Var x("x"), y("y"), c("c"), n("n"); + top(x, y, c, n) = sqrt(input); + } +#endif // HAVE_HALIDE #ifdef HAVE_DNN_NGRAPH std::shared_ptr initNgraphAPI(const std::shared_ptr& node) { - CV_Error(Error::StsNotImplemented, ""); + return std::make_shared(node); } #endif // HAVE_DNN_NGRAPH -#ifdef HAVE_VULKAN - std::shared_ptr initVkCom() - { - // TODO: add vkcom implementation - return std::shared_ptr(); - } -#endif // HAVE_VULKAN - - bool tryQuantize(const std::vector > &scales, - const std::vector > &zeropoints, LayerParams& params) - { - float inpScale = scales[0][0], outScale = scales[1][0]; - int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; - - Mat lookUpTable(1, 256, CV_8S); - int8_t* table = lookUpTable.ptr(); - for (int i = -128; i < 128; i++) - { - float x = inpScale*(i - inpZp); - float y = x > 0 ? x + log(1. + exp(-x)) : log(1. + exp(x)); - int quantized = outZp + (int)std::round(y/outScale); - table[i+128] = saturate_cast(quantized); - } - params.blobs.clear(); - params.blobs.push_back(lookUpTable); - return true; - } - - int64 getFLOPSPerElement() const { return 5; } + int64 getFLOPSPerElement() const { return 1; } }; +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "SqrtForward"; + +struct NotFunctor : public BaseDefaultFunctor +{ + typedef NotLayer Layer; + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE; + } + + inline float calculate(float x) const + { + return floor(1.f - x); + } + +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream); + } +#endif + +#ifdef HAVE_HALIDE + void attachHalide(const Halide::Expr& input, Halide::Func& top) + { + Halide::Var x("x"), y("y"), c("c"), n("n"); + top(x, y, c, n) = floor(1.0f - input); + } +#endif // HAVE_HALIDE + + int64 getFLOPSPerElement() const { return 2; } +}; + +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "NotForward"; + struct PowerFunctor : public BaseFunctor { typedef PowerLayer Layer; @@ -1583,7 +1452,7 @@ struct PowerFunctor : public BaseFunctor int64 getFLOPSPerElement() const { return power == 1 ? 2 : 10; } }; -struct ExpFunctor : public BaseFunctor +struct ExpFunctor : public BaseDefaultFunctor { typedef ExpLayer Layer; float base, scale, shift; @@ -1609,47 +1478,16 @@ struct ExpFunctor : public BaseFunctor backendId == DNN_BACKEND_HALIDE || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + inline float calculate(float x) const { - float a = normScale, b = normShift; - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) - { - for( int i = 0; i < len; i++ ) - { - float x = srcptr[i]; - dstptr[i] = exp(a*x + b); - } - } + return exp(normScale * x + normShift); } -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) + inline void setKernelParams(ocl::Kernel& kernel) const { - std::vector inputs; - std::vector outputs; - - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); - - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; - - ocl::Kernel kernel("ExpForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); - kernel.set(3, (float)normScale); - kernel.set(4, (float)normShift); - - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); - } - return true; + kernel.set(3, normScale); + kernel.set(4, normShift); } -#endif #ifdef HAVE_CUDA Ptr initCUDA(int target, csl::Stream stream) @@ -1666,13 +1504,6 @@ struct ExpFunctor : public BaseFunctor } #endif // HAVE_HALIDE -#ifdef HAVE_DNN_IE_NN_BUILDER_2019 - InferenceEngine::Builder::Layer initInfEngineBuilderAPI() - { - CV_Error(Error::StsNotImplemented, ""); - } -#endif // HAVE_DNN_IE_NN_BUILDER_2019 - #ifdef HAVE_DNN_NGRAPH std::shared_ptr initNgraphAPI(const std::shared_ptr& node) { @@ -1686,17 +1517,12 @@ struct ExpFunctor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH -#ifdef HAVE_VULKAN - std::shared_ptr initVkCom() - { - // TODO: add vkcom implementation - return std::shared_ptr(); - } -#endif // HAVE_VULKAN - int64 getFLOPSPerElement() const { return 3; } }; +template<> +const char* const ExpFunctor::BaseDefaultFunctor::ocl_kernel_name = "ExpForward"; + struct ChannelsPReLUFunctor : public BaseFunctor { typedef ChannelsPReLULayer Layer; @@ -1917,6 +1743,55 @@ Ptr BNLLLayer::create(const LayerParams& params) return l; } + +Ptr CeilLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + +Ptr FloorLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + +Ptr LogLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + +Ptr RoundLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + +Ptr SqrtLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + +Ptr NotLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + Ptr PowerLayer::create(const LayerParams& params) { float power = params.get("power", 1.0f); diff --git a/modules/dnn/src/layers/scale_layer.cpp b/modules/dnn/src/layers/scale_layer.cpp index 001db24a2d..003f78dc1d 100644 --- a/modules/dnn/src/layers/scale_layer.cpp +++ b/modules/dnn/src/layers/scale_layer.cpp @@ -38,6 +38,7 @@ public: hasBias = params.get("bias_term", false); axis = params.get("axis", 1); hasWeights = false; + mode = params.get("mode", "scale"); } bool getMemoryShapes(const std::vector &inputs, @@ -59,6 +60,10 @@ public: virtual bool supportBackend(int backendId) CV_OVERRIDE { + if (mode != "scale") + { + return backendId == DNN_BACKEND_OPENCV; + } return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA || backendId == DNN_BACKEND_HALIDE || @@ -66,6 +71,20 @@ public: (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && axis > 0); } + template + void handleCompare(const Mat& a, const T& b, Mat& dst, const int spatialSize) + { + Mat out(1, spatialSize, CV_8U); + if (mode == "equal") + compare(a, b, out, CMP_EQ); + else if (mode == "greater") + compare(a, b, out, CMP_GT); + else + compare(a, b, out, CMP_LT); + + out.convertTo(dst, CV_32F, 1. / 255.); + } + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE { CV_TRACE_FUNCTION(); @@ -123,7 +142,16 @@ public: float b = biasesData ? biasesData[j] : 0; Mat inpSlice(1, spatialSize, CV_32F, inpData); Mat outSlice(1, spatialSize, CV_32F, outData); - inpSlice.convertTo(outSlice, CV_32F, w, b); + + if (mode == "scale") + { + inpSlice.convertTo(outSlice, CV_32F, w, b); + } + else + { + handleCompare(inpSlice, b, outSlice, spatialSize); + } + inpData += spatialSize; outData += spatialSize; } @@ -142,7 +170,16 @@ public: add(outSlice, bias, outSlice); } else if (hasBias) - add(inpSlice, bias, outSlice); + { + if (mode == "scale") + { + add(inpSlice, bias, outSlice); + } + else + { + handleCompare(inpSlice, bias, outSlice, numWeights); + } + } inpData += numWeights; outData += numWeights; } @@ -385,6 +422,18 @@ Ptr ShiftLayer::create(const LayerParams& params) return Ptr(new ScaleLayerImpl(scaleParams)); } +Ptr CompareLayer::create(const LayerParams& params) +{ + LayerParams compareParams; + compareParams.name = params.name; + compareParams.type = "Scale"; + compareParams.blobs = params.blobs; + compareParams.set("bias_term", true); + compareParams.set("axis", 0); + compareParams.set("mode", params.get("mode")); + return Ptr(new ScaleLayerImpl(compareParams)); +} + class DataAugmentationLayerImpl CV_FINAL : public DataAugmentationLayer { public: diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 7c230f28c8..4914810df8 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -118,6 +118,8 @@ private: void parseRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseElu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseTanh (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseAbs (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseCompare (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parsePRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseLRN (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseInstanceNormalization(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); @@ -1375,6 +1377,38 @@ void ONNXImporter::parseTanh(LayerParams& layerParams, const opencv_onnx::NodePr addLayer(layerParams, node_proto); } +void ONNXImporter::parseAbs(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "AbsVal"; + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseCompare(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 2); + const std::string& layer_type = node_proto.op_type(); + + bool is_const_0 = layer_id.find(node_proto.input(0)) == layer_id.end(); + bool is_const_1 = layer_id.find(node_proto.input(1)) == layer_id.end(); + + if (is_const_0 || is_const_1) + { + Mat blob = getBlob(node_proto, static_cast(is_const_1)); + blob = blob.reshape(1, 1); + layerParams.blobs.push_back(blob); + } + + layerParams.type = "Compare"; + + if (layer_type == "Equal") + layerParams.set("mode", "equal"); + else if (layer_type == "Greater") + layerParams.set("mode", "greater"); + else + layerParams.set("mode", "less"); + addLayer(layerParams, node_proto); +} + void ONNXImporter::parsePRelu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { layerParams.type = "PReLU"; @@ -2904,6 +2938,8 @@ const ONNXImporter::DispatchMap ONNXImporter::buildDispatchMap() dispatch["Relu"] = &ONNXImporter::parseRelu; dispatch["Elu"] = &ONNXImporter::parseElu; dispatch["Tanh"] = &ONNXImporter::parseTanh; + dispatch["Abs"] = &ONNXImporter::parseAbs; + dispatch["Equal"] = dispatch["Greater"] = dispatch["Less"] = &ONNXImporter::parseCompare; dispatch["PRelu"] = &ONNXImporter::parsePRelu; dispatch["LRN"] = &ONNXImporter::parseLRN; dispatch["InstanceNormalization"] = &ONNXImporter::parseInstanceNormalization; diff --git a/modules/dnn/src/opencl/activations.cl b/modules/dnn/src/opencl/activations.cl index 68f0dd7268..bc2a105aba 100644 --- a/modules/dnn/src/opencl/activations.cl +++ b/modules/dnn/src/opencl/activations.cl @@ -151,3 +151,39 @@ __kernel void ExpForward(const int n, __global const T* in, __global T* out, out[index] = exp(normShift + normScale * in[index]); } } + +__kernel void CeilForward(const int n, __global T* in, __global T* out) { + int index = get_global_id(0); + if(index < n) + out[index] = ceil(in[index]); +} + +__kernel void FloorForward(const int n, __global T* in, __global T* out) { + int index = get_global_id(0); + if(index < n) + out[index] = floor(in[index]); +} + +__kernel void LogForward(const int n, __global T* in, __global T* out) { + int index = get_global_id(0); + if(index < n) + out[index] = log(in[index]); +} + +__kernel void RoundForward(const int n, __global T* in, __global T* out) { + int index = get_global_id(0); + if(index < n) + out[index] = rint(in[index]); +} + +__kernel void SqrtForward(const int n, __global T* in, __global T* out) { + int index = get_global_id(0); + if(index < n) + out[index] = sqrt(in[index]); +} + +__kernel void NotForward(const int n, __global T* in, __global T* out) { + int index = get_global_id(0); + if(index < n) + out[index] = floor(1.0f - in[index]); +} diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 5d324b8aac..b5b277a492 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -353,6 +353,82 @@ TEST_P(Test_ONNX_layers, Exp) testONNXModels("exp"); } +TEST_P(Test_ONNX_layers, Elementwise_Ceil) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + testONNXModels("ceil"); +} + +TEST_P(Test_ONNX_layers, Elementwise_Floor) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + testONNXModels("floor"); +} + +TEST_P(Test_ONNX_layers, Elementwise_Log) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + testONNXModels("log"); +} + +TEST_P(Test_ONNX_layers, Elementwise_Round) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + testONNXModels("round"); +} + +TEST_P(Test_ONNX_layers, Elementwise_Sqrt) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + testONNXModels("sqrt"); +} + +TEST_P(Test_ONNX_layers, Elementwise_not) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + testONNXModels("not"); +} + +TEST_P(Test_ONNX_layers, Compare) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + testONNXModels("equal"); + testONNXModels("greater"); + testONNXModels("less"); +} + +TEST_P(Test_ONNX_layers, CompareSameDims) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + testONNXModels("equal_same_dims", npy, 0, 0, false, true, 2); + testONNXModels("greater_same_dims", npy, 0, 0, false, true, 2); + testONNXModels("less_same_dims", npy, 0, 0, false, true, 2); +} + TEST_P(Test_ONNX_layers, Concatenation) { if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) From f9e747dbc62277330cdade648ef53f5e5e6c69f9 Mon Sep 17 00:00:00 2001 From: Wehzie <39304339+Wehzie@users.noreply.github.com> Date: Thu, 14 Oct 2021 22:39:49 +0300 Subject: [PATCH 002/226] Fixed typo in CV_Error message Error was "Input parameters must be a matrices!", but "matrices" is plural and doesn't allow the unspecific article "a". --- modules/calib3d/src/calibration.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/calib3d/src/calibration.cpp b/modules/calib3d/src/calibration.cpp index 710b1d7659..650735f035 100644 --- a/modules/calib3d/src/calibration.cpp +++ b/modules/calib3d/src/calibration.cpp @@ -1737,7 +1737,7 @@ void cvCalibrationMatrixValues( const CvMat *calibMatr, CvSize imgSize, CV_Error(CV_StsNullPtr, "Some of parameters is a NULL pointer!"); if(!CV_IS_MAT(calibMatr)) - CV_Error(CV_StsUnsupportedFormat, "Input parameters must be a matrices!"); + CV_Error(CV_StsUnsupportedFormat, "Input parameters must be matrices!"); double dummy = .0; Point2d pp; @@ -3078,7 +3078,7 @@ cvDecomposeProjectionMatrix( const CvMat *projMatr, CvMat *calibMatr, CV_Error(CV_StsNullPtr, "Some of parameters is a NULL pointer!"); if(!CV_IS_MAT(projMatr) || !CV_IS_MAT(calibMatr) || !CV_IS_MAT(rotMatr) || !CV_IS_MAT(posVect)) - CV_Error(CV_StsUnsupportedFormat, "Input parameters must be a matrices!"); + CV_Error(CV_StsUnsupportedFormat, "Input parameters must be matrices!"); if(projMatr->cols != 4 || projMatr->rows != 3) CV_Error(CV_StsUnmatchedSizes, "Size of projection matrix must be 3x4!"); From 1926e919bead35b3428482e30d50a830c3253d89 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 18 Oct 2021 04:46:00 +0000 Subject: [PATCH 003/226] dnn(int8): fix using of incorrect UMat constructor --- modules/dnn/src/int8layers/quantization_utils.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/dnn/src/int8layers/quantization_utils.cpp b/modules/dnn/src/int8layers/quantization_utils.cpp index 0346f147ba..d72487639e 100644 --- a/modules/dnn/src/int8layers/quantization_utils.cpp +++ b/modules/dnn/src/int8layers/quantization_utils.cpp @@ -52,9 +52,9 @@ public: if (inputs_.depth() == CV_16S) { - UMat inputFp32(shape(inputs[0]), CV_32F); + UMat inputFp32; convertFp16(inputs[0], inputFp32); - inputFp32.copyTo(inputs[0]); + inputs[0] = inputFp32; // replace } inputs[0].convertTo(outputs[0], CV_8S, 1.f/scale, zeropoint); @@ -118,7 +118,7 @@ public: inputs_.getUMatVector(inputs); outputs_.getUMatVector(outputs); - UMat outputFp32(shape(outputs[0]), CV_32F); + UMat outputFp32; inputs[0].convertTo(outputFp32, CV_32F, scale, -(scale*zeropoint)); if (outputs_.depth() == CV_16S) From f8f9f3c43851695726bf32d07bbd176a082755fd Mon Sep 17 00:00:00 2001 From: Sergiu Deitsch Date: Mon, 18 Oct 2021 14:56:15 +0200 Subject: [PATCH 004/226] fixed AVX compile error Some older compilers do not allow to pass a `const int` as an immediate. Use an unnamed enum instead. --- modules/core/include/opencv2/core/hal/intrin_avx.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/include/opencv2/core/hal/intrin_avx.hpp b/modules/core/include/opencv2/core/hal/intrin_avx.hpp index 54e8927192..09ff566473 100644 --- a/modules/core/include/opencv2/core/hal/intrin_avx.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_avx.hpp @@ -2379,7 +2379,7 @@ inline void v_load_deinterleave( const unsigned* ptr, v_uint32x8& a, v_uint32x8& __m256i ab0 = _mm256_loadu_si256((const __m256i*)ptr); __m256i ab1 = _mm256_loadu_si256((const __m256i*)(ptr + 8)); - const int sh = 0+2*4+1*16+3*64; + enum { sh = 0+2*4+1*16+3*64 }; __m256i p0 = _mm256_shuffle_epi32(ab0, sh); __m256i p1 = _mm256_shuffle_epi32(ab1, sh); __m256i pl = _mm256_permute2x128_si256(p0, p1, 0 + 2*16); From 0cf79155d4e06cfcc64cf5eb2acc2fccc2af3db2 Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Mon, 18 Oct 2021 19:20:55 +0300 Subject: [PATCH 005/226] Merge pull request #20773 from sivanov-work:merge_vpl_source_unite G-API: oneVPL (simplification) unite components in entire VPL source * Unify components in VPLSource * Revert back decode WRN & Add compile guard * Address come comments * Add source alias * Apply comment for exception handling --- modules/gapi/CMakeLists.txt | 2 + .../opencv2/gapi/streaming/onevpl/source.hpp | 2 + .../gapi/samples/onevpl_infer_single_roi.cpp | 4 +- .../onevpl/accelerators/accel_policy_dx11.cpp | 3 +- .../streaming/onevpl/cfg_params_parser.cpp | 123 +++++ .../streaming/onevpl/cfg_params_parser.hpp | 43 ++ .../onevpl/engine/processing_engine_base.cpp | 2 +- .../onevpl/engine/processing_engine_base.hpp | 3 + .../gapi/src/streaming/onevpl/source_priv.cpp | 332 ++++++++++++- .../gapi/src/streaming/onevpl/source_priv.hpp | 21 + modules/gapi/src/streaming/onevpl/utils.cpp | 454 ++++++++++++++++++ modules/gapi/src/streaming/onevpl/utils.hpp | 24 +- 12 files changed, 1001 insertions(+), 12 deletions(-) create mode 100644 modules/gapi/src/streaming/onevpl/cfg_params_parser.cpp create mode 100644 modules/gapi/src/streaming/onevpl/cfg_params_parser.hpp create mode 100644 modules/gapi/src/streaming/onevpl/utils.cpp diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 5fd108c0ec..917d1e0814 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -168,6 +168,8 @@ set(gapi_srcs src/streaming/onevpl/source_priv.cpp src/streaming/onevpl/file_data_provider.cpp src/streaming/onevpl/cfg_params.cpp + src/streaming/onevpl/cfg_params_parser.cpp + src/streaming/onevpl/utils.cpp src/streaming/onevpl/data_provider_interface_exception.cpp src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp src/streaming/onevpl/accelerators/surface/surface.cpp diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp index b7bf4f2ffb..a8dbefdf50 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp @@ -51,6 +51,8 @@ private: }; } // namespace onevpl +using GVPLSource = onevpl::GSource; + template GAPI_EXPORTS_W cv::Ptr inline make_onevpl_src(Args&&... args) { diff --git a/modules/gapi/samples/onevpl_infer_single_roi.cpp b/modules/gapi/samples/onevpl_infer_single_roi.cpp index 9a3abdf392..f3aee09d42 100644 --- a/modules/gapi/samples/onevpl_infer_single_roi.cpp +++ b/modules/gapi/samples/onevpl_infer_single_roi.cpp @@ -19,6 +19,7 @@ const std::string keys = "{ input | | Path to the input demultiplexed video file }" "{ output | | Path to the output RAW video file. Use .avi extension }" "{ facem | face-detection-adas-0001.xml | Path to OpenVINO IE face detection model (.xml) }" + "{ faced | CPU | Target device for face detection model (e.g. CPU, GPU, VPU, ...) }" "{ cfg_params | :;: | Semicolon separated list of oneVPL mfxVariants which is used for configuring source (see `MFXSetConfigFilterProperty` by https://spec.oneapi.io/versions/latest/elements/oneVPL/source/index.html) }"; @@ -198,7 +199,8 @@ int main(int argc, char *argv[]) { auto face_net = cv::gapi::ie::Params { face_model_path, // path to topology IR - get_weights_path(face_model_path) // path to weights + get_weights_path(face_model_path), // path to weights + cmd.get("faced"), // device specifier }; auto kernels = cv::gapi::kernels < custom::OCVLocateROI diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp index cb27df8661..8365dd20e3 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp @@ -30,7 +30,8 @@ namespace gapi { namespace wip { namespace onevpl { -VPLDX11AccelerationPolicy::VPLDX11AccelerationPolicy() +VPLDX11AccelerationPolicy::VPLDX11AccelerationPolicy() : + hw_handle(nullptr) { #ifdef CPU_ACCEL_ADAPTER adapter.reset(new VPLCPUAccelerationPolicy); diff --git a/modules/gapi/src/streaming/onevpl/cfg_params_parser.cpp b/modules/gapi/src/streaming/onevpl/cfg_params_parser.cpp new file mode 100644 index 0000000000..a683c7478e --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/cfg_params_parser.cpp @@ -0,0 +1,123 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include + +#include +#include + +#include "streaming/onevpl/cfg_params_parser.hpp" +#include "streaming/onevpl/utils.hpp" +#include "logger.hpp" + +#ifdef HAVE_ONEVPL +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +template <> +struct ParamCreator { + template + CfgParam create (const std::string& name, ValueType&& value) { + return CfgParam::create(name, std::forward(value), is_major_flag); + } + bool is_major_flag = false; +}; + +template <> +struct ParamCreator { + template + mfxVariant create (const std::string& name, ValueType&& value) { + static_assert(std::is_same::type, mfxU32>::value, + "ParamCreator supports mfxU32 at the moment. " + "Feel free to extend for more types"); + return create_impl(name, value); + } +private: + mfxVariant create_impl(const std::string&, mfxU32 value) { + mfxVariant ret; + ret.Type = MFX_VARIANT_TYPE_U32; + ret.Data.U32 = value; + return ret; + } +}; + +template +std::vector get_params_from_string(const std::string& str) { + std::vector ret; + std::string::size_type pos = 0; + std::string::size_type endline_pos = std::string::npos; + do + { + endline_pos = str.find_first_of("\r\n", pos); + std::string line = str.substr(pos, endline_pos == std::string::npos ? std::string::npos : endline_pos - pos); + if (line.empty()) break; + + std::string::size_type name_endline_pos = line.find(':'); + if (name_endline_pos == std::string::npos) { + throw std::runtime_error("Cannot parse param from string: " + line + + ". Name and value should be separated by \":\"" ); + } + + std::string name = line.substr(0, name_endline_pos); + std::string value = line.substr(name_endline_pos + 2); + + ParamCreator creator; + if (name == "mfxImplDescription.Impl") { + ret.push_back(creator.create(name, cstr_to_mfx_impl(value.c_str()))); + } else if (name == "mfxImplDescription.mfxDecoderDescription.decoder.CodecID") { + ret.push_back(creator.create(name, cstr_to_mfx_codec_id(value.c_str()))); + } else if (name == "mfxImplDescription.AccelerationMode") { + ret.push_back(creator.create(name, cstr_to_mfx_accel_mode(value.c_str()))); + } else if (name == "mfxImplDescription.ApiVersion.Version") { + ret.push_back(creator.create(name, cstr_to_mfx_version(value.c_str()))); + } else { + GAPI_LOG_DEBUG(nullptr, "Cannot parse configuration param, name: " << name << + ", value: " << value); + } + + pos = endline_pos + 1; + } + while (endline_pos != std::string::npos); + + return ret; +} + +template +std::vector get_params_from_string(const std::string& str); +template +std::vector get_params_from_string(const std::string& str); + +mfxVariant cfg_param_to_mfx_variant(const CfgParam& cfg_val) { + const CfgParam::name_t& name = cfg_val.get_name(); + mfxVariant ret; + cv::util::visit(cv::util::overload_lambdas( + [&ret](uint8_t value) { ret.Type = MFX_VARIANT_TYPE_U8; ret.Data.U8 = value; }, + [&ret](int8_t value) { ret.Type = MFX_VARIANT_TYPE_I8; ret.Data.I8 = value; }, + [&ret](uint16_t value) { ret.Type = MFX_VARIANT_TYPE_U16; ret.Data.U16 = value; }, + [&ret](int16_t value) { ret.Type = MFX_VARIANT_TYPE_I16; ret.Data.I16 = value; }, + [&ret](uint32_t value) { ret.Type = MFX_VARIANT_TYPE_U32; ret.Data.U32 = value; }, + [&ret](int32_t value) { ret.Type = MFX_VARIANT_TYPE_I32; ret.Data.I32 = value; }, + [&ret](uint64_t value) { ret.Type = MFX_VARIANT_TYPE_U64; ret.Data.U64 = value; }, + [&ret](int64_t value) { ret.Type = MFX_VARIANT_TYPE_I64; ret.Data.I64 = value; }, + [&ret](float_t value) { ret.Type = MFX_VARIANT_TYPE_F32; ret.Data.F32 = value; }, + [&ret](double_t value) { ret.Type = MFX_VARIANT_TYPE_F64; ret.Data.F64 = value; }, + [&ret](void* value) { ret.Type = MFX_VARIANT_TYPE_PTR; ret.Data.Ptr = value; }, + [&ret, &name] (const std::string& value) { + auto parsed = get_params_from_string(name + ": " + value + "\n"); + if (parsed.empty()) { + throw std::logic_error("Unsupported parameter, name: " + name + ", value: " + value); + } + ret = *parsed.begin(); + }), cfg_val.get_value()); + return ret; +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/cfg_params_parser.hpp b/modules/gapi/src/streaming/onevpl/cfg_params_parser.hpp new file mode 100644 index 0000000000..6247eef916 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/cfg_params_parser.hpp @@ -0,0 +1,43 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_CFG_PARAM_PARSER_HPP +#define GAPI_STREAMING_ONEVPL_CFG_PARAM_PARSER_HPP + +#ifdef HAVE_ONEVPL +#if (MFX_VERSION >= 2000) +#include +#endif // MFX_VERSION + +#include +#include + +#include +#include + +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +template +std::vector get_params_from_string(const std::string& str); + +template +struct ParamCreator { + template + ReturnType create(const std::string& name, ValueType&& value); +}; + +mfxVariant cfg_param_to_mfx_variant(const CfgParam& value); +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_CFG_PARAM_PARSER_HPP diff --git a/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp index 3161b1627e..382c3ae88b 100644 --- a/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp +++ b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp @@ -117,7 +117,7 @@ mfxStatus ReadEncodedStream(mfxBitstream &bs, std::shared_ptr& da return MFX_ERR_NOT_ENOUGH_BUFFER; } - std::copy_n(p0, bs.DataLength, p1); + std::copy_n(p1, bs.DataLength, p0); bs.DataOffset = 0; bs.DataLength += static_cast(data_provider->fetch_data(bs.MaxLength - bs.DataLength, diff --git a/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp index 7d4869ac66..b307c18112 100644 --- a/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp +++ b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp @@ -11,6 +11,8 @@ #include "streaming/onevpl/engine/engine_session.hpp" #include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS +#ifdef HAVE_ONEVPL + namespace cv { namespace gapi { namespace wip { @@ -93,4 +95,5 @@ mfxStatus ReadEncodedStream(mfxBitstream &bs, std::shared_ptr& da } // namespace gapi } // namespace cv +#endif // HAVE_ONEVPL #endif // GAPI_STREAMING_ONEVPL_ENGINE_PROCESSING_ENGINE_BASE_HPP diff --git a/modules/gapi/src/streaming/onevpl/source_priv.cpp b/modules/gapi/src/streaming/onevpl/source_priv.cpp index ed838d6adc..28d438a947 100644 --- a/modules/gapi/src/streaming/onevpl/source_priv.cpp +++ b/modules/gapi/src/streaming/onevpl/source_priv.cpp @@ -7,6 +7,12 @@ #include #include +#include "streaming/onevpl/engine/decode/decode_engine_legacy.hpp" +#include "streaming/onevpl/accelerators/accel_policy_dx11.hpp" +#include "streaming/onevpl/accelerators/accel_policy_cpu.hpp" +#include "streaming/onevpl/utils.hpp" +#include "streaming/onevpl/cfg_params_parser.hpp" + #include "streaming/onevpl/source_priv.hpp" #include "logger.hpp" @@ -32,32 +38,346 @@ namespace cv { namespace gapi { namespace wip { namespace onevpl { + +enum { + VPL_NEW_API_MAJOR_VERSION = 2, + VPL_NEW_API_MINOR_VERSION = 2 +}; + + GSource::Priv::Priv() : - mfx_handle(MFXLoad()) + mfx_handle(MFXLoad()), + mfx_impl_description(), + mfx_handle_configs(), + cfg_params(), + mfx_session(), + description(), + description_is_valid(false), + engine() { GAPI_LOG_INFO(nullptr, "Initialized MFX handle: " << mfx_handle); - description_is_valid = false; } -GSource::Priv::Priv(std::shared_ptr, const std::vector&) : +GSource::Priv::Priv(std::shared_ptr provider, const std::vector& params) : GSource::Priv() { + // Enable Config + if (params.empty()) + { + GAPI_LOG_INFO(nullptr, "No special cfg params requested - use default"); + this->cfg_params = getDefaultCfgParams(); + } + else + { + this->cfg_params = params; + } + + GAPI_LOG_DEBUG(nullptr, "Requested cfg params count: " << cfg_params.size()); + this->mfx_handle_configs.resize(cfg_params.size()); + + // Build VPL handle config from major input params + // VPL dispatcher then uses this config handle to look up for all existing VPL impl + // satisfying major input params and available in the system + GAPI_LOG_INFO(nullptr, "Creating VPL config from input params"); + auto cfg_param_it = cfg_params.begin(); + for (mfxConfig& cfg_inst : mfx_handle_configs) { + cfg_inst = MFXCreateConfig(mfx_handle); + GAPI_Assert(cfg_inst && "MFXCreateConfig failed"); + + if (!cfg_param_it->is_major()) { + GAPI_LOG_DEBUG(nullptr, "Skip not major param: " << cfg_param_it->get_name()); + ++cfg_param_it; + continue; + } + + GAPI_LOG_DEBUG(nullptr, "Apply major param: " << cfg_param_it->get_name()); + mfxVariant mfx_param = cfg_param_to_mfx_variant(*cfg_param_it); + mfxStatus sts = MFXSetConfigFilterProperty(cfg_inst, + (mfxU8 *)cfg_param_it->get_name().c_str(), + mfx_param); + if (sts != MFX_ERR_NONE ) + { + GAPI_LOG_WARNING(nullptr, "MFXSetConfigFilterProperty failed, error: " << + mfxstatus_to_string(sts) << + " - for \"" << cfg_param_it->get_name() << "\""); + GAPI_Assert(false && "MFXSetConfigFilterProperty failed"); + } + + ++cfg_param_it; + } + + // collect optional-preferred input parameters from input params + // which may (optionally) or may not be used to choose the most preferrable + // VPL implementation (for example, specific API version or Debug/Release VPL build) + std::vector preferred_params; + std::copy_if(cfg_params.begin(), cfg_params.end(), std::back_inserter(preferred_params), + [] (const CfgParam& param) { return !param.is_major(); }); + std::sort(preferred_params.begin(), preferred_params.end()); + + GAPI_LOG_DEBUG(nullptr, "Find MFX better implementation from handle: " << mfx_handle << + " is satisfying preferrable params count: " << preferred_params.size()); + int i = 0; + mfxImplDescription *idesc = nullptr; + std::vector available_impl_descriptions; + std::map matches_count; + while (MFX_ERR_NONE == MFXEnumImplementations(mfx_handle, + i, + MFX_IMPLCAPS_IMPLDESCSTRUCTURE, + reinterpret_cast(&idesc))) { + + available_impl_descriptions.push_back(idesc); + + std::stringstream ss; + mfxHDL hImplPath = nullptr; + if (MFX_ERR_NONE == MFXEnumImplementations(mfx_handle, i, MFX_IMPLCAPS_IMPLPATH, &hImplPath)) { + if (hImplPath) { + ss << "Implementation path: " << reinterpret_cast(hImplPath) << std::endl; + MFXDispReleaseImplDescription(mfx_handle, hImplPath); + } + } + ss << *idesc << std::endl; + + GAPI_LOG_INFO(nullptr, "Implementation index: " << i << "\n" << ss.str()); + + // Only one VPL implementation is required for GSource here. + // Let's find intersection params from available impl with preferrable input params + // to find best match. + // An available VPL implementation with max matching count + std::vector impl_params = get_params_from_string(ss.str()); + std::sort(impl_params.begin(), impl_params.end()); + GAPI_LOG_DEBUG(nullptr, "Find implementation cfg params count" << impl_params.size()); + + std::vector matched_params; + std::set_intersection(impl_params.begin(), impl_params.end(), + preferred_params.begin(), preferred_params.end(), + std::back_inserter(matched_params)); + + if (preferred_params.empty()) { + // in case of no input preferrance we consider all params are matched + // for the first available VPL implementation. It will be a chosen one + matches_count.emplace(impl_params.size(), i++); + GAPI_LOG_DEBUG(nullptr, "No preferrable params, use the first one implementation"); + break; + } else { + GAPI_LOG_DEBUG(nullptr, "Equal param intersection count: " << matched_params.size()); + matches_count.emplace(matches_count.size(), i++); + } + } + + // Extract the most suitable VPL implementation by max score + auto max_match_it = matches_count.rbegin(); + GAPI_Assert(max_match_it != matches_count.rend() && + "Cannot find matched MFX implementation for requested configuration"); + + int impl_number = max_match_it->second; + GAPI_LOG_INFO(nullptr, "Chosen implementation index: " << impl_number); + + // release unusable impl available_impl_descriptions + std::swap(mfx_impl_description, available_impl_descriptions[impl_number]); + for (mfxImplDescription* unusable_impl_descr : available_impl_descriptions) { + if (unusable_impl_descr) { + MFXDispReleaseImplDescription(mfx_handle, unusable_impl_descr); + } + } + available_impl_descriptions.clear(); + + // create session for implementation + mfxStatus sts = MFXCreateSession(mfx_handle, impl_number, &mfx_session); + if (MFX_ERR_NONE != sts) { + GAPI_LOG_WARNING(nullptr, "Cannot create MFX Session for implementation index:" << + std::to_string(impl_number) << + ", error: " << mfxstatus_to_string(sts)); + } + + GAPI_LOG_INFO(nullptr, "Initialized MFX session: " << mfx_session); + + // initialize decoder + // Find codec ID from config + auto dec_it = std::find_if(cfg_params.begin(), cfg_params.end(), [] (const CfgParam& value) { + return value.get_name() == "mfxImplDescription.mfxDecoderDescription.decoder.CodecID"; + }); + GAPI_Assert (dec_it != cfg_params.end() && "Cannot determine DecoderID from oneVPL config. Abort"); + + // create session driving engine if required + if (!engine) { + std::unique_ptr acceleration = initializeHWAccel(); + + // TODO Add factory static method in ProcessingEngineBase + if (mfx_impl_description->ApiVersion.Major >= VPL_NEW_API_MAJOR_VERSION) { + GAPI_Assert(false && + "GSource mfx_impl_description->ApiVersion.Major >= VPL_NEW_API_MAJOR_VERSION" + " - is not implemented"); + } else { + engine.reset(new VPLLegacyDecodeEngine(std::move(acceleration))); + } + } + + //create decoder for session accoring to header recovered from source file + DecoderParams decoder_param = create_decoder_from_file(*dec_it, provider); + + // create engine session for processing mfx session pipeline + engine->initialize_session(mfx_session, std::move(decoder_param), + provider); + + //prepare session for processing + engine->process(mfx_session); } GSource::Priv::~Priv() { + GAPI_LOG_INFO(nullptr, "Unload MFX implementation description: " << mfx_impl_description); + MFXDispReleaseImplDescription(mfx_handle, mfx_impl_description); GAPI_LOG_INFO(nullptr, "Unload MFX handle: " << mfx_handle); MFXUnload(mfx_handle); } -bool GSource::Priv::pull(cv::gapi::wip::Data&) +DecoderParams GSource::Priv::create_decoder_from_file(const CfgParam& decoder_cfg, + std::shared_ptr provider) { - return false; + GAPI_DbgAssert(provider && "Cannot create decoder, data provider is nullptr"); + + mfxBitstream bitstream{}; + const int BITSTREAM_BUFFER_SIZE = 2000000; + bitstream.MaxLength = BITSTREAM_BUFFER_SIZE; + bitstream.Data = (mfxU8 *)calloc(bitstream.MaxLength, sizeof(mfxU8)); + if(!bitstream.Data) { + throw std::runtime_error("Cannot allocate bitstream.Data bytes: " + + std::to_string(bitstream.MaxLength * sizeof(mfxU8))); + } + + mfxVariant decoder = cfg_param_to_mfx_variant(decoder_cfg); + // according to oneVPL documentation references + // https://spec.oneapi.io/versions/latest/elements/oneVPL/source/API_ref/VPL_disp_api_struct.html + // mfxVariant is an `union` type and considered different meaning for different param ids + // So CodecId has U32 data type + bitstream.CodecId = decoder.Data.U32; + + mfxStatus sts = ReadEncodedStream(bitstream, provider); + if(MFX_ERR_NONE != sts) { + throw std::runtime_error("Error reading bitstream, error: " + + mfxstatus_to_string(sts)); + } + + // Retrieve the frame information from input stream + mfxVideoParam mfxDecParams {}; + mfxDecParams.mfx.CodecId = decoder.Data.U32; + mfxDecParams.IOPattern = MFX_IOPATTERN_OUT_SYSTEM_MEMORY;//MFX_IOPATTERN_OUT_VIDEO_MEMORY; + sts = MFXVideoDECODE_DecodeHeader(mfx_session, &bitstream, &mfxDecParams); + if(MFX_ERR_NONE != sts) { + throw std::runtime_error("Error decoding header, error: " + + mfxstatus_to_string(sts)); + } + + // Input parameters finished, now initialize decode + sts = MFXVideoDECODE_Init(mfx_session, &mfxDecParams); + if (MFX_ERR_NONE != sts) { + throw std::runtime_error("Error initializing Decode, error: " + + mfxstatus_to_string(sts)); + } + + // set valid description + description.size = cv::Size { + mfxDecParams.mfx.FrameInfo.Width, + mfxDecParams.mfx.FrameInfo.Height}; + switch(mfxDecParams.mfx.FrameInfo.FourCC) { + case MFX_FOURCC_I420: + GAPI_Assert(false && "Cannot create GMetaArg description: " + "MediaFrame doesn't support I420 type"); + case MFX_FOURCC_NV12: + description.fmt = cv::MediaFormat::NV12; + break; + default: + { + GAPI_LOG_WARNING(nullptr, "Cannot create GMetaArg description: " + "MediaFrame unknown 'fmt' type: " << + std::to_string(mfxDecParams.mfx.FrameInfo.FourCC)); + GAPI_Assert(false && "Cannot create GMetaArg description: invalid value"); + } + } + description_is_valid = true; + + return {bitstream, mfxDecParams}; +} + +std::unique_ptr GSource::Priv::initializeHWAccel() +{ + std::unique_ptr ret; + + auto accel_mode_it = std::find_if(cfg_params.begin(), cfg_params.end(), [] (const CfgParam& value) { + return value.get_name() == "mfxImplDescription.AccelerationMode"; + }); + if (accel_mode_it == cfg_params.end()) + { + GAPI_LOG_DEBUG(nullptr, "No HW Accel requested. Use CPU"); + + ret.reset(new VPLCPUAccelerationPolicy); + return ret; + } + + GAPI_LOG_DEBUG(nullptr, "Add HW acceleration support"); + mfxVariant accel_mode = cfg_param_to_mfx_variant(*accel_mode_it); + + switch(accel_mode.Data.U32) { + case MFX_ACCEL_MODE_VIA_D3D11: + { + std::unique_ptr cand(new VPLDX11AccelerationPolicy); + ret = std::move(cand); + break; + } + case MFX_ACCEL_MODE_NA: + { + std::unique_ptr cand(new VPLCPUAccelerationPolicy); + ret = std::move(cand); + break; + } + default: + { + GAPI_LOG_WARNING(nullptr, "Cannot initialize HW Accel: " + "invalid accelerator type: " << + std::to_string(accel_mode.Data.U32)); + GAPI_Assert(false && "Cannot initialize HW Accel"); + } + } + + return ret; +} + +const std::vector& GSource::Priv::getDefaultCfgParams() +{ + static const std::vector def_params = + get_params_from_string( + "mfxImplDescription.Impl: MFX_IMPL_TYPE_HARDWARE\n" + "mfxImplDescription.AccelerationMode: MFX_ACCEL_MODE_VIA_D3D11\n"); + + return def_params; +} + +const std::vector& GSource::Priv::getCfgParams() const +{ + return cfg_params; +} + +bool GSource::Priv::pull(cv::gapi::wip::Data& data) +{ + ProcessingEngineBase::ExecutionStatus status = ProcessingEngineBase::ExecutionStatus::Continue; + while (0 == engine->get_ready_frames_count() && + status == ProcessingEngineBase::ExecutionStatus::Continue) { + status = engine->process(mfx_session); + } + + if (engine->get_ready_frames_count()) { + engine->get_frame(data); + return true; + } else { + return false; + } } GMetaArg GSource::Priv::descr_of() const { - return {}; + GAPI_Assert(description_is_valid); + GMetaArg arg(description); + return arg; } } // namespace onevpl } // namespace wip diff --git a/modules/gapi/src/streaming/onevpl/source_priv.hpp b/modules/gapi/src/streaming/onevpl/source_priv.hpp index c278934253..cdaab4eb6a 100644 --- a/modules/gapi/src/streaming/onevpl/source_priv.hpp +++ b/modules/gapi/src/streaming/onevpl/source_priv.hpp @@ -25,23 +25,44 @@ #include +#include "streaming/onevpl/engine/processing_engine_base.hpp" + namespace cv { namespace gapi { namespace wip { namespace onevpl { +struct VPLAccelerationPolicy; +class ProcessingEngineBase; + struct GSource::Priv { explicit Priv(std::shared_ptr provider, const std::vector& params); ~Priv(); + static const std::vector& getDefaultCfgParams(); + const std::vector& getCfgParams() const; + bool pull(cv::gapi::wip::Data& data); GMetaArg descr_of() const; private: Priv(); + DecoderParams create_decoder_from_file(const CfgParam& decoder, + std::shared_ptr provider); + std::unique_ptr initializeHWAccel(); + mfxLoader mfx_handle; + mfxImplDescription *mfx_impl_description; + std::vector mfx_handle_configs; + std::vector cfg_params; + + mfxSession mfx_session; + + cv::GFrameDesc description; bool description_is_valid; + + std::unique_ptr engine; }; } // namespace onevpl } // namespace wip diff --git a/modules/gapi/src/streaming/onevpl/utils.cpp b/modules/gapi/src/streaming/onevpl/utils.cpp new file mode 100644 index 0000000000..72b5c75303 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/utils.cpp @@ -0,0 +1,454 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include + +#include +#include + +#ifdef HAVE_ONEVPL + +#include "streaming/onevpl/utils.hpp" +#include "logger.hpp" + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +const char* mfx_impl_to_cstr(const mfxIMPL impl) { + switch (impl) { + case MFX_IMPL_TYPE_SOFTWARE: + return "MFX_IMPL_TYPE_SOFTWARE"; + case MFX_IMPL_TYPE_HARDWARE: + return "MFX_IMPL_TYPE_HARDWARE"; + default: + return "unknown mfxIMPL"; + } +} + +mfxIMPL cstr_to_mfx_impl(const char* cstr) { + if (!strcmp(cstr, "MFX_IMPL_TYPE_SOFTWARE")) { + return MFX_IMPL_TYPE_SOFTWARE; + } else if (!strcmp(cstr, "MFX_IMPL_TYPE_HARDWARE")) { + return MFX_IMPL_TYPE_HARDWARE; + } + + throw std::logic_error(std::string("Invalid \"mfxImplDescription.Impl\":") + cstr); +} + +const char* mfx_accel_mode_to_cstr (const mfxAccelerationMode mode) { + switch (mode) { + case MFX_ACCEL_MODE_NA: + return "MFX_ACCEL_MODE_NA"; + case MFX_ACCEL_MODE_VIA_D3D9: + return "MFX_ACCEL_MODE_VIA_D3D9"; + case MFX_ACCEL_MODE_VIA_D3D11: + return "MFX_ACCEL_MODE_VIA_D3D11"; + case MFX_ACCEL_MODE_VIA_VAAPI: + return "MFX_ACCEL_MODE_VIA_VAAPI"; + case MFX_ACCEL_MODE_VIA_VAAPI_DRM_MODESET: + return "MFX_ACCEL_MODE_VIA_VAAPI_DRM_MODESET"; + case MFX_ACCEL_MODE_VIA_VAAPI_GLX: + return "MFX_ACCEL_MODE_VIA_VAAPI_GLX"; + case MFX_ACCEL_MODE_VIA_VAAPI_X11: + return "MFX_ACCEL_MODE_VIA_VAAPI_X11"; + case MFX_ACCEL_MODE_VIA_VAAPI_WAYLAND: + return "MFX_ACCEL_MODE_VIA_VAAPI_WAYLAND"; + case MFX_ACCEL_MODE_VIA_HDDLUNITE: + return "MFX_ACCEL_MODE_VIA_HDDLUNITE"; + default: + return "unknown mfxAccelerationMode"; + } + return "unknown mfxAccelerationMode"; +} + +mfxAccelerationMode cstr_to_mfx_accel_mode(const char* cstr) { + if (!strcmp(cstr, "MFX_ACCEL_MODE_NA")) { + return MFX_ACCEL_MODE_NA; + } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_D3D9")) { + return MFX_ACCEL_MODE_VIA_D3D9; + } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_D3D11")) { + return MFX_ACCEL_MODE_VIA_D3D11; + } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_VAAPI")) { + return MFX_ACCEL_MODE_VIA_VAAPI; + } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_VAAPI_DRM_MODESET")) { + return MFX_ACCEL_MODE_VIA_VAAPI_DRM_MODESET; + } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_VAAPI_GLX")) { + return MFX_ACCEL_MODE_VIA_VAAPI_GLX; + } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_VAAPI_X11")) { + return MFX_ACCEL_MODE_VIA_VAAPI_X11; + } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_VAAPI_WAYLAND")) { + return MFX_ACCEL_MODE_VIA_VAAPI_WAYLAND; + } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_HDDLUNITE")) { + return MFX_ACCEL_MODE_VIA_HDDLUNITE; + } + throw std::logic_error(std::string("Invalid \"mfxImplDescription.AccelerationMode\":") + cstr); +} + +const char* mfx_resource_type_to_cstr (const mfxResourceType type) { + switch (type) { + case MFX_RESOURCE_SYSTEM_SURFACE: + return "MFX_RESOURCE_SYSTEM_SURFACE"; + case MFX_RESOURCE_VA_SURFACE: + return "MFX_RESOURCE_VA_SURFACE"; + case MFX_RESOURCE_VA_BUFFER: + return "MFX_RESOURCE_VA_BUFFER"; + case MFX_RESOURCE_DX9_SURFACE: + return "MFX_RESOURCE_DX9_SURFACE"; + case MFX_RESOURCE_DX11_TEXTURE: + return "MFX_RESOURCE_DX11_TEXTURE"; + case MFX_RESOURCE_DX12_RESOURCE: + return "MFX_RESOURCE_DX12_RESOURCE"; + case MFX_RESOURCE_DMA_RESOURCE: + return "MFX_RESOURCE_DMA_RESOURCE"; + case MFX_RESOURCE_HDDLUNITE_REMOTE_MEMORY: + return "MFX_RESOURCE_HDDLUNITE_REMOTE_MEMORY"; + default: + return "unknown mfxResourceType"; + } +} + +mfxResourceType cstr_to_mfx_resource_type(const char* cstr) { + if (!strcmp(cstr, "MFX_RESOURCE_SYSTEM_SURFACE")) { + return MFX_RESOURCE_SYSTEM_SURFACE; + } else if (!strcmp(cstr, "MFX_RESOURCE_VA_SURFACE")) { + return MFX_RESOURCE_VA_SURFACE; + } else if (!strcmp(cstr, "MFX_RESOURCE_VA_BUFFER")) { + return MFX_RESOURCE_VA_BUFFER; + } else if (!strcmp(cstr, "MFX_RESOURCE_DX9_SURFACE")) { + return MFX_RESOURCE_DX9_SURFACE; + } else if (!strcmp(cstr, "MFX_RESOURCE_DX11_TEXTURE")) { + return MFX_RESOURCE_DX11_TEXTURE; + } else if (!strcmp(cstr, "MFX_RESOURCE_DX12_RESOURCE")) { + return MFX_RESOURCE_DX12_RESOURCE; + } else if (!strcmp(cstr, "MFX_RESOURCE_DMA_RESOURCE")) { + return MFX_RESOURCE_DMA_RESOURCE; + } else if (!strcmp(cstr, "MFX_RESOURCE_HDDLUNITE_REMOTE_MEMORY")) { + return MFX_RESOURCE_HDDLUNITE_REMOTE_MEMORY; + } + throw std::logic_error(std::string("Invalid \"decoder.Profiles.MemDesc.MemHandleType\":") + cstr); +} + +mfxU32 cstr_to_mfx_codec_id(const char* cstr) { + if (!strcmp(cstr, "MFX_CODEC_AVC")) { + return MFX_CODEC_AVC; + } else if (!strcmp(cstr, "MFX_CODEC_HEVC")) { + return MFX_CODEC_HEVC; + } else if (!strcmp(cstr, "MFX_CODEC_MPEG2")) { + return MFX_CODEC_MPEG2; + } else if (!strcmp(cstr, "MFX_CODEC_VC1")) { + return MFX_CODEC_VC1; + } else if (!strcmp(cstr, "MFX_CODEC_CAPTURE")) { + return MFX_CODEC_CAPTURE; + } else if (!strcmp(cstr, "MFX_CODEC_VP9")) { + return MFX_CODEC_VP9; + } else if (!strcmp(cstr, "MFX_CODEC_AV1")) { + return MFX_CODEC_AV1; + } + throw std::logic_error(std::string("Cannot parse \"mfxImplDescription.mfxDecoderDescription.decoder.CodecID\" value: ") + cstr); +} + +const char* mfx_codec_type_to_cstr(const mfxU32 fourcc, const mfxU32 type) { + switch (fourcc) { + case MFX_CODEC_JPEG: { + switch (type) { + case MFX_PROFILE_UNKNOWN: + return "MFX_PROFILE_UNKNOWN"; + case MFX_PROFILE_JPEG_BASELINE: + return "MFX_PROFILE_JPEG_BASELINE"; + default: + return "::max(); + } + + const char* delim = strchr(cstr, '.'); + if (!delim) { + // in digital form - return as is + return std::stoul(cstr, nullptr, 10); + } + std::string major (cstr, delim - cstr); + std::string minor (delim + 1); + mfxU32 major_val = std::stoul(major, nullptr, 10); + mfxU32 minor_val = std::stoul(minor, nullptr, 10); + + // pack to digital form + return {major_val << 16 | minor_val}; +} + +std::ostream& operator<< (std::ostream& out, const mfxImplDescription& idesc) +{ + out << "mfxImplDescription.Version: " << static_cast(idesc.Version.Major) + << "." << static_cast(idesc.Version.Minor) << std::endl; + out << "mfxImplDescription.Impl: " << mfx_impl_to_cstr(idesc.Impl) << std::endl; + out << "(*)mfxImplDescription.AccelerationMode: " << mfx_accel_mode_to_cstr(idesc.AccelerationMode) << std::endl; + out << "mfxImplDescription.ApiVersion: " << idesc.ApiVersion.Major << "." << idesc.ApiVersion.Minor << std::endl; + out << "(*)mfxImplDescription.ApiVersion.Version: " << idesc.ApiVersion.Version << std::endl; + out << "mfxImplDescription.ImplName: " << idesc.ImplName << std::endl; + out << "mfxImplDescription.License: " << idesc.License << std::endl; + out << "mfxImplDescription.Keywords: " << idesc.Keywords << std::endl; + out << "mfxImplDescription.VendorID: " << idesc.VendorID << std::endl; + out << "mfxImplDescription.VendorImplID: " << idesc.VendorImplID << std::endl; + + const mfxAccelerationModeDescription &accel = idesc.AccelerationModeDescription; + out << "mfxImplDescription.mfxAccelerationMode.Version: " << static_cast(accel.Version.Major) + << "." << static_cast(accel.Version.Minor) << std::endl; + for (int mode = 0; mode < accel.NumAccelerationModes; mode++) { + out << "mfxImplDescription.mfxAccelerationMode.Mode: " << mfx_accel_mode_to_cstr(accel.Mode[mode]) << std::endl; + } + + const mfxDeviceDescription &dev = idesc.Dev; + out << "mfxImplDescription.mfxDeviceDescription.Version: " << static_cast(dev.Version.Major) + << "." << static_cast(dev.Version.Minor) << std::endl; + out << "mfxImplDescription.mfxDeviceDescription.DeviceID: " << dev.DeviceID << std::endl; + for (int subdevice = 0; subdevice < dev.NumSubDevices; subdevice++) { + out << "mfxImplDescription.mfxDeviceDescription.subdevices.Index: " << dev.SubDevices[subdevice].Index << std::endl; + out << "mfxImplDescription.mfxDeviceDescription.subdevices.SubDeviceID: " << dev.SubDevices[subdevice].SubDeviceID << std::endl; + } + + /* mfxDecoderDescription */ + const mfxDecoderDescription &dec = idesc.Dec; + out << "mfxImplDescription.mfxDecoderDescription.Version: " << static_cast(dec.Version.Major) + << "." << static_cast(dec.Version.Minor) << std::endl; + for (int codec = 0; codec < dec.NumCodecs; codec++) { + auto cid = dec.Codecs[codec].CodecID; + out << "(*)mfxImplDescription.mfxDecoderDescription.decoder.CodecID: " << cid;//(cid & 0xff) << "." << (cid >> 8 & 0xff) << "." << (cid >> 16 & 0xff) << "." << (cid >> 24 & 0xff) << std::endl; + out << "mfxImplDescription.mfxDecoderDescription.decoder.MaxcodecLevel: " << dec.Codecs[codec].MaxcodecLevel << std::endl; + for (int profile = 0; profile < dec.Codecs[codec].NumProfiles; profile++) { + out << "mfxImplDescription.mfxDecoderDescription.decoder.Profiles: " + << mfx_codec_type_to_cstr(dec.Codecs[codec].CodecID, + dec.Codecs[codec].Profiles[profile].Profile) << std::endl; + for (int memtype = 0; memtype < dec.Codecs[codec].Profiles[profile].NumMemTypes; memtype++) { + out << "mfxImplDescription.mfxDecoderDescription.decoder.Profiles.MemDesc.MemHandleType: " + << mfx_resource_type_to_cstr(dec.Codecs[codec].Profiles[profile].MemDesc[memtype].MemHandleType) << std::endl; + out << "mfxImplDescription.mfxDecoderDescription.decoder.Profiles.MemDesc.Width.Min: " + << dec.Codecs[codec].Profiles[profile].MemDesc[memtype].Width.Min << std::endl; + out << "mfxImplDescription.mfxDecoderDescription.decoder.Profiles.MemDesc.Width.Max: " + << dec.Codecs[codec].Profiles[profile].MemDesc[memtype].Width.Max << std::endl; + out << "mfxImplDescription.mfxDecoderDescription.decoder.Profiles.MemDesc.Width.Step: " + << dec.Codecs[codec].Profiles[profile].MemDesc[memtype].Width.Step << std::endl; + out << "mfxImplDescription.mfxDecoderDescription.decoder.Profiles.MemDesc.Height.Min: " + << dec.Codecs[codec].Profiles[profile].MemDesc[memtype].Height.Min << std::endl; + out << "mfxImplDescription.mfxDecoderDescription.decoder.Profiles.MemDesc.Height.Max: " + << dec.Codecs[codec].Profiles[profile].MemDesc[memtype].Height.Max << std::endl; + out << "mfxImplDescription.mfxDecoderDescription.decoder.Profiles.MemDesc.Height.Step: " + << dec.Codecs[codec].Profiles[profile].MemDesc[memtype].Height.Step << std::endl; + } + } + } + + out << "mfxImplDescription.NumExtParam: " << idesc.NumExtParam << std::endl; + + out << "\n(*) - configurable params" << std::endl; + return out; +} + +std::string mfxstatus_to_string(mfxStatus err) { + switch(err) + { + case MFX_ERR_NONE: + return "MFX_ERR_NONE"; + case MFX_ERR_UNKNOWN: + return "MFX_ERR_UNKNOWN"; + case MFX_ERR_NULL_PTR: + return "MFX_ERR_NULL_PTR"; + case MFX_ERR_UNSUPPORTED: + return "MFX_ERR_UNSUPPORTED"; + case MFX_ERR_MEMORY_ALLOC: + return "MFX_ERR_MEMORY_ALLOC"; + case MFX_ERR_NOT_ENOUGH_BUFFER: + return "MFX_ERR_NOT_ENOUGH_BUFFER"; + case MFX_ERR_INVALID_HANDLE: + return "MFX_ERR_INVALID_HANDLE"; + case MFX_ERR_LOCK_MEMORY: + return "MFX_ERR_LOCK_MEMORY"; + case MFX_ERR_NOT_INITIALIZED: + return "MFX_ERR_NOT_INITIALIZED"; + case MFX_ERR_NOT_FOUND: + return "MFX_ERR_NOT_FOUND"; + case MFX_ERR_MORE_DATA: + return "MFX_ERR_MORE_DATA"; + case MFX_ERR_MORE_SURFACE: + return "MFX_ERR_MORE_SURFACE"; + case MFX_ERR_ABORTED: + return "MFX_ERR_ABORTED"; + case MFX_ERR_DEVICE_LOST: + return "MFX_ERR_DEVICE_LOST"; + case MFX_ERR_INCOMPATIBLE_VIDEO_PARAM: + return "MFX_ERR_INCOMPATIBLE_VIDEO_PARAM"; + case MFX_ERR_INVALID_VIDEO_PARAM: + return "MFX_ERR_INVALID_VIDEO_PARAM"; + case MFX_ERR_UNDEFINED_BEHAVIOR: + return "MFX_ERR_UNDEFINED_BEHAVIOR"; + case MFX_ERR_DEVICE_FAILED: + return "MFX_ERR_DEVICE_FAILED"; + case MFX_ERR_MORE_BITSTREAM: + return "MFX_ERR_MORE_BITSTREAM"; + case MFX_ERR_GPU_HANG: + return "MFX_ERR_GPU_HANG"; + case MFX_ERR_REALLOC_SURFACE: + return "MFX_ERR_REALLOC_SURFACE"; + case MFX_ERR_RESOURCE_MAPPED: + return "MFX_ERR_RESOURCE_MAPPED"; + case MFX_ERR_NOT_IMPLEMENTED: + return "MFX_ERR_NOT_IMPLEMENTED"; + case MFX_WRN_DEVICE_BUSY: + return "MFX_WRN_DEVICE_BUSY"; + case MFX_WRN_VIDEO_PARAM_CHANGED: + return "MFX_WRN_VIDEO_PARAM_CHANGED"; + + + default: + break; + } + + std::string ret(""; + return ret; +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/utils.hpp b/modules/gapi/src/streaming/onevpl/utils.hpp index 0512c4f687..94f5a249e8 100644 --- a/modules/gapi/src/streaming/onevpl/utils.hpp +++ b/modules/gapi/src/streaming/onevpl/utils.hpp @@ -26,9 +26,27 @@ namespace gapi { namespace wip { namespace onevpl { -inline std::string mfxstatus_to_string(mfxStatus) { - return "UNKNOWN"; -} +const char* mfx_impl_to_cstr(const mfxIMPL impl); + +mfxIMPL cstr_to_mfx_impl(const char* cstr); + +const char* mfx_accel_mode_to_cstr (const mfxAccelerationMode mode); + +mfxAccelerationMode cstr_to_mfx_accel_mode(const char* cstr); + +const char* mfx_resource_type_to_cstr (const mfxResourceType type); + +mfxResourceType cstr_to_mfx_resource_type(const char* cstr); + +mfxU32 cstr_to_mfx_codec_id(const char* cstr); + +const char* mfx_codec_type_to_cstr(const mfxU32 fourcc, const mfxU32 type); + +mfxU32 cstr_to_mfx_version(const char* cstr); + +std::string mfxstatus_to_string(mfxStatus err); + +std::ostream& operator<< (std::ostream& out, const mfxImplDescription& idesc); } // namespace onevpl } // namespace wip From b5a9a6793b3622b01fe6c7b025f85074fec99491 Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Mon, 18 Oct 2021 19:31:48 +0300 Subject: [PATCH 006/226] Merge pull request #20856 from TolyaTalamanov:at/cfg-batch-size G-API: Extend ie::Params to specify batch size * Add cfgBatchSize to ie::Params * Fix comments to review --- .../opencv2/gapi/infer/bindings_ie.hpp | 3 ++ .../gapi/include/opencv2/gapi/infer/ie.hpp | 31 +++++++++-- modules/gapi/src/backends/ie/bindings_ie.cpp | 6 +++ modules/gapi/src/backends/ie/giebackend.cpp | 5 +- modules/gapi/src/backends/ie/util.hpp | 4 +- .../gapi/test/infer/gapi_infer_ie_test.cpp | 54 ++++++++++++++++++- 6 files changed, 94 insertions(+), 9 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/infer/bindings_ie.hpp b/modules/gapi/include/opencv2/gapi/infer/bindings_ie.hpp index 92ef2101a1..94272dea55 100644 --- a/modules/gapi/include/opencv2/gapi/infer/bindings_ie.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/bindings_ie.hpp @@ -44,6 +44,9 @@ public: GAPI_WRAP PyParams& cfgNumRequests(size_t nireq); + GAPI_WRAP + PyParams& cfgBatchSize(const size_t size); + GBackend backend() const; std::string tag() const; cv::util::any params() const; diff --git a/modules/gapi/include/opencv2/gapi/infer/ie.hpp b/modules/gapi/include/opencv2/gapi/infer/ie.hpp index 2be739e518..0e9c127fa1 100644 --- a/modules/gapi/include/opencv2/gapi/infer/ie.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/ie.hpp @@ -79,6 +79,8 @@ struct ParamDesc { // NB: An optional config to setup RemoteContext for IE cv::util::any context_config; + + size_t batch_size; }; } // namespace detail @@ -120,7 +122,8 @@ public: , {} , {} , 1u - , {}} { + , {} + , 1u} { }; /** @overload @@ -141,7 +144,8 @@ public: , {} , {} , 1u - , {}} { + , {} + , 1u} { }; /** @brief Specifies sequence of network input layers names for inference. @@ -316,6 +320,19 @@ public: return *this; } + /** @brief Specifies the inference batch size. + + The function is used to specify inference batch size. + Follow https://docs.openvinotoolkit.org/latest/classInferenceEngine_1_1CNNNetwork.html#a8e9d19270a48aab50cb5b1c43eecb8e9 for additional information + + @param size batch size which will be used. + @return reference to this parameter structure. + */ + Params& cfgBatchSize(const size_t size) { + desc.batch_size = size; + return *this; + } + // BEGIN(G-API's network parametrization API) GBackend backend() const { return cv::gapi::ie::backend(); } std::string tag() const { return Net::tag(); } @@ -350,7 +367,7 @@ public: const std::string &device) : desc{ model, weights, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Load, true, {}, {}, {}, 1u, - {}}, + {}, 1u}, m_tag(tag) { }; @@ -368,7 +385,7 @@ public: const std::string &device) : desc{ model, {}, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Import, true, {}, {}, {}, 1u, - {}}, + {}, 1u}, m_tag(tag) { }; @@ -435,6 +452,12 @@ public: return *this; } + /** @see ie::Params::cfgBatchSize */ + Params& cfgBatchSize(const size_t size) { + desc.batch_size = size; + return *this; + } + // BEGIN(G-API's network parametrization API) GBackend backend() const { return cv::gapi::ie::backend(); } std::string tag() const { return m_tag; } diff --git a/modules/gapi/src/backends/ie/bindings_ie.cpp b/modules/gapi/src/backends/ie/bindings_ie.cpp index 5874fe1378..39f07d28e5 100644 --- a/modules/gapi/src/backends/ie/bindings_ie.cpp +++ b/modules/gapi/src/backends/ie/bindings_ie.cpp @@ -49,3 +49,9 @@ cv::gapi::ie::PyParams& cv::gapi::ie::PyParams::cfgNumRequests(size_t nireq) { m_priv->cfgNumRequests(nireq); return *this; } + +cv::gapi::ie::PyParams& +cv::gapi::ie::PyParams::cfgBatchSize(const size_t size) { + m_priv->cfgBatchSize(size); + return *this; +} diff --git a/modules/gapi/src/backends/ie/giebackend.cpp b/modules/gapi/src/backends/ie/giebackend.cpp index 03584c9561..c69302baca 100644 --- a/modules/gapi/src/backends/ie/giebackend.cpp +++ b/modules/gapi/src/backends/ie/giebackend.cpp @@ -237,6 +237,7 @@ struct IEUnit { if (params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { net = cv::gimpl::ie::wrap::readNetwork(params); + net.setBatchSize(params.batch_size); inputs = net.getInputsInfo(); outputs = net.getOutputsInfo(); } else if (params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Import) { @@ -1412,11 +1413,11 @@ std::vector cv::gapi::ie::util::to_ocv(const IE::SizeVector &dims) { return toCV(dims); } -IE::Blob::Ptr cv::gapi::ie::util::to_ie(cv::Mat &blob) { +IE::Blob::Ptr cv::gapi::ie::util::to_ie(const cv::Mat &blob) { return wrapIE(blob, cv::gapi::ie::TraitAs::IMAGE); } -IE::Blob::Ptr cv::gapi::ie::util::to_ie(cv::Mat &y_plane, cv::Mat &uv_plane) { +IE::Blob::Ptr cv::gapi::ie::util::to_ie(const cv::Mat &y_plane, const cv::Mat &uv_plane) { auto y_blob = wrapIE(y_plane, cv::gapi::ie::TraitAs::IMAGE); auto uv_blob = wrapIE(uv_plane, cv::gapi::ie::TraitAs::IMAGE); #if INF_ENGINE_RELEASE >= 2021010000 diff --git a/modules/gapi/src/backends/ie/util.hpp b/modules/gapi/src/backends/ie/util.hpp index 080c88498f..e2f1f44140 100644 --- a/modules/gapi/src/backends/ie/util.hpp +++ b/modules/gapi/src/backends/ie/util.hpp @@ -27,8 +27,8 @@ namespace util { // test suite only. GAPI_EXPORTS std::vector to_ocv(const InferenceEngine::SizeVector &dims); GAPI_EXPORTS cv::Mat to_ocv(InferenceEngine::Blob::Ptr blob); -GAPI_EXPORTS InferenceEngine::Blob::Ptr to_ie(cv::Mat &blob); -GAPI_EXPORTS InferenceEngine::Blob::Ptr to_ie(cv::Mat &y_plane, cv::Mat &uv_plane); +GAPI_EXPORTS InferenceEngine::Blob::Ptr to_ie(const cv::Mat &blob); +GAPI_EXPORTS InferenceEngine::Blob::Ptr to_ie(const cv::Mat &y_plane, const cv::Mat &uv_plane); }}}} diff --git a/modules/gapi/test/infer/gapi_infer_ie_test.cpp b/modules/gapi/test/infer/gapi_infer_ie_test.cpp index b7ea891b81..49cf47048e 100644 --- a/modules/gapi/test/infer/gapi_infer_ie_test.cpp +++ b/modules/gapi/test/infer/gapi_infer_ie_test.cpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2019-2020 Intel Corporation +// Copyright (C) 2019-2021 Intel Corporation #include "../test_precomp.hpp" @@ -2187,6 +2187,58 @@ TEST_F(LimitedSourceInfer, ReleaseFrameAsync) run(max_frames, resources_limit, nireq); } +TEST(TestAgeGenderIE, InferWithBatch) +{ + initDLDTDataPath(); + + constexpr int batch_size = 4; + cv::gapi::ie::detail::ParamDesc params; + params.model_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml"); + params.weights_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin"); + params.device_id = "CPU"; + + cv::Mat in_mat({batch_size, 3, 320, 240}, CV_8U); + cv::randu(in_mat, 0, 255); + + cv::Mat gapi_age, gapi_gender; + + // Load & run IE network + IE::Blob::Ptr ie_age, ie_gender; + { + auto plugin = cv::gimpl::ie::wrap::getPlugin(params); + auto net = cv::gimpl::ie::wrap::readNetwork(params); + setNetParameters(net); + net.setBatchSize(batch_size); + auto this_network = cv::gimpl::ie::wrap::loadNetwork(plugin, net, params); + auto infer_request = this_network.CreateInferRequest(); + infer_request.SetBlob("data", cv::gapi::ie::util::to_ie(in_mat)); + infer_request.Infer(); + ie_age = infer_request.GetBlob("age_conv3"); + ie_gender = infer_request.GetBlob("prob"); + } + + // Configure & run G-API + using AGInfo = std::tuple; + G_API_NET(AgeGender, , "test-age-gender"); + + cv::GMat in; + cv::GMat age, gender; + std::tie(age, gender) = cv::gapi::infer(in); + cv::GComputation comp(cv::GIn(in), cv::GOut(age, gender)); + + auto pp = cv::gapi::ie::Params { + params.model_path, params.weights_path, params.device_id + }.cfgOutputLayers({ "age_conv3", "prob" }) + .cfgBatchSize(batch_size); + + comp.apply(cv::gin(in_mat), cv::gout(gapi_age, gapi_gender), + cv::compile_args(cv::gapi::networks(pp))); + + // Validate with IE itself (avoid DNN module dependency here) + normAssert(cv::gapi::ie::util::to_ocv(ie_age), gapi_age, "Test age output" ); + normAssert(cv::gapi::ie::util::to_ocv(ie_gender), gapi_gender, "Test gender output"); +} + } // namespace opencv_test #endif // HAVE_INF_ENGINE From b5fcb06a7612e2f0a1f9c0ea2820a34ef0da4206 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 18 Oct 2021 07:15:15 +0000 Subject: [PATCH 007/226] core(SIMD): update int64 SSE constructor --- .../include/opencv2/core/hal/intrin_sse.hpp | 12 ++++++++++++ modules/core/test/test_intrin_utils.hpp | 17 +++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/modules/core/include/opencv2/core/hal/intrin_sse.hpp b/modules/core/include/opencv2/core/hal/intrin_sse.hpp index f4b43a2d7a..2244717e19 100644 --- a/modules/core/include/opencv2/core/hal/intrin_sse.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_sse.hpp @@ -244,7 +244,13 @@ struct v_uint64x2 explicit v_uint64x2(__m128i v) : val(v) {} v_uint64x2(uint64 v0, uint64 v1) { +#if defined(_MSC_VER) && _MSC_VER >= 1920/*MSVS 2019*/ && defined(_M_X64) + val = _mm_setr_epi64x((int64_t)v0, (int64_t)v1); +#elif defined(__GNUC__) + val = _mm_setr_epi64((__m64)v0, (__m64)v1); +#else val = _mm_setr_epi32((int)v0, (int)(v0 >> 32), (int)v1, (int)(v1 >> 32)); +#endif } uint64 get0() const @@ -272,7 +278,13 @@ struct v_int64x2 explicit v_int64x2(__m128i v) : val(v) {} v_int64x2(int64 v0, int64 v1) { +#if defined(_MSC_VER) && _MSC_VER >= 1920/*MSVS 2019*/ && defined(_M_X64) + val = _mm_setr_epi64x((int64_t)v0, (int64_t)v1); +#elif defined(__GNUC__) + val = _mm_setr_epi64((__m64)v0, (__m64)v1); +#else val = _mm_setr_epi32((int)v0, (int)(v0 >> 32), (int)v1, (int)(v1 >> 32)); +#endif } int64 get0() const diff --git a/modules/core/test/test_intrin_utils.hpp b/modules/core/test/test_intrin_utils.hpp index 5c22caaf12..3f196f1342 100644 --- a/modules/core/test/test_intrin_utils.hpp +++ b/modules/core/test/test_intrin_utils.hpp @@ -373,6 +373,23 @@ template struct TheTest EXPECT_EQ((LaneType)12, vx_setall_res2_[i]); } +#if CV_SIMD_WIDTH == 16 + { + uint64 a = CV_BIG_INT(0x7fffffffffffffff); + uint64 b = (uint64)CV_BIG_INT(0xcfffffffffffffff); + v_uint64x2 uint64_vec(a, b); + EXPECT_EQ(a, uint64_vec.get0()); + EXPECT_EQ(b, v_extract_n<1>(uint64_vec)); + } + { + int64 a = CV_BIG_INT(0x7fffffffffffffff); + int64 b = CV_BIG_INT(-1); + v_int64x2 int64_vec(a, b); + EXPECT_EQ(a, int64_vec.get0()); + EXPECT_EQ(b, v_extract_n<1>(int64_vec)); + } +#endif + return *this; } From 9a9e457dd6b1a1ad1a3d6dc293eba57b1907810e Mon Sep 17 00:00:00 2001 From: Michel Promonet Date: Mon, 18 Oct 2021 18:30:13 +0200 Subject: [PATCH 008/226] Allow to set av_log_set_level to reduce ffmpeg level below AV_LOG_ERROR --- modules/videoio/src/cap_ffmpeg_impl.hpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/modules/videoio/src/cap_ffmpeg_impl.hpp b/modules/videoio/src/cap_ffmpeg_impl.hpp index 937d348215..6877a963ef 100644 --- a/modules/videoio/src/cap_ffmpeg_impl.hpp +++ b/modules/videoio/src/cap_ffmpeg_impl.hpp @@ -850,6 +850,7 @@ static void ffmpeg_log_callback(void *ptr, int level, const char *fmt, va_list v static bool skip_header = false; static int prev_level = -1; CV_UNUSED(ptr); + if (level>av_log_get_level()) return; if (!skip_header || level != prev_level) printf("[OPENCV:FFMPEG:%02d] ", level); vprintf(fmt, vargs); size_t fmt_len = strlen(fmt); @@ -866,15 +867,21 @@ class InternalFFMpegRegister static void initLogger_() { - #ifndef NO_GETENV +#ifndef NO_GETENV char* debug_option = getenv("OPENCV_FFMPEG_DEBUG"); - if (debug_option != NULL) + char* level_option = getenv("OPENCV_FFMPEG_LOGLEVEL"); + int level = AV_LOG_VERBOSE; + if (level_option != NULL) { - av_log_set_level(AV_LOG_VERBOSE); + level = atoi(level_option); + } + if ( (debug_option != NULL) || (level_option != NULL) ) + { + av_log_set_level(level); av_log_set_callback(ffmpeg_log_callback); } else - #endif +#endif { av_log_set_level(AV_LOG_ERROR); } From 6d5fdfbf73070e79bb6b061fd96827e8d43b037f Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 19 Oct 2021 09:28:12 +0000 Subject: [PATCH 009/226] samples: fix build without threading support --- samples/CMakeLists.txt | 1 + .../videoio/orbbec_astra/orbbec_astra.cpp | 13 +++++++++++++ samples/dnn/object_detection.cpp | 16 ++++++++++------ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 9d8f588083..910454ca71 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -120,6 +120,7 @@ else() find_package(Threads) endif() if((TARGET Threads::Threads OR HAVE_THREADS) AND NOT OPENCV_EXAMPLES_DISABLE_THREADS) + set(HAVE_THREADS 1) add_definitions(-DHAVE_THREADS=1) endif() diff --git a/samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp b/samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp index 581b1768f1..6bb9f904af 100644 --- a/samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp +++ b/samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp @@ -4,6 +4,17 @@ #include #include + + +#if !defined(HAVE_THREADS) +int main() +{ + std::cout << "This sample is built without threading support. Sample code is disabled." << std::endl; + return 0; +} +#else + + #include #include #include @@ -200,3 +211,5 @@ int main() return 0; } + +#endif diff --git a/samples/dnn/object_detection.cpp b/samples/dnn/object_detection.cpp index 5ff112fe5d..6fc8b2ab61 100644 --- a/samples/dnn/object_detection.cpp +++ b/samples/dnn/object_detection.cpp @@ -5,7 +5,11 @@ #include #include -#ifdef CV_CXX11 +#if defined(CV_CXX11) && defined(HAVE_THREADS) +#define USE_THREADS 1 +#endif + +#ifdef USE_THREADS #include #include #include @@ -56,7 +60,7 @@ void drawPred(int classId, float conf, int left, int top, int right, int bottom, void callback(int pos, void* userdata); -#ifdef CV_CXX11 +#ifdef USE_THREADS template class QueueFPS : public std::queue { @@ -106,7 +110,7 @@ private: TickMeter tm; std::mutex mutex; }; -#endif // CV_CXX11 +#endif // USE_THREADS int main(int argc, char** argv) { @@ -171,7 +175,7 @@ int main(int argc, char** argv) else cap.open(parser.get("device")); -#ifdef CV_CXX11 +#ifdef USE_THREADS bool process = true; // Frames capturing thread @@ -271,7 +275,7 @@ int main(int argc, char** argv) framesThread.join(); processingThread.join(); -#else // CV_CXX11 +#else // USE_THREADS if (asyncNumReq) CV_Error(Error::StsNotImplemented, "Asynchronous forward is supported only with Inference Engine backend."); @@ -302,7 +306,7 @@ int main(int argc, char** argv) imshow(kWinName, frame); } -#endif // CV_CXX11 +#endif // USE_THREADS return 0; } From b3f966e2ca82475837bbcfeccae72d9623a295c2 Mon Sep 17 00:00:00 2001 From: rogday Date: Tue, 19 Oct 2021 16:29:22 +0300 Subject: [PATCH 010/226] Merge pull request #20883 from rogday:eltwise_refactoring * backport elementwise_layers refactor * keep NULL --- modules/dnn/src/layers/elementwise_layers.cpp | 415 ++++-------------- 1 file changed, 96 insertions(+), 319 deletions(-) diff --git a/modules/dnn/src/layers/elementwise_layers.cpp b/modules/dnn/src/layers/elementwise_layers.cpp index 6e7fa43a70..d8f0b654d3 100644 --- a/modules/dnn/src/layers/elementwise_layers.cpp +++ b/modules/dnn/src/layers/elementwise_layers.cpp @@ -500,16 +500,9 @@ struct ReLU6Functor : public BaseFunctor int64 getFLOPSPerElement() const { return 2; } }; -struct TanHFunctor : public BaseFunctor +template +struct BaseDefaultFunctor : public BaseFunctor { - typedef TanHLayer Layer; - - bool supportBackend(int backendId, int) - { - return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE || - backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; - } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const { for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) @@ -517,7 +510,7 @@ struct TanHFunctor : public BaseFunctor for( int i = 0; i < len; i++ ) { float x = srcptr[i]; - dstptr[i] = tanh(x); + dstptr[i] = static_cast(this)->calculate(x); } } } @@ -537,10 +530,11 @@ struct TanHFunctor : public BaseFunctor UMat& src = inputs[i]; UMat& dst = outputs[i]; - ocl::Kernel kernel("TanHForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); + ocl::Kernel kernel(ocl_kernel_name, ocl::dnn::activations_oclsrc, buildopt); + kernel.set(0, static_cast(src.total())); kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); + static_cast(this)->setKernelParams(kernel); size_t gSize = src.total(); CV_Assert(kernel.run(1, &gSize, NULL, false)); @@ -550,6 +544,41 @@ struct TanHFunctor : public BaseFunctor } #endif + inline void setKernelParams(ocl::Kernel& kernel) const {} + +#ifdef HAVE_DNN_IE_NN_BUILDER_2019 + InferenceEngine::Builder::Layer initInfEngineBuilderAPI() + { + CV_Error(Error::StsNotImplemented, ""); + } +#endif // HAVE_DNN_IE_NN_BUILDER_2019 + +#ifdef HAVE_DNN_NGRAPH + std::shared_ptr initNgraphAPI(const std::shared_ptr& node) + { + CV_Error(Error::StsNotImplemented, ""); + } +#endif // HAVE_DNN_NGRAPH + +private: + static const char* const ocl_kernel_name; +}; + +struct TanHFunctor : public BaseDefaultFunctor +{ + typedef TanHLayer Layer; + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE || + backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; + } + + inline float calculate(float x) const + { + return tanh(x); + } + #ifdef HAVE_HALIDE void attachHalide(const Halide::Expr& input, Halide::Func& top) { @@ -575,56 +604,24 @@ struct TanHFunctor : public BaseFunctor int64 getFLOPSPerElement() const { return 1; } }; -struct SwishFunctor : public BaseFunctor +template<> +const char* const TanHFunctor::BaseDefaultFunctor::ocl_kernel_name = "TanHForward"; + +struct SwishFunctor : public BaseDefaultFunctor { typedef SwishLayer Layer; bool supportBackend(int backendId, int) { return backendId == DNN_BACKEND_OPENCV || - backendId == DNN_BACKEND_HALIDE || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH;; + backendId == DNN_BACKEND_HALIDE || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + inline float calculate(float x) const { - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) - { - for( int i = 0; i < len; i++ ) - { - float x = srcptr[i]; - dstptr[i] = x / (1.0f + exp(-x)); - } - } + return x / (1.f + exp(-x)); } -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) - { - std::vector inputs; - std::vector outputs; - - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); - - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; - - ocl::Kernel kernel("SwishForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); - - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); - } - - return true; - } -#endif - #ifdef HAVE_HALIDE void attachHalide(const Halide::Expr& input, Halide::Func& top) { @@ -633,13 +630,6 @@ struct SwishFunctor : public BaseFunctor } #endif // HAVE_HALIDE -#ifdef HAVE_DNN_IE_NN_BUILDER_2019 - InferenceEngine::Builder::Layer initInfEngineBuilderAPI() - { - CV_Error(Error::StsNotImplemented, ""); - } -#endif // HAVE_DNN_IE_NN_BUILDER_2019 - #ifdef HAVE_DNN_NGRAPH std::shared_ptr initNgraphAPI(const std::shared_ptr& node) { @@ -651,7 +641,10 @@ struct SwishFunctor : public BaseFunctor int64 getFLOPSPerElement() const { return 3; } }; -struct MishFunctor : public BaseFunctor +template<> +const char* const SwishFunctor::BaseDefaultFunctor::ocl_kernel_name = "SwishForward"; + +struct MishFunctor : public BaseDefaultFunctor { typedef MishLayer Layer; @@ -661,53 +654,18 @@ struct MishFunctor : public BaseFunctor backendId == DNN_BACKEND_HALIDE || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + inline float calculate(float x) const { - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) + // Use fast approximation introduced in https://github.com/opencv/opencv/pull/17200 + if (x >= 8.f) { - for( int i = 0; i < len; i++ ) - { - // Use fast approximation introduced in https://github.com/opencv/opencv/pull/17200 - float x = srcptr[i]; - if (x >= 8.f) - dstptr[i] = x; - else - { - float eX = exp(x); - float n = (eX + 2) * eX; - dstptr[i] = (x * n) / (n + 2); - } - } - } - } - -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) - { - std::vector inputs; - std::vector outputs; - - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); - - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; - - ocl::Kernel kernel("MishForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); - - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); + return x; } - return true; + float eX = exp(x); + float n = (eX + 2.f) * eX; + return (x * n) / (n + 2.f); } -#endif #ifdef HAVE_HALIDE void attachHalide(const Halide::Expr& input, Halide::Func& top) @@ -717,13 +675,6 @@ struct MishFunctor : public BaseFunctor } #endif // HAVE_HALIDE -#ifdef HAVE_DNN_IE_NN_BUILDER_2019 - InferenceEngine::Builder::Layer initInfEngineBuilderAPI() - { - CV_Error(Error::StsNotImplemented, ""); - } -#endif // HAVE_DNN_IE_NN_BUILDER_2019 - #ifdef HAVE_DNN_NGRAPH std::shared_ptr initNgraphAPI(const std::shared_ptr& node) { @@ -740,7 +691,10 @@ struct MishFunctor : public BaseFunctor int64 getFLOPSPerElement() const { return 3; } }; -struct SigmoidFunctor : public BaseFunctor +template<> +const char* const MishFunctor::BaseDefaultFunctor::ocl_kernel_name = "MishForward"; + +struct SigmoidFunctor : public BaseDefaultFunctor { typedef SigmoidLayer Layer; @@ -750,46 +704,11 @@ struct SigmoidFunctor : public BaseFunctor backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + inline float calculate(float x) const { - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) - { - for( int i = 0; i < len; i++ ) - { - float x = srcptr[i]; - dstptr[i] = 1.f/(1.f + exp(-x)); - } - } + return 1.f / (1.f + exp(-x)); } -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) - { - std::vector inputs; - std::vector outputs; - - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); - - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; - - ocl::Kernel kernel("SigmoidForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); - - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); - } - - return true; - } -#endif - #ifdef HAVE_HALIDE void attachHalide(const Halide::Expr& input, Halide::Func& top) { @@ -815,7 +734,10 @@ struct SigmoidFunctor : public BaseFunctor int64 getFLOPSPerElement() const { return 3; } }; -struct ELUFunctor : public BaseFunctor +template<> +const char* const SigmoidFunctor::BaseDefaultFunctor::ocl_kernel_name = "SigmoidForward"; + +struct ELUFunctor : public BaseDefaultFunctor { typedef ELULayer Layer; @@ -825,46 +747,11 @@ struct ELUFunctor : public BaseFunctor backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + inline float calculate(float x) const { - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) - { - for(int i = 0; i < len; i++ ) - { - float x = srcptr[i]; - dstptr[i] = x >= 0.f ? x : exp(x) - 1; - } - } + return x >= 0.f ? x : exp(x) - 1.f; } -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) - { - std::vector inputs; - std::vector outputs; - - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); - - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; - - ocl::Kernel kernel("ELUForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); - - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); - } - - return true; - } -#endif - #ifdef HAVE_HALIDE void attachHalide(const Halide::Expr& input, Halide::Func& top) { @@ -890,7 +777,10 @@ struct ELUFunctor : public BaseFunctor int64 getFLOPSPerElement() const { return 2; } }; -struct AbsValFunctor : public BaseFunctor +template<> +const char* const ELUFunctor::BaseDefaultFunctor::ocl_kernel_name = "ELUForward"; + +struct AbsValFunctor : public BaseDefaultFunctor { typedef AbsLayer Layer; @@ -903,46 +793,11 @@ struct AbsValFunctor : public BaseFunctor return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + inline float calculate(float x) const { - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) - { - for( int i = 0; i < len; i++ ) - { - float x = srcptr[i]; - dstptr[i] = abs(x); - } - } + return abs(x); } -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) - { - std::vector inputs; - std::vector outputs; - - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); - - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; - - ocl::Kernel kernel("AbsValForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); - - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); - } - - return true; - } -#endif - #ifdef HAVE_HALIDE void attachHalide(const Halide::Expr& input, Halide::Func& top) { @@ -971,7 +826,10 @@ struct AbsValFunctor : public BaseFunctor int64 getFLOPSPerElement() const { return 1; } }; -struct BNLLFunctor : public BaseFunctor +template<> +const char* const AbsValFunctor::BaseDefaultFunctor::ocl_kernel_name = "AbsValForward"; + +struct BNLLFunctor : public BaseDefaultFunctor { typedef BNLLLayer Layer; @@ -980,47 +838,12 @@ struct BNLLFunctor : public BaseFunctor return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + inline float calculate(float x) const { - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) - { - for( int i = 0; i < len; i++ ) - { - float x = srcptr[i]; - // https://github.com/BVLC/caffe/blame/1.0/src/caffe/layers/bnll_layer.cpp#L17 - dstptr[i] = x > 0 ? x + log(1. + exp(-x)) : log(1. + exp(x)); - } - } + // https://github.com/BVLC/caffe/blame/1.0/src/caffe/layers/bnll_layer.cpp#L17 + return x > 0 ? x + log(1.f + exp(-x)) : log(1.f + exp(x)); } -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) - { - std::vector inputs; - std::vector outputs; - - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); - - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; - - ocl::Kernel kernel("BNLLForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); - - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); - } - - return true; - } -#endif - #ifdef HAVE_HALIDE void attachHalide(const Halide::Expr& input, Halide::Func& top) { @@ -1030,23 +853,12 @@ struct BNLLFunctor : public BaseFunctor } #endif // HAVE_HALIDE -#ifdef HAVE_DNN_IE_NN_BUILDER_2019 - InferenceEngine::Builder::Layer initInfEngineBuilderAPI() - { - CV_Error(Error::StsNotImplemented, ""); - } -#endif // HAVE_DNN_IE_NN_BUILDER_2019 - -#ifdef HAVE_DNN_NGRAPH - std::shared_ptr initNgraphAPI(const std::shared_ptr& node) - { - CV_Error(Error::StsNotImplemented, ""); - } -#endif // HAVE_DNN_NGRAPH - int64 getFLOPSPerElement() const { return 5; } }; +template<> +const char* const BNLLFunctor::BaseDefaultFunctor::ocl_kernel_name = "BNLLForward"; + struct PowerFunctor : public BaseFunctor { typedef PowerLayer Layer; @@ -1206,7 +1018,7 @@ struct PowerFunctor : public BaseFunctor int64 getFLOPSPerElement() const { return power == 1 ? 2 : 10; } }; -struct ExpFunctor : public BaseFunctor +struct ExpFunctor : public BaseDefaultFunctor { typedef ExpLayer Layer; float base, scale, shift; @@ -1232,47 +1044,16 @@ struct ExpFunctor : public BaseFunctor backendId == DNN_BACKEND_HALIDE || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + inline float calculate(float x) const { - float a = normScale, b = normShift; - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) - { - for( int i = 0; i < len; i++ ) - { - float x = srcptr[i]; - dstptr[i] = exp(a*x + b); - } - } + return exp(normScale * x + normShift); } -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) + inline void setKernelParams(ocl::Kernel& kernel) const { - std::vector inputs; - std::vector outputs; - - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); - - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; - - ocl::Kernel kernel("ExpForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); - kernel.set(3, (float)normScale); - kernel.set(4, (float)normShift); - - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); - } - return true; + kernel.set(3, normScale); + kernel.set(4, normShift); } -#endif #ifdef HAVE_HALIDE void attachHalide(const Halide::Expr& input, Halide::Func& top) @@ -1282,13 +1063,6 @@ struct ExpFunctor : public BaseFunctor } #endif // HAVE_HALIDE -#ifdef HAVE_DNN_IE_NN_BUILDER_2019 - InferenceEngine::Builder::Layer initInfEngineBuilderAPI() - { - CV_Error(Error::StsNotImplemented, ""); - } -#endif // HAVE_DNN_IE_NN_BUILDER_2019 - #ifdef HAVE_DNN_NGRAPH std::shared_ptr initNgraphAPI(const std::shared_ptr& node) { @@ -1305,6 +1079,9 @@ struct ExpFunctor : public BaseFunctor int64 getFLOPSPerElement() const { return 3; } }; +template<> +const char* const ExpFunctor::BaseDefaultFunctor::ocl_kernel_name = "ExpForward"; + struct ChannelsPReLUFunctor : public BaseFunctor { typedef ChannelsPReLULayer Layer; From 7da51787b9b27495ea754ad441781f6a9e472d3e Mon Sep 17 00:00:00 2001 From: Zhuo Zhang Date: Tue, 19 Oct 2021 21:30:27 +0800 Subject: [PATCH 011/226] Merge pull request #20900 from zchrissirhcz:3.4-hwfeatures-support-qnx * fix: correctly check neon flags for QNX platform * refactor: change __QNXNTO__ to __QNX__ --- modules/core/src/system.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core/src/system.cpp b/modules/core/src/system.cpp index 27142a4034..49e146372e 100644 --- a/modules/core/src/system.cpp +++ b/modules/core/src/system.cpp @@ -129,7 +129,7 @@ void* allocSingletonNewBuffer(size_t size) { return malloc(size); } #if defined __ANDROID__ || defined __unix__ || defined __FreeBSD__ || defined __OpenBSD__ || defined __HAIKU__ # include # include -#if defined __QNXNTO__ +#if defined __QNX__ # include #else # include @@ -545,7 +545,7 @@ struct HWFeatures } #endif // CV_CPUID_X86 - #if defined __ANDROID__ || defined __linux__ || defined __FreeBSD__ + #if defined __ANDROID__ || defined __linux__ || defined __FreeBSD__ || defined __QNX__ #ifdef __aarch64__ have[CV_CPU_NEON] = true; have[CV_CPU_FP16] = true; From f77fdc0ce86a668f8acdfd94bf4bcbd65126c6f3 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 19 Oct 2021 09:28:12 +0000 Subject: [PATCH 012/226] samples: fix build without threading support --- samples/CMakeLists.txt | 1 + samples/dnn/object_detection.cpp | 16 ++++++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 87f5cfcf88..68afc487a2 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -114,6 +114,7 @@ else() find_package(Threads) endif() if((TARGET Threads::Threads OR HAVE_THREADS) AND NOT OPENCV_EXAMPLES_DISABLE_THREADS) + set(HAVE_THREADS 1) add_definitions(-DHAVE_THREADS=1) endif() diff --git a/samples/dnn/object_detection.cpp b/samples/dnn/object_detection.cpp index c7e42430fe..5d201de3b5 100644 --- a/samples/dnn/object_detection.cpp +++ b/samples/dnn/object_detection.cpp @@ -5,7 +5,11 @@ #include #include -#ifdef CV_CXX11 +#if defined(CV_CXX11) && defined(HAVE_THREADS) +#define USE_THREADS 1 +#endif + +#ifdef USE_THREADS #include #include #include @@ -51,7 +55,7 @@ void drawPred(int classId, float conf, int left, int top, int right, int bottom, void callback(int pos, void* userdata); -#ifdef CV_CXX11 +#ifdef USE_THREADS template class QueueFPS : public std::queue { @@ -101,7 +105,7 @@ private: TickMeter tm; std::mutex mutex; }; -#endif // CV_CXX11 +#endif // USE_THREADS int main(int argc, char** argv) { @@ -166,7 +170,7 @@ int main(int argc, char** argv) else cap.open(parser.get("device")); -#ifdef CV_CXX11 +#ifdef USE_THREADS bool process = true; // Frames capturing thread @@ -266,7 +270,7 @@ int main(int argc, char** argv) framesThread.join(); processingThread.join(); -#else // CV_CXX11 +#else // USE_THREADS if (async) CV_Error(Error::StsNotImplemented, "Asynchronous forward is supported only with Inference Engine backend."); @@ -297,7 +301,7 @@ int main(int argc, char** argv) imshow(kWinName, frame); } -#endif // CV_CXX11 +#endif // USE_THREADS return 0; } From 805c2832c19b4d8d281561338739ae96e014a328 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 20 Oct 2021 05:45:40 +0000 Subject: [PATCH 013/226] 4.x: drop DISABLE_OPENCV_24_COMPATIBILITY macro not used in 4.x code --- samples/CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 9d8f588083..8b1907f380 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -110,8 +110,6 @@ if(MSVC) endif() endif() -add_definitions(-DDISABLE_OPENCV_24_COMPATIBILITY=1) # Avoid C-like legacy API - if(OPENCV_EXAMPLES_DISABLE_THREADS) # nothing elseif(MSVC OR APPLE) From 9379e85e23b2feda1317d5fec2117f1ebe0b9193 Mon Sep 17 00:00:00 2001 From: Lukas Weber Date: Wed, 20 Oct 2021 09:07:48 +0200 Subject: [PATCH 014/226] changed no longer patented SIFT --- samples/python/stitching_detailed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/python/stitching_detailed.py b/samples/python/stitching_detailed.py index 4ee29048d1..dfa88beba1 100644 --- a/samples/python/stitching_detailed.py +++ b/samples/python/stitching_detailed.py @@ -36,7 +36,7 @@ except (AttributeError, cv.error) as e: # if SURF not available, ORB is default FEATURES_FIND_CHOICES['orb'] = cv.ORB.create try: - FEATURES_FIND_CHOICES['sift'] = cv.xfeatures2d_SIFT.create + FEATURES_FIND_CHOICES['sift'] = cv.SIFT_create except AttributeError: print("SIFT not available") try: From 1f9a7b8fd30f19e42fd29d1b8102d8ac466c0414 Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Wed, 20 Oct 2021 12:43:32 +0300 Subject: [PATCH 015/226] Merge pull request #20738 from sivanov-work:merge_master_vpl_dev_select G-API: oneVPL - Implement IDeviceSelector & default cfg_param-based selector * Initial commit * Add MACRO undef * Change IDeviceSelector, Change Inf sample for choose external device * Fix compilation * Address some comments * Fix compilation * Add missing header * Add EXPORT to dev selector * Add guard * Remove enum type attr * Fix compilation without VPL * Add HAVE_INFER guard in sample * Remove unusable include from tests * Remove unusable include from sample * Remove cl_d3d11 header from unit test --- modules/gapi/CMakeLists.txt | 15 + .../onevpl/device_selector_interface.hpp | 102 ++++++ .../opencv2/gapi/streaming/onevpl/source.hpp | 24 ++ .../gapi/samples/onevpl_infer_single_roi.cpp | 138 +++++++- .../onevpl/accelerators/accel_policy_dx11.hpp | 1 + .../onevpl/cfg_param_device_selector.cpp | 314 ++++++++++++++++++ .../onevpl/cfg_param_device_selector.hpp | 44 +++ .../onevpl/device_selector_interface.cpp | 87 +++++ modules/gapi/src/streaming/onevpl/source.cpp | 63 +++- .../gapi/src/streaming/onevpl/source_priv.cpp | 6 +- .../gapi/src/streaming/onevpl/source_priv.hpp | 3 +- .../gapi_streaming_vpl_device_selector.cpp | 229 +++++++++++++ 12 files changed, 1018 insertions(+), 8 deletions(-) create mode 100644 modules/gapi/include/opencv2/gapi/streaming/onevpl/device_selector_interface.hpp create mode 100644 modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp create mode 100644 modules/gapi/src/streaming/onevpl/cfg_param_device_selector.hpp create mode 100644 modules/gapi/src/streaming/onevpl/device_selector_interface.cpp create mode 100644 modules/gapi/test/streaming/gapi_streaming_vpl_device_selector.cpp diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 917d1e0814..61ab5397d7 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -181,6 +181,9 @@ set(gapi_srcs src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp src/streaming/onevpl/engine/decode/decode_session.cpp + src/streaming/onevpl/cfg_param_device_selector.cpp + src/streaming/onevpl/device_selector_interface.cpp + # Utils (ITT tracing) src/utils/itt.cpp ) @@ -283,3 +286,15 @@ endif() ocv_add_perf_tests() ocv_add_samples() + + +# Required for sample with inference on host +if (TARGET example_gapi_onevpl_infer_single_roi) + if(OPENCV_GAPI_INF_ENGINE) + ocv_target_link_libraries(example_gapi_onevpl_infer_single_roi PRIVATE ${INF_ENGINE_TARGET}) + ocv_target_compile_definitions(example_gapi_onevpl_infer_single_roi PRIVATE -DHAVE_INF_ENGINE) + endif() + if(HAVE_D3D11 AND HAVE_OPENCL) + ocv_target_include_directories(example_gapi_onevpl_infer_single_roi SYSTEM PRIVATE ${OPENCL_INCLUDE_DIRS}) + endif() +endif() diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/device_selector_interface.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/device_selector_interface.hpp new file mode 100644 index 0000000000..ca19849d72 --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/device_selector_interface.hpp @@ -0,0 +1,102 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_DEVICE_SELECTOR_INTERFACE_HPP +#define GAPI_STREAMING_ONEVPL_DEVICE_SELECTOR_INTERFACE_HPP + +#include +#include +#include +#include + +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +enum class AccelType : uint8_t { + HOST, + DX11, + + LAST_VALUE = std::numeric_limits::max() +}; + +GAPI_EXPORTS const char* to_cstring(AccelType type); + +struct IDeviceSelector; +struct GAPI_EXPORTS Device { + friend struct IDeviceSelector; + using Ptr = void*; + + ~Device(); + const std::string& get_name() const; + Ptr get_ptr() const; + AccelType get_type() const; +private: + Device(Ptr device_ptr, const std::string& device_name, + AccelType device_type); + + std::string name; + Ptr ptr; + AccelType type; +}; + +struct GAPI_EXPORTS Context { + friend struct IDeviceSelector; + using Ptr = void*; + + ~Context(); + Ptr get_ptr() const; + AccelType get_type() const; +private: + Context(Ptr ctx_ptr, AccelType ctx_type); + Ptr ptr; + AccelType type; +}; + +struct GAPI_EXPORTS IDeviceSelector { + using Ptr = std::shared_ptr; + + struct GAPI_EXPORTS Score { + friend struct IDeviceSelector; + using Type = int16_t; + static constexpr Type MaxActivePriority = std::numeric_limits::max(); + static constexpr Type MinActivePriority = 0; + static constexpr Type MaxPassivePriority = MinActivePriority - 1; + static constexpr Type MinPassivePriority = std::numeric_limits::min(); + + Score(Type val); + ~Score(); + + operator Type () const; + Type get() const; + friend bool operator< (Score lhs, Score rhs) { + return lhs.get() < rhs.get(); + } + private: + Type value; + }; + + using DeviceScoreTable = std::map; + using DeviceContexts = std::vector; + + virtual ~IDeviceSelector(); + virtual DeviceScoreTable select_devices() const = 0; + virtual DeviceContexts select_context() = 0; +protected: + template + static Entity create(Args &&...args) { + return Entity(std::forward(args)...); + } +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // GAPI_STREAMING_ONEVPL_DEVICE_SELECTOR_INTERFACE_HPP diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp index a8dbefdf50..6334480c1b 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp @@ -12,6 +12,7 @@ #include #include #include +#include namespace cv { namespace gapi { @@ -38,8 +39,31 @@ public: GSource(const std::string& filePath, const CfgParams& cfg_params = CfgParams{}); + + GSource(const std::string& filePath, + const CfgParams& cfg_params, + const std::string& device_id, + void* accel_device_ptr, + void* accel_ctx_ptr); + + GSource(const std::string& filePath, + const CfgParams& cfg_params, + std::shared_ptr selector); + + GSource(std::shared_ptr source, const CfgParams& cfg_params = CfgParams{}); + + GSource(std::shared_ptr source, + const CfgParams& cfg_params, + const std::string& device_id, + void* accel_device_ptr, + void* accel_ctx_ptr); + + GSource(std::shared_ptr source, + const CfgParams& cfg_params, + std::shared_ptr selector); + ~GSource() override; bool pull(cv::gapi::wip::Data& data) override; diff --git a/modules/gapi/samples/onevpl_infer_single_roi.cpp b/modules/gapi/samples/onevpl_infer_single_roi.cpp index f3aee09d42..deca86f1ca 100644 --- a/modules/gapi/samples/onevpl_infer_single_roi.cpp +++ b/modules/gapi/samples/onevpl_infer_single_roi.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -10,8 +11,30 @@ #include #include #include +#include #include // CommandLineParser +#ifdef HAVE_INF_ENGINE +#include // ParamMap + +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 +#pragma comment(lib,"d3d11.lib") + +// get rid of generate macro max/min/etc from DX side +#define D3D11_NO_HELPERS +#define NOMINMAX +#include +#include +#include +#pragma comment(lib, "dxgi") +#undef NOMINMAX +#undef D3D11_NO_HELPERS + +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX +#endif // HAVE_INF_ENGINE + const std::string about = "This is an OpenCV-based version of oneVPLSource decoder example"; const std::string keys = @@ -36,6 +59,37 @@ std::string get_weights_path(const std::string &model_path) { CV_Assert(ext == ".xml"); return model_path.substr(0u, sz - EXT_LEN) + ".bin"; } + +#ifdef HAVE_INF_ENGINE +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 + +using AccelParamsType = std::tuple, CComPtr>; + +AccelParamsType create_device_with_ctx(CComPtr adapter) { + UINT flags = 0; + D3D_FEATURE_LEVEL feature_levels[] = { D3D_FEATURE_LEVEL_11_1, + D3D_FEATURE_LEVEL_11_0, + }; + D3D_FEATURE_LEVEL featureLevel; + ID3D11Device* ret_device_ptr = nullptr; + ID3D11DeviceContext* ret_ctx_ptr = nullptr; + HRESULT err = D3D11CreateDevice(adapter, D3D_DRIVER_TYPE_UNKNOWN, + nullptr, flags, + feature_levels, + ARRAYSIZE(feature_levels), + D3D11_SDK_VERSION, &ret_device_ptr, + &featureLevel, &ret_ctx_ptr); + if (FAILED(err)) { + throw std::runtime_error("Cannot create D3D11CreateDevice, error: " + + std::to_string(HRESULT_CODE(err))); + } + + return std::make_tuple(ret_device_ptr, ret_ctx_ptr); +} +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX +#endif // HAVE_INF_ENGINE } // anonymous namespace namespace custom { @@ -197,11 +251,84 @@ int main(int argc, char *argv[]) { return -1; } + const std::string& device_id = cmd.get("faced"); auto face_net = cv::gapi::ie::Params { face_model_path, // path to topology IR get_weights_path(face_model_path), // path to weights - cmd.get("faced"), // device specifier + device_id }; + + // Create device_ptr & context_ptr using graphic API + // InferenceEngine requires such device & context to create its own + // remote shared context through InferenceEngine::ParamMap in + // GAPI InferenceEngine backend to provide interoperability with onevpl::GSource + // So GAPI InferenceEngine backend and onevpl::GSource MUST share the same + // device and context + void* accel_device_ptr = nullptr; + void* accel_ctx_ptr = nullptr; + +#ifdef HAVE_INF_ENGINE +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 + CComPtr dx11_dev; + CComPtr dx11_ctx; + + if (device_id.find("GPU") != std::string::npos) { + CComPtr adapter_factory; + CComPtr intel_adapters; + { + IDXGIFactory* out_factory = nullptr; + HRESULT err = CreateDXGIFactory(__uuidof(IDXGIFactory), + reinterpret_cast(&out_factory)); + if (FAILED(err)) { + std::cerr << "Cannot create CreateDXGIFactory, error: " << HRESULT_CODE(err) << std::endl; + return -1; + } + adapter_factory.Attach(out_factory); + } + + CComPtr intel_adapter; + UINT adapter_index = 0; + const unsigned int refIntelVendorID = 0x8086; + IDXGIAdapter* out_adapter = nullptr; + + while (adapter_factory->EnumAdapters(adapter_index, &out_adapter) != DXGI_ERROR_NOT_FOUND) { + DXGI_ADAPTER_DESC desc{}; + out_adapter->GetDesc(&desc); + if (desc.VendorId == refIntelVendorID) { + intel_adapter.Attach(out_adapter); + break; + } + ++adapter_index; + } + + if (!intel_adapter) { + std::cerr << "No Intel GPU adapter on aboard. Exit" << std::endl; + return -1; + } + + std::tie(dx11_dev, dx11_ctx) = create_device_with_ctx(intel_adapter); + accel_device_ptr = reinterpret_cast(dx11_dev.p); + accel_ctx_ptr = reinterpret_cast(dx11_ctx.p); + + // put accel type description for VPL source + source_cfgs.push_back(cfg::create_from_string( + "mfxImplDescription.AccelerationMode" + ":" + "MFX_ACCEL_MODE_VIA_D3D11")); + } + +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX + // set ctx_config for GPU device only - no need in case of CPU device type + if (device_id.find("GPU") != std::string::npos) { + InferenceEngine::ParamMap ctx_config({{"CONTEXT_TYPE", "VA_SHARED"}, + {"VA_DEVICE", accel_device_ptr} }); + + face_net.cfgContextParams(ctx_config); + } +#endif // HAVE_INF_ENGINE + auto kernels = cv::gapi::kernels < custom::OCVLocateROI , custom::OCVParseSSD @@ -211,7 +338,14 @@ int main(int argc, char *argv[]) { // Create source cv::Ptr cap; try { - cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs); + if (device_id.find("GPU") != std::string::npos) { + cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs, + device_id, + accel_device_ptr, + accel_ctx_ptr); + } else { + cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs); + } std::cout << "oneVPL source desription: " << cap->descr_of() << std::endl; } catch (const std::exception& ex) { std::cerr << "Cannot create source: " << ex.what() << std::endl; diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp index a875f57085..946640d886 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp @@ -22,6 +22,7 @@ #ifdef HAVE_DIRECTX #ifdef HAVE_D3D11 #define D3D11_NO_HELPERS +#define NOMINMAX #include #include #include "opencv2/core/directx.hpp" diff --git a/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp b/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp new file mode 100644 index 0000000000..c8fd49c1ad --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp @@ -0,0 +1,314 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifdef HAVE_ONEVPL +#include +#include +#include + +#include "streaming/onevpl/cfg_param_device_selector.hpp" +#include "logger.hpp" + +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 +#pragma comment(lib,"d3d11.lib") + +// get rid of generate macro max/min/etc from DX side +#define D3D11_NO_HELPERS +#define NOMINMAX +#include +#include +#include +#pragma comment(lib, "dxgi") +#undef D3D11_NO_HELPERS +#undef NOMINMAX + +#include +#include "opencv2/core/directx.hpp" +#ifdef HAVE_OPENCL +#include +#endif + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +// TODO Will be changed on generic function from `onevpl_param_parser` as soons as feature merges +static mfxVariant cfg_param_to_mfx_variant(const CfgParam& accel_param) { + mfxVariant ret; + const CfgParam::value_t& accel_val = accel_param.get_value(); + if (!cv::util::holds_alternative(accel_val)) { + // expected string or uint32_t as value + if (!cv::util::holds_alternative(accel_val)) { + throw std::logic_error("Incorrect value type of \"mfxImplDescription.AccelerationMode\" " + " std::string is expected" ); + } + ret.Type = MFX_VARIANT_TYPE_U32; + ret.Data.U32 = cv::util::get(accel_val); + return ret; + } + + const std::string& accel_val_str = cv::util::get(accel_val); + ret.Type = MFX_VARIANT_TYPE_U32; + if (accel_val_str == "MFX_ACCEL_MODE_NA") { + ret.Data.U32 = MFX_ACCEL_MODE_NA; + } else if (accel_val_str == "MFX_ACCEL_MODE_VIA_D3D9") { + ret.Data.U32 = MFX_ACCEL_MODE_VIA_D3D9; + } else if (accel_val_str == "MFX_ACCEL_MODE_VIA_D3D11") { + ret.Data.U32 = MFX_ACCEL_MODE_VIA_D3D11; + } else if (accel_val_str == "MFX_ACCEL_MODE_VIA_VAAPI") { + ret.Data.U32 = MFX_ACCEL_MODE_VIA_VAAPI; + } else if (accel_val_str == "MFX_ACCEL_MODE_VIA_VAAPI_DRM_MODESET") { + ret.Data.U32 = MFX_ACCEL_MODE_VIA_VAAPI_DRM_MODESET; + } else if (accel_val_str == "MFX_ACCEL_MODE_VIA_VAAPI_GLX") { + ret.Data.U32 = MFX_ACCEL_MODE_VIA_VAAPI_GLX; + } else if (accel_val_str == "MFX_ACCEL_MODE_VIA_VAAPI_X11") { + ret.Data.U32 = MFX_ACCEL_MODE_VIA_VAAPI_X11; + } else if (accel_val_str == "MFX_ACCEL_MODE_VIA_VAAPI_WAYLAND") { + ret.Data.U32 = MFX_ACCEL_MODE_VIA_VAAPI_WAYLAND; + } else if (accel_val_str == "MFX_ACCEL_MODE_VIA_HDDLUNITE") { + ret.Data.U32 = MFX_ACCEL_MODE_VIA_HDDLUNITE; + } + return ret; +} + +CfgParamDeviceSelector::CfgParamDeviceSelector(const CfgParams& cfg_params) : + suggested_device(IDeviceSelector::create(nullptr, "CPU", AccelType::HOST)), + suggested_context(IDeviceSelector::create(nullptr, AccelType::HOST)) { + + auto accel_mode_it = + std::find_if(cfg_params.begin(), cfg_params.end(), [] (const CfgParam& value) { + return value.get_name() == "mfxImplDescription.AccelerationMode"; + }); + if (accel_mode_it == cfg_params.end()) + { + GAPI_LOG_DEBUG(nullptr, "No HW Accel requested. Use default CPU"); + return; + } + + GAPI_LOG_DEBUG(nullptr, "Add HW acceleration support"); + mfxVariant accel_mode = cfg_param_to_mfx_variant(*accel_mode_it); + + switch(accel_mode.Data.U32) { + case MFX_ACCEL_MODE_VIA_D3D11: { +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 + ID3D11Device *hw_handle = nullptr; + ID3D11DeviceContext* device_context = nullptr; + + //Create device + UINT creationFlags = 0;//D3D11_CREATE_DEVICE_BGRA_SUPPORT; + +#if defined _DEBUG || defined CV_STATIC_ANALYSIS + // If the project is in a debug build, enable debugging via SDK Layers with this flag. + creationFlags |= D3D11_CREATE_DEVICE_DEBUG; +#endif + + D3D_FEATURE_LEVEL featureLevels[] = { D3D_FEATURE_LEVEL_11_1, + D3D_FEATURE_LEVEL_11_0, + }; + D3D_FEATURE_LEVEL featureLevel; + + CComPtr adapter_factory; + CComPtr intel_adapters; + { + IDXGIFactory* out_factory = nullptr; + HRESULT err = CreateDXGIFactory(__uuidof(IDXGIFactory), + reinterpret_cast(&out_factory)); + if (FAILED(err)) { + throw std::runtime_error("Cannot create CreateDXGIFactory, error: " + std::to_string(HRESULT_CODE(err))); + } + adapter_factory.Attach(out_factory); + } + + CComPtr intel_adapter; + UINT adapter_index = 0; + const unsigned int refIntelVendorID = 0x8086; + IDXGIAdapter* out_adapter = nullptr; + + while (adapter_factory->EnumAdapters(adapter_index, &out_adapter) != DXGI_ERROR_NOT_FOUND) { + DXGI_ADAPTER_DESC desc{}; + out_adapter->GetDesc(&desc); + if (desc.VendorId == refIntelVendorID) { + intel_adapter.Attach(out_adapter); + break; + } + ++adapter_index; + } + + if (!intel_adapter) { + throw std::runtime_error("No Intel GPU adapter on aboard"); + } + + // Create the Direct3D 11 API device object and a corresponding context. + HRESULT err = D3D11CreateDevice(intel_adapter, + D3D_DRIVER_TYPE_UNKNOWN, + nullptr, creationFlags, + featureLevels, ARRAYSIZE(featureLevels), + D3D11_SDK_VERSION, + &hw_handle, &featureLevel, + &device_context); + if(FAILED(err)) { + throw std::logic_error("Cannot create D3D11CreateDevice, error: " + std::to_string(HRESULT_CODE(err))); + } + + // oneVPL recommendation + { + ID3D11Multithread *pD11Multithread = nullptr; + device_context->QueryInterface(IID_PPV_ARGS(&pD11Multithread)); + pD11Multithread->SetMultithreadProtected(true); + pD11Multithread->Release(); + } + + suggested_device = IDeviceSelector::create(hw_handle, "GPU", AccelType::DX11); + suggested_context = IDeviceSelector::create(device_context, AccelType::DX11); +#else + GAPI_LOG_WARNING(nullptr, "Unavailable \"mfxImplDescription.AccelerationMode: MFX_ACCEL_MODE_VIA_D3D11\"" + "was chosen for current project configuration"); + throw std::logic_error("Unsupported \"mfxImplDescription.AccelerationMode: MFX_ACCEL_MODE_VIA_D3D11\""); +#endif // HAVE_DIRECTX +#endif // HAVE_D3D11 + break; + } + case MFX_ACCEL_MODE_NA: { + // nothing to do + break; + } + default: + throw std::logic_error("Unsupported \"mfxImplDescription.AccelerationMode\" requested: " + + std::to_string(accel_mode.Data.U32)); + break; + } +} + +CfgParamDeviceSelector::CfgParamDeviceSelector(Device::Ptr device_ptr, + const std::string& device_id, + Context::Ptr ctx_ptr, + const CfgParams& cfg_params) : + suggested_device(IDeviceSelector::create(nullptr, "CPU", AccelType::HOST)), + suggested_context(IDeviceSelector::create(nullptr, AccelType::HOST)) { + auto accel_mode_it = + std::find_if(cfg_params.begin(), cfg_params.end(), [] (const CfgParam& value) { + return value.get_name() == "mfxImplDescription.AccelerationMode"; + }); + if (accel_mode_it == cfg_params.end()) { + GAPI_LOG_WARNING(nullptr, "Cannot deternime \"device_ptr\" type. " + "Make sure a param \"mfxImplDescription.AccelerationMode\" " + "presents in configurations and has correct value according to " + "\"device_ptr\" type"); + throw std::logic_error("Missing \"mfxImplDescription.AccelerationMode\" param"); + } + + GAPI_LOG_DEBUG(nullptr, "Turn on HW acceleration support for device: " << + device_ptr << + ", context: " << ctx_ptr); + if (!device_ptr) { + GAPI_LOG_WARNING(nullptr, "Empty \"device_ptr\" is not allowed when " + "param \"mfxImplDescription.AccelerationMode\" existed"); + throw std::logic_error("Invalid param: \"device_ptr\""); + } + + if (!ctx_ptr) { + GAPI_LOG_WARNING(nullptr, "Empty \"ctx_ptr\" is not allowed"); + throw std::logic_error("Invalid param: \"ctx_ptr\""); + } + mfxVariant accel_mode = cfg_param_to_mfx_variant(*accel_mode_it); + + switch(accel_mode.Data.U32) { + case MFX_ACCEL_MODE_VIA_D3D11: { +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 + suggested_device = IDeviceSelector::create(device_ptr, device_id, AccelType::DX11); + ID3D11Device* dx_device_ptr = + reinterpret_cast(suggested_device.get_ptr()); + dx_device_ptr->AddRef(); + + suggested_context = IDeviceSelector::create(ctx_ptr, AccelType::DX11); + ID3D11DeviceContext* dx_ctx_ptr = + reinterpret_cast(suggested_context.get_ptr()); + dx_ctx_ptr->AddRef(); +#else + GAPI_LOG_WARNING(nullptr, "Unavailable \"mfxImplDescription.AccelerationMode: MFX_ACCEL_MODE_VIA_D3D11\"" + "was chosen for current project configuration"); + throw std::logic_error("Unsupported \"mfxImplDescription.AccelerationMode: MFX_ACCEL_MODE_VIA_D3D11\""); +#endif // HAVE_DIRECTX +#endif // HAVE_D3D11 + break; + } + case MFX_ACCEL_MODE_NA: { + GAPI_LOG_WARNING(nullptr, "Incompatible \"mfxImplDescription.AccelerationMode: MFX_ACCEL_MODE_NA\" with " + "\"device_ptr\" and \"ctx_ptr\" arguments. " + "You should not clarify these arguments with \"MFX_ACCEL_MODE_NA\" mode"); + throw std::logic_error("Incompatible param: MFX_ACCEL_MODE_NA"); + } + default: + throw std::logic_error("Unsupported \"mfxImplDescription.AccelerationMode\" requested: " + + std::to_string(accel_mode.Data.U32)); + break; + } +} + +CfgParamDeviceSelector::~CfgParamDeviceSelector() { + GAPI_LOG_INFO(nullptr, "release context: " << suggested_context.get_ptr()); + AccelType ctype = suggested_context.get_type(); + switch(ctype) { + case AccelType::HOST: + //nothing to do + break; + case AccelType::DX11: { +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 + ID3D11DeviceContext* device_ctx_ptr = + reinterpret_cast(suggested_context.get_ptr()); + device_ctx_ptr->Release(); + device_ctx_ptr = nullptr; +#endif // HAVE_DIRECTX +#endif // HAVE_D3D11 + break; + } + default: + break; + } + + GAPI_LOG_INFO(nullptr, "release device by name: " << + suggested_device.get_name() << + ", ptr: " << suggested_device.get_ptr()); + AccelType dtype = suggested_device.get_type(); + switch(dtype) { + case AccelType::HOST: + //nothing to do + break; + case AccelType::DX11: { +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 + ID3D11Device* device_ptr = reinterpret_cast(suggested_device.get_ptr()); + device_ptr->Release(); + device_ptr = nullptr; +#endif // HAVE_DIRECTX +#endif // HAVE_D3D11 + break; + } + default: + break; + } +} + +CfgParamDeviceSelector::DeviceScoreTable CfgParamDeviceSelector::select_devices() const { + return {std::make_pair(Score::MaxActivePriority, suggested_device)}; +} + +CfgParamDeviceSelector::DeviceContexts CfgParamDeviceSelector::select_context() { + return {suggested_context}; +} + +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.hpp b/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.hpp new file mode 100644 index 0000000000..2a55fb09cf --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.hpp @@ -0,0 +1,44 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_CFG_PARAM_DEVICE_SELECTOR_HPP +#define GAPI_STREAMING_ONEVPL_CFG_PARAM_DEVICE_SELECTOR_HPP + +#ifdef HAVE_ONEVPL + +#include +#include +#include + +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +struct GAPI_EXPORTS CfgParamDeviceSelector final: public IDeviceSelector { + CfgParamDeviceSelector(const CfgParams& params = {}); + CfgParamDeviceSelector(Device::Ptr device_ptr, + const std::string& device_id, + Context::Ptr ctx_ptr, + const CfgParams& params); + ~CfgParamDeviceSelector(); + + DeviceScoreTable select_devices() const override; + DeviceContexts select_context() override; + +private: + Device suggested_device; + Context suggested_context; +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif //HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_CFG_PARAM_DEVICE_SELECTOR_HPP diff --git a/modules/gapi/src/streaming/onevpl/device_selector_interface.cpp b/modules/gapi/src/streaming/onevpl/device_selector_interface.cpp new file mode 100644 index 0000000000..1ac88bd807 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/device_selector_interface.cpp @@ -0,0 +1,87 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include +#include +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +const char* to_cstring(AccelType type) { + + switch(type) { + case AccelType::HOST: + return "HOST"; + case AccelType::DX11: + return "DX11"; + default: + GAPI_DbgAssert(false && "Unexpected AccelType"); + break; + } + return "UNKNOWN"; +} + +Device::Device(Ptr device_ptr, const std::string& device_name, AccelType device_type) : + name(device_name), + ptr(device_ptr), + type(device_type) { +} + +Device::~Device() { +} + +const std::string& Device::get_name() const { + return name; +} + +Device::Ptr Device::get_ptr() const { + return ptr; +} + +AccelType Device::get_type() const { + return type; +} + +Context::Context(Ptr ctx_ptr, AccelType ctx_type) : + ptr(ctx_ptr), + type(ctx_type) { +} + +Context::~Context() { +} + +Context::Ptr Context::get_ptr() const { + return ptr; +} + +AccelType Context::get_type() const { + return type; +} + +IDeviceSelector::Score::Score(Type val) : + value(val) { +} + +IDeviceSelector::Score::~Score() { +} + +IDeviceSelector::Score::operator Type () const { + return value; +} +IDeviceSelector::Score::Type IDeviceSelector::Score::get() const { + return value; +} + +IDeviceSelector::~IDeviceSelector() { +} + +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/source.cpp b/modules/gapi/src/streaming/onevpl/source.cpp index 0decb1358b..806017a90d 100644 --- a/modules/gapi/src/streaming/onevpl/source.cpp +++ b/modules/gapi/src/streaming/onevpl/source.cpp @@ -8,6 +8,8 @@ #include "streaming/onevpl/source_priv.hpp" #include "streaming/onevpl/file_data_provider.hpp" +#include "streaming/onevpl/cfg_param_device_selector.hpp" + namespace cv { namespace gapi { namespace wip { @@ -15,27 +17,82 @@ namespace onevpl { #ifdef HAVE_ONEVPL GSource::GSource(const std::string& filePath, const CfgParams& cfg_params) : - GSource(std::unique_ptr(new GSource::Priv(std::make_shared(filePath), - cfg_params))) { + GSource(filePath, cfg_params, std::make_shared(cfg_params)) { + if (filePath.empty()) { + util::throw_error(std::logic_error("Cannot create 'GSource' on empty source file name")); + } +} +GSource::GSource(const std::string& filePath, + const CfgParams& cfg_params, + const std::string& device_id, + void* accel_device_ptr, + void* accel_ctx_ptr) : + GSource(filePath, cfg_params, + std::make_shared(accel_device_ptr, device_id, + accel_ctx_ptr, cfg_params)) { +} + +GSource::GSource(const std::string& filePath, + const CfgParams& cfg_params, + std::shared_ptr selector) : + GSource(std::make_shared(filePath), cfg_params, selector) { if (filePath.empty()) { util::throw_error(std::logic_error("Cannot create 'GSource' on empty source file name")); } } GSource::GSource(std::shared_ptr source, const CfgParams& cfg_params) : - GSource(std::unique_ptr(new GSource::Priv(source, cfg_params))) { + GSource(source, cfg_params, + std::make_shared(cfg_params)) { } + +GSource::GSource(std::shared_ptr source, + const CfgParams& cfg_params, + const std::string& device_id, + void* accel_device_ptr, + void* accel_ctx_ptr) : + GSource(source, cfg_params, + std::make_shared(accel_device_ptr, device_id, + accel_ctx_ptr, cfg_params)) { +} + +// common delegating parameters c-tor +GSource::GSource(std::shared_ptr source, + const CfgParams& cfg_params, + std::shared_ptr selector) : + GSource(std::unique_ptr(new GSource::Priv(source, cfg_params, selector))) { +} + #else GSource::GSource(const std::string&, const CfgParams&) { GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); } +GSource::GSource(const std::string&, const CfgParams&, const std::string&, + void*, void*) { + GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); +} + +GSource::GSource(const std::string&, const CfgParams&, std::shared_ptr) { + GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); +} + GSource::GSource(std::shared_ptr, const CfgParams&) { GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); } + +GSource::GSource(std::shared_ptr, const CfgParams&, + const std::string&, void*, void*) { + GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); +} + +GSource::GSource(std::shared_ptr, const CfgParams&, std::shared_ptr) { + GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); +} #endif +// final delegating c-tor GSource::GSource(std::unique_ptr&& impl) : IStreamSource(), m_priv(std::move(impl)) { diff --git a/modules/gapi/src/streaming/onevpl/source_priv.cpp b/modules/gapi/src/streaming/onevpl/source_priv.cpp index 28d438a947..d00074925d 100644 --- a/modules/gapi/src/streaming/onevpl/source_priv.cpp +++ b/modules/gapi/src/streaming/onevpl/source_priv.cpp @@ -58,8 +58,10 @@ GSource::Priv::Priv() : GAPI_LOG_INFO(nullptr, "Initialized MFX handle: " << mfx_handle); } -GSource::Priv::Priv(std::shared_ptr provider, const std::vector& params) : - GSource::Priv() +GSource::Priv::Priv(std::shared_ptr provider, + const std::vector& params, + std::shared_ptr) : + GSource::Priv() { // Enable Config if (params.empty()) diff --git a/modules/gapi/src/streaming/onevpl/source_priv.hpp b/modules/gapi/src/streaming/onevpl/source_priv.hpp index cdaab4eb6a..955184c05c 100644 --- a/modules/gapi/src/streaming/onevpl/source_priv.hpp +++ b/modules/gapi/src/streaming/onevpl/source_priv.hpp @@ -38,7 +38,8 @@ class ProcessingEngineBase; struct GSource::Priv { explicit Priv(std::shared_ptr provider, - const std::vector& params); + const std::vector& params, + std::shared_ptr selector); ~Priv(); static const std::vector& getDefaultCfgParams(); diff --git a/modules/gapi/test/streaming/gapi_streaming_vpl_device_selector.cpp b/modules/gapi/test/streaming/gapi_streaming_vpl_device_selector.cpp new file mode 100644 index 0000000000..2f42742b88 --- /dev/null +++ b/modules/gapi/test/streaming/gapi_streaming_vpl_device_selector.cpp @@ -0,0 +1,229 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + + +#include "../test_precomp.hpp" + +#include "../common/gapi_tests_common.hpp" + +#include +#include + +#include + +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 +#pragma comment(lib,"d3d11.lib") + +// get rid of generate macro max/min/etc from DX side +#define D3D11_NO_HELPERS +#define NOMINMAX +#include +#include +#include "opencv2/core/directx.hpp" +#undef D3D11_NO_HELPERS +#undef NOMINMAX +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX + +#ifdef HAVE_ONEVPL +#include +#include "streaming/onevpl/cfg_param_device_selector.hpp" + +namespace opencv_test +{ +namespace +{ + +void test_dev_eq(const typename cv::gapi::wip::onevpl::IDeviceSelector::DeviceScoreTable::value_type &scored_device, + cv::gapi::wip::onevpl::IDeviceSelector::Score expected_score, + cv::gapi::wip::onevpl::AccelType expected_type, + cv::gapi::wip::onevpl::Device::Ptr expected_ptr) { + EXPECT_EQ(std::get<0>(scored_device), expected_score); + EXPECT_EQ(std::get<1>(scored_device).get_type(), expected_type); + EXPECT_EQ(std::get<1>(scored_device).get_ptr(), expected_ptr); +} + +void test_ctx_eq(const typename cv::gapi::wip::onevpl::IDeviceSelector::DeviceContexts::value_type &ctx, + cv::gapi::wip::onevpl::AccelType expected_type, + cv::gapi::wip::onevpl::Context::Ptr expected_ptr) { + EXPECT_EQ(ctx.get_type(), expected_type); + EXPECT_EQ(ctx.get_ptr(), expected_ptr); +} + +void test_host_dev_eq(const typename cv::gapi::wip::onevpl::IDeviceSelector::DeviceScoreTable::value_type &scored_device, + cv::gapi::wip::onevpl::IDeviceSelector::Score expected_score) { + test_dev_eq(scored_device, expected_score, + cv::gapi::wip::onevpl::AccelType::HOST, nullptr); +} + +void test_host_ctx_eq(const typename cv::gapi::wip::onevpl::IDeviceSelector::DeviceContexts::value_type &ctx) { + test_ctx_eq(ctx, cv::gapi::wip::onevpl::AccelType::HOST, nullptr); +} + +TEST(OneVPL_Source_Device_Selector_CfgParam, DefaultDevice) +{ + using namespace cv::gapi::wip::onevpl; + CfgParamDeviceSelector selector; + IDeviceSelector::DeviceScoreTable devs = selector.select_devices(); + EXPECT_EQ(devs.size(), 1); + test_host_dev_eq(*devs.begin(), IDeviceSelector::Score::MaxActivePriority); + + IDeviceSelector::DeviceContexts ctxs = selector.select_context(); + EXPECT_EQ(ctxs.size(), 1); + test_host_ctx_eq(*ctxs.begin()); +} + +TEST(OneVPL_Source_Device_Selector_CfgParam, DefaultDeviceWithEmptyCfgParam) +{ + using namespace cv::gapi::wip::onevpl; + std::vector empty_params; + CfgParamDeviceSelector selector(empty_params); + IDeviceSelector::DeviceScoreTable devs = selector.select_devices(); + EXPECT_EQ(devs.size(), 1); + test_host_dev_eq(*devs.begin(), IDeviceSelector::Score::MaxActivePriority); + IDeviceSelector::DeviceContexts ctxs = selector.select_context(); + EXPECT_EQ(ctxs.size(), 1); + test_host_ctx_eq(*ctxs.begin()); +} + +TEST(OneVPL_Source_Device_Selector_CfgParam, DefaultDeviceWithAccelNACfgParam) +{ + using namespace cv::gapi::wip::onevpl; + std::vector cfg_params_w_no_accel; + cfg_params_w_no_accel.push_back(CfgParam::create("mfxImplDescription.AccelerationMode", + MFX_ACCEL_MODE_NA)); + CfgParamDeviceSelector selector(cfg_params_w_no_accel); + IDeviceSelector::DeviceScoreTable devs = selector.select_devices(); + EXPECT_EQ(devs.size(), 1); + test_host_dev_eq(*devs.begin(), IDeviceSelector::Score::MaxActivePriority); + + IDeviceSelector::DeviceContexts ctxs = selector.select_context(); + EXPECT_EQ(ctxs.size(), 1); + test_host_ctx_eq(*ctxs.begin()); +} + +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 +TEST(OneVPL_Source_Device_Selector_CfgParam, DefaultDeviceWithEmptyCfgParam_DX11_ENABLED) +{ + using namespace cv::gapi::wip::onevpl; + std::vector empty_params; + CfgParamDeviceSelector selector(empty_params); + IDeviceSelector::DeviceScoreTable devs = selector.select_devices(); + EXPECT_EQ(devs.size(), 1); + test_host_dev_eq(*devs.begin(), IDeviceSelector::Score::MaxActivePriority); + + IDeviceSelector::DeviceContexts ctxs = selector.select_context(); + EXPECT_EQ(ctxs.size(), 1); + test_host_ctx_eq(*ctxs.begin()); +} + +TEST(OneVPL_Source_Device_Selector_CfgParam, DefaultDeviceWithDX11AccelCfgParam_DX11_ENABLED) +{ + using namespace cv::gapi::wip::onevpl; + std::vector cfg_params_w_dx11; + cfg_params_w_dx11.push_back(CfgParam::create("mfxImplDescription.AccelerationMode", + MFX_ACCEL_MODE_VIA_D3D11)); + std::unique_ptr selector_ptr; + EXPECT_NO_THROW(selector_ptr.reset(new CfgParamDeviceSelector(cfg_params_w_dx11))); + IDeviceSelector::DeviceScoreTable devs = selector_ptr->select_devices(); + + EXPECT_EQ(devs.size(), 1); + test_dev_eq(*devs.begin(), IDeviceSelector::Score::MaxActivePriority, + AccelType::DX11, + std::get<1>(*devs.begin()).get_ptr() /* compare just type */); + + IDeviceSelector::DeviceContexts ctxs = selector_ptr->select_context(); + EXPECT_EQ(ctxs.size(), 1); + EXPECT_TRUE(ctxs.begin()->get_ptr()); +} + +TEST(OneVPL_Source_Device_Selector_CfgParam, NULLDeviceWithDX11AccelCfgParam_DX11_ENABLED) +{ + using namespace cv::gapi::wip::onevpl; + std::vector cfg_params_w_dx11; + cfg_params_w_dx11.push_back(CfgParam::create("mfxImplDescription.AccelerationMode", + MFX_ACCEL_MODE_VIA_D3D11)); + Device::Ptr empty_device_ptr = nullptr; + Context::Ptr empty_ctx_ptr = nullptr; + EXPECT_THROW(CfgParamDeviceSelector sel(empty_device_ptr, "GPU", + empty_ctx_ptr, + cfg_params_w_dx11), + std::logic_error); // empty_device_ptr must be invalid +} + +TEST(OneVPL_Source_Device_Selector_CfgParam, ExternalDeviceWithDX11AccelCfgParam_DX11_ENABLED) +{ + using namespace cv::gapi::wip::onevpl; + ID3D11Device *device = nullptr; + ID3D11DeviceContext* device_context = nullptr; + { + UINT flags = 0; + D3D_FEATURE_LEVEL features[] = { D3D_FEATURE_LEVEL_11_1, + D3D_FEATURE_LEVEL_11_0, + }; + D3D_FEATURE_LEVEL feature_level; + + // Create the Direct3D 11 API device object and a corresponding context. + HRESULT err = D3D11CreateDevice(nullptr, D3D_DRIVER_TYPE_HARDWARE, + nullptr, flags, + features, + ARRAYSIZE(features), D3D11_SDK_VERSION, + &device, &feature_level, &device_context); + EXPECT_FALSE(FAILED(err)); + } + + std::unique_ptr selector_ptr; + std::vector cfg_params_w_dx11; + cfg_params_w_dx11.push_back(CfgParam::create("mfxImplDescription.AccelerationMode", + MFX_ACCEL_MODE_VIA_D3D11)); + EXPECT_NO_THROW(selector_ptr.reset(new CfgParamDeviceSelector(device, "GPU", + device_context, + cfg_params_w_dx11))); + IDeviceSelector::DeviceScoreTable devs = selector_ptr->select_devices(); + + EXPECT_EQ(devs.size(), 1); + test_dev_eq(*devs.begin(), IDeviceSelector::Score::MaxActivePriority, + AccelType::DX11, device); + + IDeviceSelector::DeviceContexts ctxs = selector_ptr->select_context(); + EXPECT_EQ(ctxs.size(), 1); + EXPECT_EQ(reinterpret_cast(ctxs.begin()->get_ptr()), + device_context); +} + +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX + +#ifndef HAVE_DIRECTX +#ifndef HAVE_D3D11 +TEST(OneVPL_Source_Device_Selector_CfgParam, DX11DeviceFromCfgParamWithDX11Disabled) +{ + using namespace cv::gapi::wip::onevpl; + std::vector cfg_params_w_non_existed_dx11; + cfg_params_w_not_existed_dx11.push_back(CfgParam::create("mfxImplDescription.AccelerationMode", + MFX_ACCEL_MODE_VIA_D3D11)); + EXPECT_THROW(CfgParamDeviceSelector{cfg_params_w_non_existed_dx11}, + std::logic_error); +} +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX + +TEST(OneVPL_Source_Device_Selector_CfgParam, UnknownPtrDeviceFromCfgParam) +{ + using namespace cv::gapi::wip::onevpl; + std::vector empty_params; + Device::Ptr empty_device_ptr = nullptr; + Context::Ptr empty_ctx_ptr = nullptr; + EXPECT_THROW(CfgParamDeviceSelector sel(empty_device_ptr, "", + empty_ctx_ptr, + empty_params), + std::logic_error); // params must describe device_ptr explicitly +} +} +} // namespace opencv_test +#endif // HAVE_ONEVPL From f36c268b9ed70e4a7051084350c35a98de5ff1de Mon Sep 17 00:00:00 2001 From: MaximMilashchenko <67949029+MaximMilashchenko@users.noreply.github.com> Date: Wed, 20 Oct 2021 16:18:24 +0300 Subject: [PATCH 016/226] Merge pull request #19721 from MaximMilashchenko:Audio add audio support in cap_msmf * audio msmf * fixed warnings * minor fix * fixed SampleTime MSMF * minor fix, fixed audio test, retrieveAudioFrame * fixed warnings * impelemented sync audio and video stream with start offset * fixed error * fixed docs * fixed audio sample * CAP_PROP_AUDIO_POS, minor fixed * fixed warnings * videoio(MSMF): update audio test checks, add debug logging * fixed * fixed desynchronization of time positions, warnings * fixed warnings * videoio(audio): tune tests checks * videoio(audio): update properties description * build warnings Co-authored-by: Alexander Alekhin --- modules/videoio/include/opencv2/videoio.hpp | 11 + modules/videoio/src/cap_msmf.cpp | 998 +++++++++++++++--- modules/videoio/test/test_audio.cpp | 273 +++++ modules/videoio/test/test_microphone.cpp | 41 + samples/cpp/videocapture_audio.cpp | 59 ++ .../cpp/videocapture_audio_combination.cpp | 69 ++ samples/cpp/videocapture_microphone.cpp | 57 + 7 files changed, 1366 insertions(+), 142 deletions(-) create mode 100644 modules/videoio/test/test_audio.cpp create mode 100644 modules/videoio/test/test_microphone.cpp create mode 100644 samples/cpp/videocapture_audio.cpp create mode 100644 samples/cpp/videocapture_audio_combination.cpp create mode 100644 samples/cpp/videocapture_microphone.cpp diff --git a/modules/videoio/include/opencv2/videoio.hpp b/modules/videoio/include/opencv2/videoio.hpp index 3276d0d5e4..4b5bc135bc 100644 --- a/modules/videoio/include/opencv2/videoio.hpp +++ b/modules/videoio/include/opencv2/videoio.hpp @@ -189,6 +189,17 @@ enum VideoCaptureProperties { CAP_PROP_OPEN_TIMEOUT_MSEC=53, //!< (**open-only**) timeout in milliseconds for opening a video capture (applicable for FFmpeg back-end only) CAP_PROP_READ_TIMEOUT_MSEC=54, //!< (**open-only**) timeout in milliseconds for reading from a video capture (applicable for FFmpeg back-end only) CAP_PROP_STREAM_OPEN_TIME_USEC =55, // #include #include +#include #include #include #include @@ -69,7 +70,6 @@ static void init_MFCreateDXGIDeviceManager() #endif #include - #include #include // QISearch @@ -108,6 +108,13 @@ public: { } + void swap(_In_ ComPtr& lp) + { + ComPtr tmp(p); + p = lp.p; + lp.p = tmp.p; + tmp = NULL; + } T** operator&() { CV_Assert(p == NULL); @@ -155,6 +162,7 @@ template inline T absDiff(T a, T b) { return a >= b ? a - b : b - a // Structure for collecting info about types of video which are supported by current video device struct MediaType { + //video param UINT32 width; UINT32 height; INT32 stride; // stride is negative if image is bottom-up @@ -165,9 +173,17 @@ struct MediaType UINT32 aspectRatioDenom; UINT32 sampleSize; UINT32 interlaceMode; + //audio param + UINT32 bit_per_sample; + UINT32 nChannels; + UINT32 nAvgBytesPerSec; + UINT32 nSamplesPerSec; + GUID majorType; // video or audio GUID subType; // fourCC + _ComPtr Type; MediaType(IMFMediaType *pType = 0) : + Type(pType), width(0), height(0), stride(0), isFixedSize(true), @@ -175,23 +191,38 @@ struct MediaType aspectRatioNum(1), aspectRatioDenom(1), sampleSize(0), interlaceMode(0), - majorType(MFMediaType_Video), + bit_per_sample(0), + nChannels(0), + nAvgBytesPerSec(0), + nSamplesPerSec(0), + majorType({ 0 }),//MFMediaType_Video subType({ 0 }) { if (pType) { - MFGetAttributeSize(pType, MF_MT_FRAME_SIZE, &width, &height); - pType->GetUINT32(MF_MT_DEFAULT_STRIDE, (UINT32*)&stride); // value is stored as UINT32 but should be casted to INT3) - pType->GetUINT32(MF_MT_FIXED_SIZE_SAMPLES, &isFixedSize); - MFGetAttributeRatio(pType, MF_MT_FRAME_RATE, &frameRateNum, &frameRateDenom); - MFGetAttributeRatio(pType, MF_MT_PIXEL_ASPECT_RATIO, &aspectRatioNum, &aspectRatioDenom); - pType->GetUINT32(MF_MT_SAMPLE_SIZE, &sampleSize); - pType->GetUINT32(MF_MT_INTERLACE_MODE, &interlaceMode); pType->GetGUID(MF_MT_MAJOR_TYPE, &majorType); pType->GetGUID(MF_MT_SUBTYPE, &subType); + if (majorType == MFMediaType_Audio) + { + pType->GetUINT32(MF_MT_AUDIO_BITS_PER_SAMPLE, &bit_per_sample); + pType->GetUINT32(MF_MT_AUDIO_NUM_CHANNELS, &nChannels); + pType->GetUINT32(MF_MT_AUDIO_AVG_BYTES_PER_SECOND, &nAvgBytesPerSec); + pType->GetUINT32(MF_MT_AUDIO_FLOAT_SAMPLES_PER_SECOND, &nSamplesPerSec); + } + else if (majorType == MFMediaType_Video) + { + MFGetAttributeSize(pType, MF_MT_FRAME_SIZE, &width, &height); + pType->GetUINT32(MF_MT_DEFAULT_STRIDE, (UINT32*)&stride); // value is stored as UINT32 but should be casted to INT3) + pType->GetUINT32(MF_MT_FIXED_SIZE_SAMPLES, &isFixedSize); + MFGetAttributeRatio(pType, MF_MT_FRAME_RATE, &frameRateNum, &frameRateDenom); + MFGetAttributeRatio(pType, MF_MT_PIXEL_ASPECT_RATIO, &aspectRatioNum, &aspectRatioDenom); + pType->GetUINT32(MF_MT_SAMPLE_SIZE, &sampleSize); + pType->GetUINT32(MF_MT_INTERLACE_MODE, &interlaceMode); + pType->GetUINT32(MF_MT_INTERLACE_MODE, &interlaceMode); + } } } - static MediaType createDefault() + static MediaType createDefault_Video() { MediaType res; res.width = 640; @@ -199,11 +230,24 @@ struct MediaType res.setFramerate(30.0); return res; } - inline bool isEmpty() const + static MediaType createDefault_Audio() { - return width == 0 && height == 0; + MediaType res; + res.majorType = MFMediaType_Audio; + res.subType = MFAudioFormat_PCM; + res.bit_per_sample = 16; + res.nChannels = 1; + res.nSamplesPerSec = 44100; + return res; } - _ComPtr createMediaType() const + inline bool isEmpty(bool flag = false) const + { + if (!flag) + return width == 0 && height == 0; + else + return nChannels == 0; + } + _ComPtr createMediaType_Video() const { _ComPtr res; MFCreateMediaType(&res); @@ -225,6 +269,22 @@ struct MediaType res->SetGUID(MF_MT_SUBTYPE, subType); return res; } + _ComPtr createMediaType_Audio() const + { + _ComPtr res; + MFCreateMediaType(&res); + if (majorType != GUID()) + res->SetGUID(MF_MT_MAJOR_TYPE, majorType); + if (subType != GUID()) + res->SetGUID(MF_MT_SUBTYPE, subType); + if (bit_per_sample != 0) + res->SetUINT32(MF_MT_AUDIO_BITS_PER_SAMPLE, bit_per_sample); + if (nChannels != 0) + res->SetUINT32(MF_MT_AUDIO_NUM_CHANNELS, nChannels); + if (nSamplesPerSec != 0) + res->SetUINT32(MF_MT_AUDIO_SAMPLES_PER_SECOND, nSamplesPerSec); + return res; + } void setFramerate(double fps) { frameRateNum = (UINT32)cvRound(fps * 1000.0); @@ -246,7 +306,7 @@ struct MediaType return wdiff + hdiff; } // check if 'this' is better than 'other' comparing to reference - bool isBetterThan(const MediaType& other, const MediaType& ref) const + bool VideoIsBetterThan(const MediaType& other, const MediaType& ref) const { const unsigned long thisDiff = resolutionDiff(ref); const unsigned long otherDiff = other.resolutionDiff(ref); @@ -268,6 +328,24 @@ struct MediaType } return false; } + bool AudioIsBetterThan(const MediaType& other, const MediaType& ref) const + { + double thisDiff = absDiff(nChannels, ref.nChannels); + double otherDiff = absDiff(other.nChannels, ref.nChannels); + if (otherDiff < thisDiff) + { + thisDiff = absDiff(bit_per_sample, ref.bit_per_sample); + otherDiff = absDiff(bit_per_sample, ref.bit_per_sample); + if (otherDiff < thisDiff) + { + thisDiff = absDiff(nSamplesPerSec, ref.nSamplesPerSec); + otherDiff = absDiff(nSamplesPerSec, ref.nSamplesPerSec); + if (otherDiff < thisDiff) + return true; + } + } + return false; + } }; void printFormat(std::ostream& out, const GUID& fmt) @@ -405,7 +483,7 @@ public: return S_OK; } - HRESULT Wait(DWORD dwMilliseconds, _ComPtr& videoSample, BOOL& pbEOS) + HRESULT Wait(DWORD dwMilliseconds, _ComPtr& mediaSample, BOOL& pbEOS) { pbEOS = FALSE; @@ -423,14 +501,14 @@ public: if (!pbEOS) { cv::AutoLock lock(m_mutex); - videoSample = m_lastSample; - CV_Assert(videoSample); + mediaSample = m_lastSample; + CV_Assert(mediaSample); m_lastSample.Release(); ResetEvent(m_hEvent); // event is auto-reset, but we need this forced reset due time gap between wait() and mutex hold. } - return m_hrStatus; } + private: // Destructor is private. Caller should call Release. virtual ~SourceReaderCB() @@ -496,22 +574,67 @@ public: } } } + void countNumberOfAudioStreams(DWORD &numberOfAudioStreams) + { + std::pair best; + std::map::const_iterator i = formats.begin(); + for (; i != formats.end(); ++i) + { + if(i->second.majorType == MFMediaType_Audio) + { + if(best.second.isEmpty() || i->first.stream != best.first.stream) + { + numberOfAudioStreams++; + best = *i; + } + } + } + } std::pair findBestVideoFormat(const MediaType& newType) { std::pair best; std::map::const_iterator i = formats.begin(); for (; i != formats.end(); ++i) { - if (i->second.majorType != MFMediaType_Video) - continue; - if (newType.isEmpty()) // file input - choose first returned media type + if (i->second.majorType == MFMediaType_Video) { - best = *i; - break; + if (best.second.isEmpty() || i->second.VideoIsBetterThan(best.second, newType)) + { + best = *i; + } } - if (best.second.isEmpty() || i->second.isBetterThan(best.second, newType)) + } + return best; + } + std::pair findBestAudioFormat(const MediaType& newType) + { + std::pair best; + std::map::const_iterator i = formats.begin(); + best = *i; + for (; i != formats.end(); ++i) + { + if (i->second.majorType == MFMediaType_Audio) { - best = *i; + if ( i->second.AudioIsBetterThan(best.second, newType)) + { + best = *i; + } + } + } + return best; + } + std::pair findAudioFormatByStream(const DWORD StreamIndex) + { + std::pair best; + std::map::const_iterator i = formats.begin(); + for (; i != formats.end(); ++i) + { + if (i->second.majorType == MFMediaType_Audio) + { + if ((*i).first.stream == StreamIndex) + { + best = *i; + } } } return best; @@ -586,21 +709,30 @@ public: virtual void close(); virtual double getProperty(int) const CV_OVERRIDE; virtual bool setProperty(int, double) CV_OVERRIDE; + bool grabAudioFrame(); + bool grabVideoFrame(); virtual bool grabFrame() CV_OVERRIDE; + bool retrieveAudioFrame(int, OutputArray); + bool retrieveVideoFrame(OutputArray); virtual bool retrieveFrame(int, cv::OutputArray) CV_OVERRIDE; virtual bool isOpened() const CV_OVERRIDE { return isOpen; } virtual int getCaptureDomain() CV_OVERRIDE { return CV_CAP_MSMF; } protected: - bool configureOutput(MediaType newType, cv::uint32_t outFormat); + bool configureOutput(); + bool configureAudioOutput(MediaType newType); + bool configureVideoOutput(MediaType newType, cv::uint32_t outFormat); bool setTime(double time, bool rough); + bool setTime(int numberFrame); bool configureHW(bool enable); + bool configureStreams(const cv::VideoCaptureParameters&); + bool setAudioProperties(const cv::VideoCaptureParameters&); template bool readComplexPropery(long prop, long& val) const; template bool writeComplexProperty(long prop, double val, long flags); _ComPtr getDefaultSourceConfig(UINT32 num = 10); - bool initStream(DWORD streamID, const MediaType& mt); + bool initStream(DWORD streamID, const MediaType mt); bool openFinalize_(const VideoCaptureParameters* params); @@ -615,17 +747,49 @@ protected: _ComPtr D3DMgr; #endif _ComPtr videoFileSource; - _ComPtr videoSample; _ComPtr readCallback; // non-NULL for "live" streams (camera capture) - DWORD dwStreamIndex; + std::vector dwStreamIndices; + std::vector<_ComPtr> audioSamples; + _ComPtr impendingVideoSample; + _ComPtr usedVideoSample; + DWORD dwVideoStreamIndex; + DWORD dwAudioStreamIndex; MediaType nativeFormat; - MediaType captureFormat; - int outputFormat; + MediaType captureVideoFormat; + MediaType captureAudioFormat; + bool device_status; //on or off + int videoStream; // look at CAP_PROP_VIDEO_STREAM + int audioStream; // look at CAP_PROP_AUDIO_STREAM + bool vEOS; + bool aEOS; + unsigned int audioBaseIndex; + int outputVideoFormat; + int outputAudioFormat; bool convertFormat; MFTIME duration; LONGLONG frameStep; - LONGLONG sampleTime; + LONGLONG nFrame; + LONGLONG impendingVideoSampleTime; + LONGLONG usedVideoSampleTime; + LONGLONG videoStartOffset; + LONGLONG videoSampleDuration; + LONGLONG requiredAudioTime; + LONGLONG audioSampleTime; + LONGLONG audioStartOffset; + LONGLONG audioSampleDuration; + LONGLONG audioTime; + LONGLONG chunkLengthOfBytes; + LONGLONG givenAudioTime; + LONGLONG numberOfAdditionalAudioBytes; // the number of additional bytes required to align the audio chunk + double bufferedAudioDuration; + LONGLONG audioSamplePos; + DWORD numberOfAudioStreams; + Mat audioFrame; + std::deque bufferAudioData; bool isOpen; + bool grabIsDone; + bool syncLastFrame; + bool lastFrame; }; CvCapture_MSMF::CvCapture_MSMF(): @@ -640,15 +804,42 @@ CvCapture_MSMF::CvCapture_MSMF(): D3DMgr(NULL), #endif videoFileSource(NULL), - videoSample(NULL), readCallback(NULL), - dwStreamIndex(0), - outputFormat(CV_CAP_MODE_BGR), + impendingVideoSample(NULL), + usedVideoSample(NULL), + dwVideoStreamIndex(0), + dwAudioStreamIndex(0), + device_status(false), + videoStream(0), + audioStream(-1), + vEOS(false), + aEOS(false), + audioBaseIndex(1), + outputVideoFormat(CV_CAP_MODE_BGR), + outputAudioFormat(CV_16S), convertFormat(true), duration(0), frameStep(0), - sampleTime(0), - isOpen(false) + nFrame(0), + impendingVideoSampleTime(0), + usedVideoSampleTime(0), + videoStartOffset(-1), + videoSampleDuration(0), + requiredAudioTime(0), + audioSampleTime(0), + audioStartOffset(-1), + audioSampleDuration(0), + audioTime(0), + chunkLengthOfBytes(0), + givenAudioTime(0), + numberOfAdditionalAudioBytes(0), + bufferedAudioDuration(0), + audioSamplePos(0), + numberOfAudioStreams(0), + isOpen(false), + grabIsDone(false), + syncLastFrame(true), + lastFrame(false) { } @@ -663,29 +854,37 @@ void CvCapture_MSMF::close() if (isOpen) { isOpen = false; - videoSample.Release(); + usedVideoSample.Release(); + for (auto item : audioSamples) + item.Release(); videoFileSource.Release(); + device_status = false; camid = -1; filename.clear(); } readCallback.Release(); } -bool CvCapture_MSMF::initStream(DWORD streamID, const MediaType& mt) +bool CvCapture_MSMF::initStream(DWORD streamID, const MediaType mt) { CV_LOG_DEBUG(NULL, "Init stream " << streamID << " with MediaType " << mt); - _ComPtr mediaTypeOut = mt.createMediaType(); - if (FAILED(videoFileSource->SetStreamSelection((DWORD)MF_SOURCE_READER_ALL_STREAMS, false))) + _ComPtr mediaTypesOut; + if (mt.majorType == MFMediaType_Audio) { - CV_LOG_WARNING(NULL, "Failed to reset streams"); - return false; + captureAudioFormat = mt; + mediaTypesOut = mt.createMediaType_Audio(); + } + if (mt.majorType == MFMediaType_Video) + { + captureVideoFormat = mt; + mediaTypesOut = mt.createMediaType_Video(); } if (FAILED(videoFileSource->SetStreamSelection(streamID, true))) { CV_LOG_WARNING(NULL, "Failed to select stream " << streamID); return false; } - HRESULT hr = videoFileSource->SetCurrentMediaType(streamID, NULL, mediaTypeOut.Get()); + HRESULT hr = videoFileSource->SetCurrentMediaType(streamID, NULL, mediaTypesOut.Get()); if (hr == MF_E_TOPO_CODEC_NOT_FOUND) { CV_LOG_WARNING(NULL, "Failed to set mediaType (stream " << streamID << ", " << mt << "(codec not found)"); @@ -701,7 +900,7 @@ bool CvCapture_MSMF::initStream(DWORD streamID, const MediaType& mt) CV_LOG_WARNING(NULL, "Failed to set mediaType (stream " << streamID << ", " << mt << "(HRESULT " << hr << ")"); return false; } - captureFormat = mt; + return true; } @@ -826,7 +1025,52 @@ bool CvCapture_MSMF::configureHW(const VideoCaptureParameters& params) return configureHW(va_type == VIDEO_ACCELERATION_D3D11 || va_type == VIDEO_ACCELERATION_ANY); } -bool CvCapture_MSMF::configureOutput(MediaType newType, cv::uint32_t outFormat) +bool CvCapture_MSMF::configureAudioOutput(MediaType newType) +{ + FormatStorage formats; + formats.read(videoFileSource.Get()); + std::pair bestMatch; + formats.countNumberOfAudioStreams(numberOfAudioStreams); + if (device_status) + bestMatch = formats.findBestAudioFormat(newType); + else + bestMatch = formats.findAudioFormatByStream(audioStream); + if (bestMatch.second.isEmpty(true)) + { + CV_LOG_DEBUG(NULL, "Can not find audio stream with requested parameters"); + return false; + } + dwAudioStreamIndex = bestMatch.first.stream; + dwStreamIndices.push_back(dwAudioStreamIndex); + MediaType newFormat = bestMatch.second; + + newFormat.majorType = MFMediaType_Audio; + newFormat.nSamplesPerSec = 44100; + switch (outputAudioFormat) + { + case CV_8S: + newFormat.subType = MFAudioFormat_PCM; + newFormat.bit_per_sample = 8; + break; + case CV_16S: + newFormat.subType = MFAudioFormat_PCM; + newFormat.bit_per_sample = 16; + break; + case CV_32S: + newFormat.subType = MFAudioFormat_PCM; + newFormat.bit_per_sample = 32; + case CV_32F: + newFormat.subType = MFAudioFormat_Float; + newFormat.bit_per_sample = 32; + break; + default: + break; + } + + return initStream(dwAudioStreamIndex, newFormat); +} + +bool CvCapture_MSMF::configureVideoOutput(MediaType newType, cv::uint32_t outFormat) { FormatStorage formats; formats.read(videoFileSource.Get()); @@ -836,9 +1080,11 @@ bool CvCapture_MSMF::configureOutput(MediaType newType, cv::uint32_t outFormat) CV_LOG_DEBUG(NULL, "Can not find video stream with requested parameters"); return false; } - dwStreamIndex = bestMatch.first.stream; + dwVideoStreamIndex = bestMatch.first.stream; + dwStreamIndices.push_back(dwVideoStreamIndex); nativeFormat = bestMatch.second; MediaType newFormat = nativeFormat; + if (convertFormat) { switch (outFormat) @@ -869,8 +1115,25 @@ bool CvCapture_MSMF::configureOutput(MediaType newType, cv::uint32_t outFormat) } // we select native format first and then our requested format (related issue #12822) if (!newType.isEmpty()) // camera input - initStream(dwStreamIndex, nativeFormat); - return initStream(dwStreamIndex, newFormat); + { + initStream(dwVideoStreamIndex, nativeFormat); + } + return initStream(dwVideoStreamIndex, newFormat); +} + +bool CvCapture_MSMF::configureOutput() +{ + if (FAILED(videoFileSource->SetStreamSelection((DWORD)MF_SOURCE_READER_ALL_STREAMS, false))) + { + CV_LOG_WARNING(NULL, "Failed to reset streams"); + return false; + } + bool tmp = true; + if (videoStream != -1) + tmp = (!device_status)? configureVideoOutput(MediaType(), outputVideoFormat) : configureVideoOutput(MediaType::createDefault_Video(), outputVideoFormat); + if (audioStream != -1) + tmp &= (!device_status)? configureAudioOutput(MediaType()) : configureAudioOutput(MediaType::createDefault_Audio()); + return tmp; } bool CvCapture_MSMF::open(int index, const cv::VideoCaptureParameters* params) @@ -882,10 +1145,19 @@ bool CvCapture_MSMF::open(int index, const cv::VideoCaptureParameters* params) if (params) { configureHW(*params); + configureStreams(*params); + } + if (videoStream != -1 && audioStream != -1 || videoStream == -1 && audioStream == -1) + { + CV_LOG_DEBUG(NULL, "Only one of the properties CAP_PROP_AUDIO_STREAM " << audioStream << " and " << CAP_PROP_VIDEO_STREAM << " must be different from -1"); + return false; } - DeviceList devices; - UINT32 count = devices.read(); + UINT32 count = 0; + if (audioStream != -1) + count = devices.read(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_AUDCAP_GUID); + if (videoStream != -1) + count = devices.read(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID); if (count == 0 || static_cast(index) > count) { CV_LOG_DEBUG(NULL, "Device " << index << " not found (total " << count << " devices)"); @@ -902,14 +1174,14 @@ bool CvCapture_MSMF::open(int index, const cv::VideoCaptureParameters* params) } isOpen = true; + device_status = true; camid = index; readCallback = cb; duration = 0; - if (configureOutput(MediaType::createDefault(), outputFormat)) + if (configureOutput()) { - frameStep = captureFormat.getFrameStep(); + frameStep = captureVideoFormat.getFrameStep(); } - if (isOpen && !openFinalize_(params)) { close(); @@ -928,8 +1200,9 @@ bool CvCapture_MSMF::open(const cv::String& _filename, const cv::VideoCapturePar if (params) { configureHW(*params); + configureStreams(*params); + setAudioProperties(*params); } - // Set source reader parameters _ComPtr attr = getDefaultSourceConfig(); cv::AutoBuffer unicodeFileName(_filename.length() + 1); @@ -937,11 +1210,11 @@ bool CvCapture_MSMF::open(const cv::String& _filename, const cv::VideoCapturePar if (SUCCEEDED(MFCreateSourceReaderFromURL(unicodeFileName.data(), attr.Get(), &videoFileSource))) { isOpen = true; - sampleTime = 0; - if (configureOutput(MediaType(), outputFormat)) + usedVideoSampleTime = 0; + if (configureOutput()) { - frameStep = captureFormat.getFrameStep(); filename = _filename; + frameStep = captureVideoFormat.getFrameStep(); PROPVARIANT var; HRESULT hr; if (SUCCEEDED(hr = videoFileSource->GetPresentationAttribute((DWORD)MF_SOURCE_READER_MEDIASOURCE, MF_PD_DURATION, &var)) && @@ -954,13 +1227,18 @@ bool CvCapture_MSMF::open(const cv::String& _filename, const cv::VideoCapturePar duration = 0; } } - if (isOpen && !openFinalize_(params)) { close(); return false; } - + if (isOpen) + if (audioStream != -1 && videoStream != -1) + { + isOpen = grabFrame(); + if (isOpen) + grabIsDone = true; + } return isOpen; } @@ -997,71 +1275,96 @@ bool CvCapture_MSMF::openFinalize_(const VideoCaptureParameters* params) return true; } -bool CvCapture_MSMF::grabFrame() +bool CvCapture_MSMF::configureStreams(const cv::VideoCaptureParameters& params) { - CV_TRACE_FUNCTION(); - if (readCallback) // async "live" capture mode + if (params.has(CAP_PROP_VIDEO_STREAM)) { - HRESULT hr = 0; - SourceReaderCB* reader = ((SourceReaderCB*)readCallback.Get()); - if (!reader->m_reader) + double value = params.get(CAP_PROP_VIDEO_STREAM); + if (value == -1 || value == 0) + videoStream = static_cast(value); + else { - // Initiate capturing with async callback - reader->m_reader = videoFileSource.Get(); - reader->m_dwStreamIndex = dwStreamIndex; - if (FAILED(hr = videoFileSource->ReadSample(dwStreamIndex, 0, NULL, NULL, NULL, NULL))) - { - CV_LOG_ERROR(NULL, "videoio(MSMF): can't grab frame - initial async ReadSample() call failed: " << hr); - reader->m_reader = NULL; - return false; - } - } - BOOL bEOS = false; - if (FAILED(hr = reader->Wait(10000, videoSample, bEOS))) // 10 sec - { - CV_LOG_WARNING(NULL, "videoio(MSMF): can't grab frame. Error: " << hr); + CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: CAP_PROP_VIDEO_STREAM parameter value is invalid/unsupported: " << value); return false; } - if (bEOS) - { - CV_LOG_WARNING(NULL, "videoio(MSMF): EOS signal. Capture stream is lost"); - return false; - } - sampleTime = reader->m_lastSampleTimestamp; - return true; } - else if (isOpen) + if (params.has(CAP_PROP_AUDIO_STREAM)) { - DWORD streamIndex, flags; - videoSample.Release(); - HRESULT hr; - for(;;) + double value = params.get(CAP_PROP_AUDIO_STREAM); + if (value == -1 || value > -1) + audioStream = static_cast(value); + else + { + CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: CAP_PROP_AUDIO_STREAM parameter value is invalid/unsupported: " << value); + return false; + } + } + return true; +} +bool CvCapture_MSMF::setAudioProperties(const cv::VideoCaptureParameters& params) +{ + if (params.has(CAP_PROP_AUDIO_DATA_DEPTH)) + { + int value = static_cast(params.get(CAP_PROP_AUDIO_DATA_DEPTH)); + if (value != CV_8S && value != CV_16S && value != CV_32S && value != CV_32F) + { + CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: CAP_PROP_AUDIO_DATA_DEPTH parameter value is invalid/unsupported: " << value); + return false; + } + else + { + outputAudioFormat = value; + } + } + if (params.has(CAP_PROP_AUDIO_SYNCHRONIZE)) + { + int value = static_cast(params.get(CAP_PROP_AUDIO_SYNCHRONIZE)); + syncLastFrame = (value != 0) ? true : false; + } + return true; +} + +bool CvCapture_MSMF::grabVideoFrame() +{ + DWORD streamIndex, flags; + HRESULT hr; + usedVideoSample.Release(); + + bool returnFlag = false; + bool stopFlag = false; + if (audioStream != -1) + { + usedVideoSample.swap(impendingVideoSample); + std::swap(usedVideoSampleTime, impendingVideoSampleTime); + } + while (!stopFlag) + { + for (;;) { CV_TRACE_REGION("ReadSample"); if (!SUCCEEDED(hr = videoFileSource->ReadSample( - dwStreamIndex, // Stream index. + dwVideoStreamIndex, // Stream index. 0, // Flags. &streamIndex, // Receives the actual stream index. &flags, // Receives status flags. - &sampleTime, // Receives the time stamp. - &videoSample // Receives the sample or NULL. + &impendingVideoSampleTime, // Receives the time stamp. + &impendingVideoSample // Receives the sample or NULL. ))) break; - if (streamIndex != dwStreamIndex) + if (streamIndex != dwVideoStreamIndex) break; if (flags & (MF_SOURCE_READERF_ERROR | MF_SOURCE_READERF_ALLEFFECTSREMOVED | MF_SOURCE_READERF_ENDOFSTREAM)) break; - if (videoSample) + if (impendingVideoSample) break; if (flags & MF_SOURCE_READERF_STREAMTICK) { CV_LOG_DEBUG(NULL, "videoio(MSMF): Stream tick detected. Retrying to grab the frame"); } } - if (SUCCEEDED(hr)) { - if (streamIndex != dwStreamIndex) + if (streamIndex != dwVideoStreamIndex) { CV_LOG_DEBUG(NULL, "videoio(MSMF): Wrong stream read. Abort capturing"); close(); @@ -1078,12 +1381,45 @@ bool CvCapture_MSMF::grabFrame() } else if (flags & MF_SOURCE_READERF_ENDOFSTREAM) { - sampleTime += frameStep; - CV_LOG_DEBUG(NULL, "videoio(MSMF): End of stream detected"); + vEOS = true; + lastFrame = true; + stopFlag = true; + if (audioStream == -1) + returnFlag = false; + else if (usedVideoSample) + returnFlag = true; + CV_LOG_DEBUG(NULL, "videoio(MSMF): End of video stream detected"); } else { - sampleTime += frameStep; + CV_LOG_DEBUG(NULL, "videoio(MSMF): got video frame with timestamp=" << impendingVideoSampleTime); + if (audioStream != -1) + { + if (!usedVideoSample) + { + usedVideoSample.swap(impendingVideoSample); + std::swap(usedVideoSampleTime, impendingVideoSampleTime); + videoStartOffset = usedVideoSampleTime; + } + else + { + stopFlag = true; + } + if (impendingVideoSample) + { + nFrame++; + videoSampleDuration = impendingVideoSampleTime - usedVideoSampleTime; + requiredAudioTime = impendingVideoSampleTime - givenAudioTime; + givenAudioTime += requiredAudioTime; + } + } + else + { + usedVideoSample.swap(impendingVideoSample); + std::swap(usedVideoSampleTime, impendingVideoSampleTime); + stopFlag = true; + nFrame++; + } if (flags & MF_SOURCE_READERF_NEWSTREAM) { CV_LOG_DEBUG(NULL, "videoio(MSMF): New stream detected"); @@ -1096,33 +1432,285 @@ bool CvCapture_MSMF::grabFrame() { CV_LOG_DEBUG(NULL, "videoio(MSMF): Stream current media type changed"); } - return true; + returnFlag = true; } } } + return returnFlag; +} + +bool CvCapture_MSMF::grabAudioFrame() +{ + DWORD streamIndex, flags; + HRESULT hr; + _ComPtr audioSample = NULL; + audioSamples.clear(); + + bool returnFlag = false; + audioTime = 0; + int numberOfSamples = -1; + if (bufferedAudioDuration*1e7 > requiredAudioTime) + return true; + while ((!vEOS) ? audioTime <= requiredAudioTime : !aEOS) + { + if (audioStartOffset - usedVideoSampleTime > videoSampleDuration) + return true; + for (;;) + { + CV_TRACE_REGION("ReadSample"); + if (!SUCCEEDED(hr = videoFileSource->ReadSample( + dwAudioStreamIndex, // Stream index. + 0, // Flags. + &streamIndex, // Receives the actual stream index. + &flags, // Receives status flags. + &audioSampleTime, // Receives the time stamp. + &audioSample // Receives the sample or NULL. + ))) + break; + if (streamIndex != dwAudioStreamIndex) + break; + if (flags & (MF_SOURCE_READERF_ERROR | MF_SOURCE_READERF_ALLEFFECTSREMOVED | MF_SOURCE_READERF_ENDOFSTREAM)) + break; + if (audioSample) + break; + if (flags & MF_SOURCE_READERF_STREAMTICK) + { + CV_LOG_DEBUG(NULL, "videoio(MSMF): Stream tick detected. Retrying to grab the frame"); + } + } + if (SUCCEEDED(hr)) + { + if (streamIndex != dwAudioStreamIndex) + { + CV_LOG_DEBUG(NULL, "videoio(MSMF): Wrong stream read. Abort capturing"); + close(); + } + else if (flags & MF_SOURCE_READERF_ERROR) + { + CV_LOG_DEBUG(NULL, "videoio(MSMF): Stream reading error. Abort capturing"); + close(); + } + else if (flags & MF_SOURCE_READERF_ALLEFFECTSREMOVED) + { + CV_LOG_DEBUG(NULL, "videoio(MSMF): Stream decoding error. Abort capturing"); + close(); + } + else if (flags & MF_SOURCE_READERF_ENDOFSTREAM) + { + aEOS = true; + if (videoStream != -1 && !vEOS) + returnFlag = true; + CV_LOG_DEBUG(NULL, "videoio(MSMF): End of audio stream detected"); + break; + } + else + { + audioSamples.push_back(audioSample); + audioSample = NULL; + numberOfSamples++; + audioSamples[numberOfSamples]->GetSampleDuration(&audioSampleDuration); + CV_LOG_DEBUG(NULL, "videoio(MSMF): got audio frame with timestamp=" << audioSampleTime << " duration=" << audioSampleDuration); + audioTime += (LONGLONG)(audioSampleDuration + bufferedAudioDuration*1e7); + if (nFrame == 1 && audioStartOffset == -1) + { + audioStartOffset = audioSampleTime - audioSampleDuration; + requiredAudioTime -= audioStartOffset; + } + if (flags & MF_SOURCE_READERF_NEWSTREAM) + { + CV_LOG_DEBUG(NULL, "videoio(MSMF): New stream detected"); + } + if (flags & MF_SOURCE_READERF_NATIVEMEDIATYPECHANGED) + { + CV_LOG_DEBUG(NULL, "videoio(MSMF): Stream native media type changed"); + } + if (flags & MF_SOURCE_READERF_CURRENTMEDIATYPECHANGED) + { + CV_LOG_DEBUG(NULL, "videoio(MSMF): Stream current media type changed"); + } + returnFlag = true; + } + } + else + { + CV_LOG_DEBUG(NULL, "videoio(MSMF): ReadSample() method is not succeeded"); + return false; + } + } + + if (!audioSamples.empty() || !bufferAudioData.empty() && aEOS) + { + _ComPtr buf = NULL; + std::vector audioDataInUse; + BYTE* ptr = NULL; + DWORD maxsize = 0, cursize = 0; + CV_TRACE_REGION("get_contiguous_buffer"); + for (auto item : audioSamples) + { + if (!SUCCEEDED(item->ConvertToContiguousBuffer(&buf))) + { + CV_TRACE_REGION("get_buffer"); + DWORD bcnt = 0; + if (!SUCCEEDED(item->GetBufferCount(&bcnt))) + break; + if (bcnt == 0) + break; + if (!SUCCEEDED(item->GetBufferByIndex(0, &buf))) + break; + } + if (!SUCCEEDED(buf->Lock(&ptr, &maxsize, &cursize))) + break; + size_t lastSize = bufferAudioData.size(); + bufferAudioData.resize(lastSize+cursize); + for (unsigned int i = 0; i < cursize; i++) + { + bufferAudioData[lastSize+i]=*(ptr+i); + } + CV_TRACE_REGION_NEXT("unlock"); + buf->Unlock(); + buf = NULL; + } + audioSamples.clear(); + + audioSamplePos += chunkLengthOfBytes/((captureAudioFormat.bit_per_sample/8)*captureAudioFormat.nChannels); + chunkLengthOfBytes = (videoStream != -1) ? (LONGLONG)((requiredAudioTime*captureAudioFormat.nSamplesPerSec*captureAudioFormat.nChannels*(captureAudioFormat.bit_per_sample)/8)/1e7) : cursize; + if ((videoStream != -1) && (chunkLengthOfBytes % ((int)(captureAudioFormat.bit_per_sample)/8* (int)captureAudioFormat.nChannels) != 0)) + { + if ( (double)audioSamplePos/captureAudioFormat.nSamplesPerSec + audioStartOffset * 1e-7 - usedVideoSampleTime * 1e-7 >= 0 ) + chunkLengthOfBytes -= numberOfAdditionalAudioBytes; + numberOfAdditionalAudioBytes = ((int)(captureAudioFormat.bit_per_sample)/8* (int)captureAudioFormat.nChannels) + - chunkLengthOfBytes % ((int)(captureAudioFormat.bit_per_sample)/8* (int)captureAudioFormat.nChannels); + chunkLengthOfBytes += numberOfAdditionalAudioBytes; + } + if (lastFrame && !syncLastFrame|| aEOS && !vEOS) + { + chunkLengthOfBytes = bufferAudioData.size(); + } + CV_Check((double)chunkLengthOfBytes, chunkLengthOfBytes >= INT_MIN || chunkLengthOfBytes <= INT_MAX, "MSMF: The chunkLengthOfBytes is out of the allowed range"); + copy(bufferAudioData.begin(), bufferAudioData.begin() + (int)chunkLengthOfBytes, std::back_inserter(audioDataInUse)); + bufferAudioData.erase(bufferAudioData.begin(), bufferAudioData.begin() + (int)chunkLengthOfBytes); + if (audioFrame.empty()) + { + switch (outputAudioFormat) + { + case CV_8S: + cv::Mat((int)chunkLengthOfBytes/(captureAudioFormat.nChannels), captureAudioFormat.nChannels, CV_8S, audioDataInUse.data()).copyTo(audioFrame); + break; + case CV_16S: + cv::Mat((int)chunkLengthOfBytes/(2*captureAudioFormat.nChannels), captureAudioFormat.nChannels, CV_16S, audioDataInUse.data()).copyTo(audioFrame); + break; + case CV_32S: + cv::Mat((int)chunkLengthOfBytes/(4*captureAudioFormat.nChannels), captureAudioFormat.nChannels, CV_32S, audioDataInUse.data()).copyTo(audioFrame); + break; + case CV_32F: + cv::Mat((int)chunkLengthOfBytes/(4*captureAudioFormat.nChannels), captureAudioFormat.nChannels, CV_32F, audioDataInUse.data()).copyTo(audioFrame); + break; + default: + break; + } + } + audioDataInUse.clear(); + audioDataInUse.shrink_to_fit(); + } + + return returnFlag; +} + +bool CvCapture_MSMF::grabFrame() +{ + CV_TRACE_FUNCTION(); + + if (grabIsDone) + { + grabIsDone = false; + return true; + } + + audioFrame = Mat(); + if (readCallback) // async "live" capture mode + { + audioSamples.push_back(NULL); + HRESULT hr = 0; + SourceReaderCB* reader = ((SourceReaderCB*)readCallback.Get()); + DWORD dwStreamIndex = 0; + if (videoStream != -1) + dwStreamIndex = dwVideoStreamIndex; + if (audioStream != -1) + dwStreamIndex = dwAudioStreamIndex; + if (!reader->m_reader) + { + // Initiate capturing with async callback + reader->m_reader = videoFileSource.Get(); + reader->m_dwStreamIndex = dwStreamIndex; + if (FAILED(hr = videoFileSource->ReadSample(dwStreamIndex, 0, NULL, NULL, NULL, NULL))) + { + CV_LOG_ERROR(NULL, "videoio(MSMF): can't grab frame - initial async ReadSample() call failed: " << hr); + reader->m_reader = NULL; + return false; + } + } + BOOL bEOS = false; + if (FAILED(hr = reader->Wait( videoStream == -1 ? INFINITE : 10000, (videoStream != -1) ? usedVideoSample : audioSamples[0], bEOS))) // 10 sec + { + CV_LOG_WARNING(NULL, "videoio(MSMF): can't grab frame. Error: " << hr); + return false; + } + if (bEOS) + { + CV_LOG_WARNING(NULL, "videoio(MSMF): EOS signal. Capture stream is lost"); + return false; + } + if (videoStream != -1) + usedVideoSampleTime = reader->m_lastSampleTimestamp; + return true; + } + else if (isOpen) + { + if (vEOS) + return false; + + bool returnFlag = true; + + if (videoStream != -1) + { + if (!vEOS) + returnFlag &= grabVideoFrame(); + if (!returnFlag) + return false; + } + + if (audioStream != -1) + { + bufferedAudioDuration = (double)(bufferAudioData.size()/((captureAudioFormat.bit_per_sample/8)*captureAudioFormat.nChannels))/captureAudioFormat.nSamplesPerSec; + audioFrame.release(); + if (!aEOS) + returnFlag &= grabAudioFrame(); + } + + return returnFlag; + } return false; } -bool CvCapture_MSMF::retrieveFrame(int, cv::OutputArray frame) +bool CvCapture_MSMF::retrieveVideoFrame(cv::OutputArray frame) { CV_TRACE_FUNCTION(); do { - if (!videoSample) + if (!usedVideoSample) break; _ComPtr buf = NULL; - CV_TRACE_REGION("get_contiguous_buffer"); - if (!SUCCEEDED(videoSample->ConvertToContiguousBuffer(&buf))) + if (!SUCCEEDED(usedVideoSample->ConvertToContiguousBuffer(&buf))) { CV_TRACE_REGION("get_buffer"); DWORD bcnt = 0; - if (!SUCCEEDED(videoSample->GetBufferCount(&bcnt))) + if (!SUCCEEDED(usedVideoSample->GetBufferCount(&bcnt))) break; if (bcnt == 0) break; - if (!SUCCEEDED(videoSample->GetBufferByIndex(0, &buf))) + if (!SUCCEEDED(usedVideoSample->GetBufferByIndex(0, &buf))) break; } @@ -1158,27 +1746,27 @@ bool CvCapture_MSMF::retrieveFrame(int, cv::OutputArray frame) break; if (convertFormat) { - if (lock2d || (unsigned int)cursize == captureFormat.sampleSize) + if (lock2d || (unsigned int)cursize == captureVideoFormat.sampleSize) { - switch (outputFormat) + switch (outputVideoFormat) { case CV_CAP_MODE_YUYV: - cv::Mat(captureFormat.height, captureFormat.width, CV_8UC2, ptr, pitch).copyTo(frame); + cv::Mat(captureVideoFormat.height, captureVideoFormat.width, CV_8UC2, ptr, pitch).copyTo(frame); break; case CV_CAP_MODE_BGR: if (captureMode == MODE_HW) - cv::cvtColor(cv::Mat(captureFormat.height, captureFormat.width, CV_8UC4, ptr, pitch), frame, cv::COLOR_BGRA2BGR); + cv::cvtColor(cv::Mat(captureVideoFormat.height, captureVideoFormat.width, CV_8UC4, ptr, pitch), frame, cv::COLOR_BGRA2BGR); else - cv::Mat(captureFormat.height, captureFormat.width, CV_8UC3, ptr, pitch).copyTo(frame); + cv::Mat(captureVideoFormat.height, captureVideoFormat.width, CV_8UC3, ptr, pitch).copyTo(frame); break; case CV_CAP_MODE_RGB: if (captureMode == MODE_HW) - cv::cvtColor(cv::Mat(captureFormat.height, captureFormat.width, CV_8UC4, ptr, pitch), frame, cv::COLOR_BGRA2BGR); + cv::cvtColor(cv::Mat(captureVideoFormat.height, captureVideoFormat.width, CV_8UC4, ptr, pitch), frame, cv::COLOR_BGRA2BGR); else - cv::cvtColor(cv::Mat(captureFormat.height, captureFormat.width, CV_8UC3, ptr, pitch), frame, cv::COLOR_BGR2RGB); + cv::cvtColor(cv::Mat(captureVideoFormat.height, captureVideoFormat.width, CV_8UC3, ptr, pitch), frame, cv::COLOR_BGR2RGB); break; case CV_CAP_MODE_GRAY: - cv::Mat(captureFormat.height, captureFormat.width, CV_8UC1, ptr, pitch).copyTo(frame); + cv::Mat(captureVideoFormat.height, captureVideoFormat.width, CV_8UC1, ptr, pitch).copyTo(frame); break; default: frame.release(); @@ -1204,30 +1792,142 @@ bool CvCapture_MSMF::retrieveFrame(int, cv::OutputArray frame) return false; } +bool CvCapture_MSMF::retrieveAudioFrame(int index, cv::OutputArray frame) +{ + CV_TRACE_FUNCTION(); + if (audioStartOffset - usedVideoSampleTime > videoSampleDuration) + { + frame.release(); + return true; + } + do + { + if (audioFrame.empty()) + { + frame.release(); + if (aEOS) + return true; + } + cv::Mat data; + switch (outputAudioFormat) + { + case CV_8S: + data = cv::Mat(1, audioFrame.rows, CV_8S); + for (int i = 0; i < audioFrame.rows; i++) + data.at(0,i) = audioFrame.at(i,index-audioBaseIndex); + break; + case CV_16S: + data = cv::Mat(1, audioFrame.rows, CV_16S); + for (int i = 0; i < audioFrame.rows; i++) + data.at(0,i) = audioFrame.at(i,index-audioBaseIndex); + break; + case CV_32S: + data = cv::Mat(1, audioFrame.rows, CV_32S); + for (int i = 0; i < audioFrame.rows; i++) + data.at(0,i) = audioFrame.at(i,index-audioBaseIndex); + break; + case CV_32F: + data = cv::Mat(1, audioFrame.rows, CV_32F); + for (int i = 0; i < audioFrame.rows; i++) + data.at(0,i) = audioFrame.at(i,index-audioBaseIndex); + break; + default: + frame.release(); + break; + } + if (!data.empty()) + data.copyTo(frame); + + return !frame.empty(); + } while (0); + + return false; +} + +bool CvCapture_MSMF::retrieveFrame(int index, cv::OutputArray frame) +{ + CV_TRACE_FUNCTION(); + if (index < 0) + return false; + if ((unsigned int)index < audioBaseIndex) + { + if (videoStream == -1) + { + frame.release(); + return false; + } + else + return retrieveVideoFrame(frame); + } + else + { + if (audioStream == -1) + { + frame.release(); + return false; + } + else + return retrieveAudioFrame(index, frame); + } +} + bool CvCapture_MSMF::setTime(double time, bool rough) { + if (videoStream == -1) + return false; + if (videoStream != -1 && audioStream != -1) + if (time != 0) + return false; PROPVARIANT var; if (SUCCEEDED(videoFileSource->GetPresentationAttribute((DWORD)MF_SOURCE_READER_MEDIASOURCE, MF_SOURCE_READER_MEDIASOURCE_CHARACTERISTICS, &var)) && var.vt == VT_UI4 && var.ulVal & MFMEDIASOURCE_CAN_SEEK) { - videoSample.Release(); + usedVideoSample.Release(); bool useGrabbing = time > 0 && !rough && !(var.ulVal & MFMEDIASOURCE_HAS_SLOW_SEEK); PropVariantClear(&var); - sampleTime = (useGrabbing && time >= frameStep) ? (LONGLONG)floor(time + 0.5) - frameStep : (LONGLONG)floor(time + 0.5); + usedVideoSampleTime = (useGrabbing) ? 0 : (LONGLONG)floor(time + 0.5); + nFrame = (useGrabbing) ? 0 : usedVideoSampleTime/frameStep; + givenAudioTime = (useGrabbing) ? 0 : nFrame*frameStep; var.vt = VT_I8; - var.hVal.QuadPart = sampleTime; + var.hVal.QuadPart = usedVideoSampleTime; bool resOK = SUCCEEDED(videoFileSource->SetCurrentPosition(GUID_NULL, var)); PropVariantClear(&var); if (resOK && useGrabbing) { LONGLONG timeborder = (LONGLONG)floor(time + 0.5) - frameStep / 2; - do { resOK = grabFrame(); videoSample.Release(); } while (resOK && sampleTime < timeborder); + do { resOK = grabFrame(); usedVideoSample.Release(); } while (resOK && usedVideoSampleTime < timeborder); } return resOK; } return false; } +bool CvCapture_MSMF::setTime(int numberFrame) +{ + if (videoStream == -1) + return false; + if (videoStream != -1 && audioStream != -1) + if (numberFrame != 0) + return false; + PROPVARIANT var; + if (SUCCEEDED(videoFileSource->GetPresentationAttribute((DWORD)MF_SOURCE_READER_MEDIASOURCE, MF_SOURCE_READER_MEDIASOURCE_CHARACTERISTICS, &var)) && + var.vt == VT_UI4 && var.ulVal & MFMEDIASOURCE_CAN_SEEK) + { + usedVideoSample.Release(); + PropVariantClear(&var); + usedVideoSampleTime = 0; + nFrame = 0; + givenAudioTime = 0; + var.vt = VT_I8; + var.hVal.QuadPart = usedVideoSampleTime; + bool resOK = SUCCEEDED(videoFileSource->SetCurrentPosition(GUID_NULL, var)); + PropVariantClear(&var); + while (resOK && nFrame < numberFrame) { resOK = grabFrame(); usedVideoSample.Release(); }; + return resOK; + } + return false; +} + template bool CvCapture_MSMF::readComplexPropery(long prop, long & val) const { @@ -1269,29 +1969,31 @@ double CvCapture_MSMF::getProperty( int property_id ) const case CV_CAP_PROP_CONVERT_RGB: return convertFormat ? 1 : 0; case CV_CAP_PROP_SAR_NUM: - return captureFormat.aspectRatioNum; + return captureVideoFormat.aspectRatioNum; case CV_CAP_PROP_SAR_DEN: - return captureFormat.aspectRatioDenom; + return captureVideoFormat.aspectRatioDenom; case CV_CAP_PROP_FRAME_WIDTH: - return captureFormat.width; + return captureVideoFormat.width; case CV_CAP_PROP_FRAME_HEIGHT: - return captureFormat.height; + return captureVideoFormat.height; case CV_CAP_PROP_FOURCC: - return captureFormat.subType.Data1; + return captureVideoFormat.subType.Data1; case CV_CAP_PROP_FPS: - return captureFormat.getFramerate(); + return captureVideoFormat.getFramerate(); case CV_CAP_PROP_FRAME_COUNT: if (duration != 0) - return floor(((double)duration / 1e7)* captureFormat.getFramerate() + 0.5); + return floor(((double)duration / 1e7)* captureVideoFormat.getFramerate() + 0.5); else break; case CV_CAP_PROP_POS_FRAMES: - return floor(((double)sampleTime / 1e7)* captureFormat.getFramerate() + 0.5); + return (double)nFrame; case CV_CAP_PROP_POS_MSEC: - return (double)sampleTime / 1e4; + return (double)usedVideoSampleTime / 1e4; + case CAP_PROP_AUDIO_POS: + return (double)audioSamplePos; case CV_CAP_PROP_POS_AVI_RATIO: if (duration != 0) - return (double)sampleTime / duration; + return (double)usedVideoSampleTime / duration; else break; case CV_CAP_PROP_BRIGHTNESS: @@ -1383,6 +2085,18 @@ double CvCapture_MSMF::getProperty( int property_id ) const case CV_CAP_PROP_ISO_SPEED: case CV_CAP_PROP_SETTINGS: case CV_CAP_PROP_BUFFERSIZE: + case CAP_PROP_AUDIO_BASE_INDEX: + return audioBaseIndex; + case CAP_PROP_AUDIO_TOTAL_STREAMS: + return numberOfAudioStreams; + case CAP_PROP_AUDIO_TOTAL_CHANNELS: + return captureAudioFormat.nChannels; + case CAP_PROP_AUDIO_SAMPLES_PER_SECOND: + return captureAudioFormat.nSamplesPerSec; + case CAP_PROP_AUDIO_DATA_DEPTH: + return outputAudioFormat; + case CAP_PROP_AUDIO_SHIFT_NSEC: + return (double)(audioStartOffset - videoStartOffset)*1e2; default: break; } @@ -1408,7 +2122,7 @@ bool CvCapture_MSMF::writeComplexProperty(long prop, double val, long flags) bool CvCapture_MSMF::setProperty( int property_id, double value ) { - MediaType newFormat = captureFormat; + MediaType newFormat = captureVideoFormat; if (isOpen) switch (property_id) { @@ -1423,45 +2137,45 @@ bool CvCapture_MSMF::setProperty( int property_id, double value ) return false; } case CV_CAP_PROP_FOURCC: - return configureOutput(newFormat, (int)cvRound(value)); + return configureVideoOutput(newFormat, (int)cvRound(value)); case CV_CAP_PROP_FORMAT: - return configureOutput(newFormat, (int)cvRound(value)); + return configureVideoOutput(newFormat, (int)cvRound(value)); case CV_CAP_PROP_CONVERT_RGB: convertFormat = (value != 0); - return configureOutput(newFormat, outputFormat); + return configureVideoOutput(newFormat, outputVideoFormat); case CV_CAP_PROP_SAR_NUM: if (value > 0) { newFormat.aspectRatioNum = (UINT32)cvRound(value); - return configureOutput(newFormat, outputFormat); + return configureVideoOutput(newFormat, outputVideoFormat); } break; case CV_CAP_PROP_SAR_DEN: if (value > 0) { newFormat.aspectRatioDenom = (UINT32)cvRound(value); - return configureOutput(newFormat, outputFormat); + return configureVideoOutput(newFormat, outputVideoFormat); } break; case CV_CAP_PROP_FRAME_WIDTH: if (value >= 0) { newFormat.width = (UINT32)cvRound(value); - return configureOutput(newFormat, outputFormat); + return configureVideoOutput(newFormat, outputVideoFormat); } break; case CV_CAP_PROP_FRAME_HEIGHT: if (value >= 0) { newFormat.height = (UINT32)cvRound(value); - return configureOutput(newFormat, outputFormat); + return configureVideoOutput(newFormat, outputVideoFormat); } break; case CV_CAP_PROP_FPS: if (value >= 0) { newFormat.setFramerate(value); - return configureOutput(newFormat, outputFormat); + return configureVideoOutput(newFormat, outputVideoFormat); } break; case CV_CAP_PROP_FRAME_COUNT: @@ -1471,8 +2185,8 @@ bool CvCapture_MSMF::setProperty( int property_id, double value ) return setTime(duration * value, true); break; case CV_CAP_PROP_POS_FRAMES: - if (std::fabs(captureFormat.getFramerate()) > 0) - return setTime(value * 1e7 / captureFormat.getFramerate(), false); + if (std::fabs(captureVideoFormat.getFramerate()) > 0) + return setTime((int)value); break; case CV_CAP_PROP_POS_MSEC: return setTime(value * 1e4, false); diff --git a/modules/videoio/test/test_audio.cpp b/modules/videoio/test/test_audio.cpp new file mode 100644 index 0000000000..3ff51e2613 --- /dev/null +++ b/modules/videoio/test/test_audio.cpp @@ -0,0 +1,273 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "test_precomp.hpp" + +namespace opencv_test { namespace { + +//file name, number of audio channels, epsilon, video type, weight, height, number of frame, number of audio samples, fps, psnr Threshold, backend +typedef std::tuple paramCombination; +//file name, number of audio channels, number of audio samples, epsilon, backend +typedef std::tuple param; + +class AudioBaseTest +{ +protected: + AudioBaseTest(){}; + void getValidAudioData() + { + const double step = 3.14/22050; + double value = 0; + validAudioData.resize(expectedNumAudioCh); + for (int nCh = 0; nCh < expectedNumAudioCh; nCh++) + { + value = 0; + for(unsigned int i = 0; i < numberOfSamples; i++) + { + if (i != 0 && i % 44100 == 0) + value = 0; + validAudioData[nCh].push_back(sin(value)); + value += step; + } + } + } + void checkAudio() + { + getValidAudioData(); + + ASSERT_EQ(expectedNumAudioCh, (int)audioData.size()); + for (unsigned int nCh = 0; nCh < audioData.size(); nCh++) + { + ASSERT_EQ(numberOfSamples, audioData[nCh].size()) << "nCh=" << nCh; + for (unsigned int i = 0; i < numberOfSamples; i++) + { + EXPECT_NEAR(validAudioData[nCh][i], audioData[nCh][i], epsilon) << "sample index=" << i << " nCh=" << nCh; + } + } + } +protected: + int expectedNumAudioCh; + unsigned int numberOfSamples; + double epsilon; + VideoCaptureAPIs backend; + std::string root; + std::string fileName; + + std::vector> validAudioData; + std::vector> audioData; + std::vector params; + + Mat audioFrame; + VideoCapture cap; +}; + +class AudioTestFixture : public AudioBaseTest, public testing::TestWithParam +{ +public: + AudioTestFixture() + { + fileName = get<0>(GetParam()); + expectedNumAudioCh = get<1>(GetParam()); + numberOfSamples = get<2>(GetParam()); + epsilon = get<3>(GetParam()); + backend = get<4>(GetParam()); + root = "audio/"; + params = { CAP_PROP_AUDIO_STREAM, 0, + CAP_PROP_VIDEO_STREAM, -1, + CAP_PROP_AUDIO_DATA_DEPTH, CV_16S }; + } + + void doTest() + { + ASSERT_TRUE(cap.open(findDataFile(root + fileName), backend, params)); + const int audioBaseIndex = static_cast(cap.get(cv::CAP_PROP_AUDIO_BASE_INDEX)); + const int numberOfChannels = (int)cap.get(CAP_PROP_AUDIO_TOTAL_CHANNELS); + ASSERT_EQ(expectedNumAudioCh, numberOfChannels); + double f = 0; + audioData.resize(numberOfChannels); + for (;;) + { + if (cap.grab()) + { + for (int nCh = 0; nCh < numberOfChannels; nCh++) + { + ASSERT_TRUE(cap.retrieve(audioFrame, audioBaseIndex)); + ASSERT_EQ(CV_16SC1, audioFrame.type()) << audioData[nCh].size(); + for (int i = 0; i < audioFrame.cols; i++) + { + f = ((double) audioFrame.at(0,i)) / (double) 32768; + audioData[nCh].push_back(f); + } + } + } + else { break; } + } + ASSERT_FALSE(audioData.empty()); + + checkAudio(); + } +}; + +const param audioParams[] = +{ + param("test_audio.wav", 1, 132300, 0.0001, cv::CAP_MSMF), + param("test_mono_audio.mp3", 1, 133104, 0.12, cv::CAP_MSMF), + param("test_stereo_audio.mp3", 2, 133104, 0.12, cv::CAP_MSMF), + param("test_audio.mp4", 1, 133104, 0.15, cv::CAP_MSMF) +}; + +class Audio : public AudioTestFixture{}; + +TEST_P(Audio, audio) +{ + if (!videoio_registry::hasBackend(cv::VideoCaptureAPIs(backend))) + throw SkipTestException(cv::videoio_registry::getBackendName(backend) + " backend was not found"); + + doTest(); +} + +INSTANTIATE_TEST_CASE_P(/**/, Audio, testing::ValuesIn(audioParams)); + +class MediaTestFixture : public AudioBaseTest, public testing::TestWithParam +{ +public: + MediaTestFixture(): + videoType(get<3>(GetParam())), + height(get<4>(GetParam())), + width(get<5>(GetParam())), + numberOfFrames(get<6>(GetParam())), + fps(get<8>(GetParam())), + psnrThreshold(get<9>(GetParam())) + { + fileName = get<0>(GetParam()); + expectedNumAudioCh = get<1>(GetParam()); + numberOfSamples = get<7>(GetParam()); + epsilon = get<2>(GetParam()); + backend = get<10>(GetParam()); + root = "audio/"; + params = { CAP_PROP_AUDIO_STREAM, 0, + CAP_PROP_VIDEO_STREAM, 0, + CAP_PROP_AUDIO_DATA_DEPTH, CV_16S }; + }; + + void doTest() + { + ASSERT_TRUE(cap.open(findDataFile(root + fileName), backend, params)); + + const int audioBaseIndex = static_cast(cap.get(cv::CAP_PROP_AUDIO_BASE_INDEX)); + const int numberOfChannels = (int)cap.get(CAP_PROP_AUDIO_TOTAL_CHANNELS); + ASSERT_EQ(expectedNumAudioCh, numberOfChannels); + + const int samplePerSecond = (int)cap.get(CAP_PROP_AUDIO_SAMPLES_PER_SECOND); + ASSERT_EQ(44100, samplePerSecond); + int samplesPerFrame = (int)(1./fps*samplePerSecond); + int audioSamplesTolerance = samplesPerFrame / 2; + + double audio0_timestamp = 0; + + Mat videoFrame; + Mat img(height, width, videoType); + audioData.resize(numberOfChannels); + for (int frame = 0; frame < numberOfFrames; frame++) + { + SCOPED_TRACE(cv::format("frame=%d", frame)); + + ASSERT_TRUE(cap.grab()); + + if (frame == 0) + { + double audio_shift = cap.get(CAP_PROP_AUDIO_SHIFT_NSEC); + double video0_timestamp = cap.get(CAP_PROP_POS_MSEC) * 1e-3; + audio0_timestamp = video0_timestamp + audio_shift * 1e-9; + std::cout << "video0 timestamp: " << video0_timestamp << " audio0 timestamp: " << audio0_timestamp << " (audio shift nanoseconds: " << audio_shift << " , seconds: " << audio_shift * 1e-9 << ")" << std::endl; + } + + ASSERT_TRUE(cap.retrieve(videoFrame)); + if (epsilon >= 0) + { + generateFrame(frame, numberOfFrames, img); + ASSERT_EQ(img.size, videoFrame.size); + double psnr = cvtest::PSNR(img, videoFrame); + EXPECT_GE(psnr, psnrThreshold); + } + + int audioFrameCols = 0; + for (int nCh = 0; nCh < numberOfChannels; nCh++) + { + ASSERT_TRUE(cap.retrieve(audioFrame, audioBaseIndex+nCh)); + if (audioFrame.empty()) + continue; + ASSERT_EQ(CV_16SC1, audioFrame.type()); + if (nCh == 0) + audioFrameCols = audioFrame.cols; + else + ASSERT_EQ(audioFrameCols, audioFrame.cols) << "channel "<< nCh; + for (int i = 0; i < audioFrame.cols; i++) + { + double f = audioFrame.at(0,i) / 32768.0; + audioData[nCh].push_back(f); + } + } + + if (frame < 5 || frame >= numberOfFrames-5) + std::cout << "frame=" << frame << ": audioFrameSize=" << audioFrameCols << " videoTimestamp=" << cap.get(CAP_PROP_POS_MSEC) << " ms" << std::endl; + else if (frame == 6) + std::cout << "frame..." << std::endl; + + if (audioFrameCols == 0) + continue; + if (frame != 0 && frame != numberOfFrames-1) + { + // validate audio position + EXPECT_NEAR( + cap.get(CAP_PROP_AUDIO_POS) / samplePerSecond + audio0_timestamp, + cap.get(CAP_PROP_POS_MSEC) * 1e-3, + (1.0 / fps) * 0.3) + << "CAP_PROP_AUDIO_POS=" << cap.get(CAP_PROP_AUDIO_POS) << " CAP_PROP_POS_MSEC=" << cap.get(CAP_PROP_POS_MSEC); + } + if (frame != 0 && frame != numberOfFrames-1 && audioData[0].size() != (size_t)numberOfSamples) + { + // validate audio frame size + EXPECT_NEAR(audioFrame.cols, samplesPerFrame, audioSamplesTolerance); + } + } + ASSERT_FALSE(cap.grab()); + ASSERT_FALSE(audioData.empty()); + + std::cout << "Total audio samples=" << audioData[0].size() << std::endl; + + if (epsilon >= 0) + checkAudio(); + } +protected: + const int videoType; + const int height; + const int width; + const int numberOfFrames; + const int fps; + const double psnrThreshold; +}; + +const paramCombination mediaParams[] = +{ + paramCombination("test_audio.mp4", 1, 0.15, CV_8UC3, 240, 320, 90, 131819, 30, 30., cv::CAP_MSMF) +#if 0 + // https://filesamples.com/samples/video/mp4/sample_960x400_ocean_with_audio.mp4 + , paramCombination("sample_960x400_ocean_with_audio.mp4", 2, -1/*eplsilon*/, CV_8UC3, 400, 960, 1116, 2056588, 30, 30., cv::CAP_MSMF) +#endif +}; + +class Media : public MediaTestFixture{}; + +TEST_P(Media, audio) +{ + if (!videoio_registry::hasBackend(cv::VideoCaptureAPIs(backend))) + throw SkipTestException(cv::videoio_registry::getBackendName(backend) + " backend was not found"); + + doTest(); +} + +INSTANTIATE_TEST_CASE_P(/**/, Media, testing::ValuesIn(mediaParams)); + +}} //namespace diff --git a/modules/videoio/test/test_microphone.cpp b/modules/videoio/test/test_microphone.cpp new file mode 100644 index 0000000000..c82a7c4eda --- /dev/null +++ b/modules/videoio/test/test_microphone.cpp @@ -0,0 +1,41 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// Usage: opencv_test_videoio --gtest_also_run_disabled_tests + +#include "test_precomp.hpp" + +namespace opencv_test { namespace { + +TEST(DISABLED_videoio_micro, basic) +{ + int cursize = 0; + int validSize = 0; + Mat frame; + + std::vector params { CAP_PROP_AUDIO_STREAM, 0, CAP_PROP_VIDEO_STREAM, -1 }; + VideoCapture cap(0, cv::CAP_MSMF, params); + ASSERT_TRUE(cap.isOpened()); + + int samplesPerSecond = (int)cap.get(cv::CAP_PROP_AUDIO_SAMPLES_PER_SECOND); + const int audio_base_index = (int)cap.get(cv::CAP_PROP_AUDIO_BASE_INDEX); + + const double cvTickFreq = cv::getTickFrequency(); + int64 sysTimePrev = cv::getTickCount(); + int64 sysTimeCurr = cv::getTickCount(); + + cout << "Audio would be captured for the next 10 seconds" << endl; + while ((sysTimeCurr-sysTimePrev)/cvTickFreq < 10) + { + if (cap.grab()) + { + ASSERT_TRUE(cap.retrieve(frame, audio_base_index)); + sysTimeCurr = cv::getTickCount(); + } + } + validSize = samplesPerSecond*(int)((sysTimeCurr-sysTimePrev)/cvTickFreq); + cursize = (int)cap.get(cv::CAP_PROP_AUDIO_POS); + ASSERT_LT(validSize - cursize, cursize*0.05); +} + +}} // namespace diff --git a/samples/cpp/videocapture_audio.cpp b/samples/cpp/videocapture_audio.cpp new file mode 100644 index 0000000000..c9f1ec94ce --- /dev/null +++ b/samples/cpp/videocapture_audio.cpp @@ -0,0 +1,59 @@ +#include +#include +#include +#include + +using namespace cv; +using namespace std; + +int main(int argc, char** argv) +{ + CommandLineParser parser(argc, argv, "{@audio||}"); + string file = parser.get("@audio"); + + if (file.empty()) + { + return 1; + } + + Mat frame; + vector> audioData; + VideoCapture cap; + vector params { CAP_PROP_AUDIO_STREAM, 0, + CAP_PROP_VIDEO_STREAM, -1, + CAP_PROP_AUDIO_DATA_DEPTH, CV_16S }; + + cap.open(file, CAP_MSMF, params); + if (!cap.isOpened()) + { + cerr << "ERROR! Can't to open file: " + file << endl; + return -1; + } + + const int audioBaseIndex = (int)cap.get(CAP_PROP_AUDIO_BASE_INDEX); + const int numberOfChannels = (int)cap.get(CAP_PROP_AUDIO_TOTAL_CHANNELS); + cout << "CAP_PROP_AUDIO_DATA_DEPTH: " << depthToString((int)cap.get(CAP_PROP_AUDIO_DATA_DEPTH)) << endl; + cout << "CAP_PROP_AUDIO_SAMPLES_PER_SECOND: " << cap.get(CAP_PROP_AUDIO_SAMPLES_PER_SECOND) << endl; + cout << "CAP_PROP_AUDIO_TOTAL_CHANNELS: " << numberOfChannels << endl; + cout << "CAP_PROP_AUDIO_TOTAL_STREAMS: " << cap.get(CAP_PROP_AUDIO_TOTAL_STREAMS) << endl; + + int numberOfSamples = 0; + audioData.resize(numberOfChannels); + for (;;) + { + if (cap.grab()) + { + for (int nCh = 0; nCh < numberOfChannels; nCh++) + { + cap.retrieve(frame, audioBaseIndex+nCh); + audioData[nCh].push_back(frame); + numberOfSamples+=frame.cols; + } + } + else { break; } + } + + cout << "Number of samples: " << numberOfSamples << endl; + + return 0; +} diff --git a/samples/cpp/videocapture_audio_combination.cpp b/samples/cpp/videocapture_audio_combination.cpp new file mode 100644 index 0000000000..7f0deecf16 --- /dev/null +++ b/samples/cpp/videocapture_audio_combination.cpp @@ -0,0 +1,69 @@ +#include +#include +#include +#include + +using namespace cv; +using namespace std; + +int main(int argc, char** argv) +{ + cv::CommandLineParser parser(argc, argv, "{@audio||}"); + string file = parser.get("@audio"); + + if (file.empty()) + { + return 1; + } + + Mat videoFrame; + Mat audioFrame; + vector> audioData; + VideoCapture cap; + vector params { CAP_PROP_AUDIO_STREAM, 0, + CAP_PROP_VIDEO_STREAM, 0, + CAP_PROP_AUDIO_DATA_DEPTH, CV_16S }; + + cap.open(file, CAP_MSMF, params); + if (!cap.isOpened()) + { + cerr << "ERROR! Can't to open file: " + file << endl; + return -1; + } + + const int audioBaseIndex = (int)cap.get(CAP_PROP_AUDIO_BASE_INDEX); + const int numberOfChannels = (int)cap.get(CAP_PROP_AUDIO_TOTAL_CHANNELS); + cout << "CAP_PROP_AUDIO_DATA_DEPTH: " << depthToString((int)cap.get(CAP_PROP_AUDIO_DATA_DEPTH)) << endl; + cout << "CAP_PROP_AUDIO_SAMPLES_PER_SECOND: " << cap.get(CAP_PROP_AUDIO_SAMPLES_PER_SECOND) << endl; + cout << "CAP_PROP_AUDIO_TOTAL_CHANNELS: " << cap.get(CAP_PROP_AUDIO_TOTAL_CHANNELS) << endl; + cout << "CAP_PROP_AUDIO_TOTAL_STREAMS: " << cap.get(CAP_PROP_AUDIO_TOTAL_STREAMS) << endl; + + int numberOfSamples = 0; + int numberOfFrames = 0; + audioData.resize(numberOfChannels); + for (;;) + { + if (cap.grab()) + { + cap.retrieve(videoFrame); + for (int nCh = 0; nCh < numberOfChannels; nCh++) + { + cap.retrieve(audioFrame, audioBaseIndex+nCh); + if (!audioFrame.empty()) + audioData[nCh].push_back(audioFrame); + numberOfSamples+=audioFrame.cols; + } + if (!videoFrame.empty()) + { + numberOfFrames++; + imshow("Live", videoFrame); + if (waitKey(5) >= 0) + break; + } + } else { break; } + } + + cout << "Number of audio samples: " << numberOfSamples << endl + << "Number of video frames: " << numberOfFrames << endl; + return 0; +} diff --git a/samples/cpp/videocapture_microphone.cpp b/samples/cpp/videocapture_microphone.cpp new file mode 100644 index 0000000000..0c69ec929d --- /dev/null +++ b/samples/cpp/videocapture_microphone.cpp @@ -0,0 +1,57 @@ +#include +#include +#include +#include + +using namespace cv; +using namespace std; + +int main(int, char**) +{ + Mat frame; + vector audioData; + VideoCapture cap; + vector params { CAP_PROP_AUDIO_STREAM, 0, + CAP_PROP_VIDEO_STREAM, -1 }; + + cap.open(0, CAP_MSMF, params); + if (!cap.isOpened()) + { + cerr << "ERROR! Can't to open microphone" << endl; + return -1; + } + + const int audioBaseIndex = (int)cap.get(CAP_PROP_AUDIO_BASE_INDEX); + const int numberOfChannels = (int)cap.get(CAP_PROP_AUDIO_TOTAL_CHANNELS); + cout << "CAP_PROP_AUDIO_DATA_DEPTH: " << depthToString((int)cap.get(CAP_PROP_AUDIO_DATA_DEPTH)) << endl; + cout << "CAP_PROP_AUDIO_SAMPLES_PER_SECOND: " << cap.get(CAP_PROP_AUDIO_SAMPLES_PER_SECOND) << endl; + cout << "CAP_PROP_AUDIO_TOTAL_CHANNELS: " << numberOfChannels << endl; + cout << "CAP_PROP_AUDIO_TOTAL_STREAMS: " << cap.get(CAP_PROP_AUDIO_TOTAL_STREAMS) << endl; + + const double cvTickFreq = getTickFrequency(); + int64 sysTimeCurr = getTickCount(); + int64 sysTimePrev = sysTimeCurr; + while ((sysTimeCurr-sysTimePrev)/cvTickFreq < 10) + { + if (cap.grab()) + { + for (int nCh = 0; nCh < numberOfChannels; nCh++) + { + cap.retrieve(frame, audioBaseIndex+nCh); + audioData.push_back(frame); + sysTimeCurr = getTickCount(); + } + } + else + { + cerr << "Grab error" << endl; + break; + } + } + int numberOfSamles = 0; + for (auto item : audioData) + numberOfSamles+=item.cols; + cout << "Number of samples: " << numberOfSamles << endl; + + return 0; +} From ce68291d83674d7cf4410cd640e723fb403bc406 Mon Sep 17 00:00:00 2001 From: Harvey Date: Thu, 21 Oct 2021 16:47:27 +0800 Subject: [PATCH 017/226] 32bit rgb bmp file should not copy data as rgba --- modules/imgcodecs/src/grfmt_bmp.cpp | 2 +- modules/imgcodecs/test/test_grfmt.cpp | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/modules/imgcodecs/src/grfmt_bmp.cpp b/modules/imgcodecs/src/grfmt_bmp.cpp index 6a56d3ab2d..aca9c7b2ba 100644 --- a/modules/imgcodecs/src/grfmt_bmp.cpp +++ b/modules/imgcodecs/src/grfmt_bmp.cpp @@ -180,7 +180,7 @@ bool BmpDecoder::readHeader() throw; } // in 32 bit case alpha channel is used - so require CV_8UC4 type - m_type = iscolor ? (m_bpp == 32 ? CV_8UC4 : CV_8UC3 ) : CV_8UC1; + m_type = iscolor ? ((m_bpp == 32 && m_rle_code != BMP_RGB) ? CV_8UC4 : CV_8UC3 ) : CV_8UC1; m_origin = m_height > 0 ? IPL_ORIGIN_BL : IPL_ORIGIN_TL; m_height = std::abs(m_height); diff --git a/modules/imgcodecs/test/test_grfmt.cpp b/modules/imgcodecs/test/test_grfmt.cpp index fa03ef4333..6866c8d092 100644 --- a/modules/imgcodecs/test/test_grfmt.cpp +++ b/modules/imgcodecs/test/test_grfmt.cpp @@ -280,6 +280,16 @@ TEST(Imgcodecs_Bmp, read_rle8) EXPECT_PRED_FORMAT2(cvtest::MatComparator(0, 0), rle, ord); } +TEST(Imgcodecs_Bmp, read_32bit_rgb) +{ + const string root = cvtest::TS::ptr()->get_data_path(); + const string filenameInput = root + "readwrite/test_32bit_rgb.bmp"; + + const Mat img = cv::imread(filenameInput, IMREAD_UNCHANGED); + ASSERT_FALSE(img.empty()); + ASSERT_EQ(CV_8UC3, img.type()); +} + #ifdef HAVE_IMGCODEC_HDR TEST(Imgcodecs_Hdr, regression) { From bac1c6d12f7dde6617050c6cc6567729299b1425 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 21 Oct 2021 09:36:25 +0000 Subject: [PATCH 018/226] hotfix: repair Clang ABI --- modules/core/include/opencv2/core/types.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/core/include/opencv2/core/types.hpp b/modules/core/include/opencv2/core/types.hpp index 1e9a8b629c..7dfadb2522 100644 --- a/modules/core/include/opencv2/core/types.hpp +++ b/modules/core/include/opencv2/core/types.hpp @@ -162,7 +162,7 @@ public: //! default constructor Point_(); Point_(_Tp _x, _Tp _y); -#if (defined(__GNUC__) && __GNUC__ < 5) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 +#if (defined(__GNUC__) && __GNUC__ < 5) && !defined(__clang__) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 Point_(const Point_& pt); Point_(Point_&& pt) CV_NOEXCEPT = default; #elif OPENCV_ABI_COMPATIBILITY < 500 @@ -172,7 +172,7 @@ public: Point_(const Size_<_Tp>& sz); Point_(const Vec<_Tp, 2>& v); -#if (defined(__GNUC__) && __GNUC__ < 5) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 +#if (defined(__GNUC__) && __GNUC__ < 5) && !defined(__clang__) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 Point_& operator = (const Point_& pt); Point_& operator = (Point_&& pt) CV_NOEXCEPT = default; #elif OPENCV_ABI_COMPATIBILITY < 500 @@ -1186,7 +1186,7 @@ template inline Point_<_Tp>::Point_(_Tp _x, _Tp _y) : x(_x), y(_y) {} -#if (defined(__GNUC__) && __GNUC__ < 5) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 +#if (defined(__GNUC__) && __GNUC__ < 5) && !defined(__clang__) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 template inline Point_<_Tp>::Point_(const Point_& pt) : x(pt.x), y(pt.y) {} @@ -1200,7 +1200,7 @@ template inline Point_<_Tp>::Point_(const Vec<_Tp,2>& v) : x(v[0]), y(v[1]) {} -#if (defined(__GNUC__) && __GNUC__ < 5) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 +#if (defined(__GNUC__) && __GNUC__ < 5) && !defined(__clang__) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 template inline Point_<_Tp>& Point_<_Tp>::operator = (const Point_& pt) { From 7febec49b2e87677d2f5b4d4ad392a4aebda430d Mon Sep 17 00:00:00 2001 From: Maksim Shabunin Date: Thu, 21 Oct 2021 16:56:11 +0300 Subject: [PATCH 019/226] Merge pull request #20614 from mshabunin:use-onevpl-load videoio: use oneVPL load mechanism, encoder bitrate estimation * videoio: updated oneVPL support - use mfxLoad * videoio: advanced bitrate estimation for MFX encoder * videoio: improved MediaSDK/oneVPL/libva detection * videoio(ffmpeg): don't try oneVPL * videoio(test): tune checks of videoio_mfx.read_write_raw tests Co-authored-by: Alexander Alekhin --- modules/videoio/cmake/detect_msdk.cmake | 25 +++++++++----- modules/videoio/src/cap_ffmpeg_hw.hpp | 4 +++ modules/videoio/src/cap_mfx_common.cpp | 35 +++++++++++++++++-- modules/videoio/src/cap_mfx_common.hpp | 45 ++++++++++++++++++++----- modules/videoio/src/cap_mfx_reader.cpp | 16 +++++---- modules/videoio/src/cap_mfx_reader.hpp | 4 +-- modules/videoio/src/cap_mfx_writer.cpp | 31 +++++++++++++++-- modules/videoio/src/cap_mfx_writer.hpp | 4 +-- modules/videoio/test/test_mfx.cpp | 22 ++++++++---- 9 files changed, 148 insertions(+), 38 deletions(-) diff --git a/modules/videoio/cmake/detect_msdk.cmake b/modules/videoio/cmake/detect_msdk.cmake index 83701425e1..406c05824b 100644 --- a/modules/videoio/cmake/detect_msdk.cmake +++ b/modules/videoio/cmake/detect_msdk.cmake @@ -3,16 +3,23 @@ set(MFX_DEFS "") if(NOT HAVE_MFX) find_package(VPL QUIET) if(VPL_FOUND) - set(MFX_INCLUDE_DIRS "") - set(MFX_LIBRARIES "${VPL_IMPORTED_TARGETS}") - set(HAVE_MFX TRUE) - list(APPEND MFX_DEFS "HAVE_ONEVPL") + message(STATUS "VPL_VERSION: ${VPL_VERSION}") + # NOTE: oneVPL since 2021.4 have version 2.4, version 2021.1 does not work + if (VPL_VERSION VERSION_LESS "2021.2" AND VPL_VERSION VERSION_GREATER "2021.0") + message(STATUS "VPL version is too old (${VPL_DIR} : ${VPL_VERSION}) - skipping") + else() + set(MFX_INCLUDE_DIRS "") + set(MFX_LIBRARIES "${VPL_IMPORTED_TARGETS}") + set(HAVE_MFX TRUE) + list(APPEND MFX_DEFS "HAVE_ONEVPL") + endif() endif() endif() +set(paths) if(NOT HAVE_MFX) - set(paths "${MFX_HOME}" ENV "MFX_HOME" ENV "INTELMEDIASDKROOT") + set(paths "${MFX_HOME}" ENV "MFX_HOME" ENV "INTELMEDIASDKROOT" ${paths}) if(MSVC) if(MSVC_VERSION LESS 1900) set(vs_suffix) @@ -31,7 +38,7 @@ if(NOT HAVE_MFX) NO_DEFAULT_PATH) find_library(MFX_LIBRARY NAMES mfx libmfx${vs_suffix} PATHS ${paths} - PATH_SUFFIXES "lib64" "lib/lin_x64" "lib/${vs_arch}" + PATH_SUFFIXES "lib64" "lib/lin_x64" "lib/${vs_arch}" "lib" NO_DEFAULT_PATH) if(MFX_INCLUDE AND MFX_LIBRARY) set(HAVE_MFX TRUE) @@ -46,10 +53,11 @@ if(NOT HAVE_MFX AND PKG_CONFIG_FOUND) endif() if(HAVE_MFX AND UNIX) + set(paths "${VA_ROOT_DIR}" ENV "VA_ROOT_DIR" ${paths}) foreach(mode NO_DEFAULT_PATH "") find_path(MFX_va_INCLUDE va/va.h PATHS ${paths} PATH_SUFFIXES "include" ${mode}) - find_library(MFX_va_LIBRARY va PATHS ${paths} PATH_SUFFIXES "lib64" "lib/lin_x64" ${mode}) - find_library(MFX_va_drm_LIBRARY va-drm PATHS ${paths} PATH_SUFFIXES "lib64" "lib/lin_x64" ${mode}) + find_library(MFX_va_LIBRARY va PATHS ${paths} PATH_SUFFIXES "lib64" "lib/lin_x64" "lib" ${mode}) + find_library(MFX_va_drm_LIBRARY va-drm PATHS ${paths} PATH_SUFFIXES "lib64" "lib/lin_x64" "lib" ${mode}) if(MFX_va_INCLUDE AND MFX_va_LIBRARY AND MFX_va_drm_LIBRARY) list(APPEND MFX_INCLUDE_DIRS "${MFX_va_INCLUDE}") list(APPEND MFX_LIBRARIES "${MFX_va_LIBRARY}" "${MFX_va_drm_LIBRARY}") @@ -61,6 +69,7 @@ if(HAVE_MFX AND UNIX) unset(MFX_va_drm_LIBRARY CACHE) endforeach() if(NOT(MFX_va_INCLUDE AND MFX_va_LIBRARY AND MFX_va_drm_LIBRARY)) + message(STATUS "libva not found - turning MFX OFF") set(HAVE_MFX FALSE) endif() diff --git a/modules/videoio/src/cap_ffmpeg_hw.hpp b/modules/videoio/src/cap_ffmpeg_hw.hpp index 2a7b249739..31db5fdd5d 100644 --- a/modules/videoio/src/cap_ffmpeg_hw.hpp +++ b/modules/videoio/src/cap_ffmpeg_hw.hpp @@ -13,6 +13,10 @@ #endif #include +#if defined(HAVE_MFX) && defined(HAVE_ONEVPL) +#undef HAVE_MFX // libav's hwcontext_qsv.h doesn't expect oneVPL headers +#endif + #ifdef HAVE_D3D11 #define D3D11_NO_HELPERS #include diff --git a/modules/videoio/src/cap_mfx_common.cpp b/modules/videoio/src/cap_mfx_common.cpp index aea5c58561..d0f94931f3 100644 --- a/modules/videoio/src/cap_mfx_common.cpp +++ b/modules/videoio/src/cap_mfx_common.cpp @@ -14,11 +14,13 @@ using namespace std; using namespace cv; +#ifndef HAVE_ONEVPL static mfxIMPL getImpl() { static const size_t res = utils::getConfigurationParameterSizeT("OPENCV_VIDEOIO_MFX_IMPL", MFX_IMPL_AUTO_ANY); return (mfxIMPL)res; } +#endif static size_t getExtraSurfaceNum() { @@ -32,19 +34,46 @@ static size_t getPoolTimeoutSec() return res; } +#ifdef HAVE_ONEVPL +// oneVPL loader singleton (HW implementation only) +static mfxLoader setupVPLLoader() +{ + mfxLoader instance = MFXLoad(); + mfxConfig cfg = MFXCreateConfig(instance); + mfxVariant impl; + impl.Type = MFX_VARIANT_TYPE_U32; + impl.Data.U32 = MFX_IMPL_TYPE_HARDWARE; + MFXSetConfigFilterProperty(cfg, (const mfxU8*)"mfxImplDescription.Impl", impl); + DBG(cerr << "MFX Load: " << instance << endl); + return instance; +} + +mfxLoader getVPLLoaderInstance() +{ + static mfxLoader instance = setupVPLLoader(); + return instance; +} +#endif + //================================================================================================== -bool DeviceHandler::init(MFXVideoSession &session) +bool DeviceHandler::init(MFXVideoSession_WRAP &session) { mfxStatus res = MFX_ERR_NONE; - mfxIMPL impl = getImpl(); mfxVersion ver = { {19, 1} }; +#ifdef HAVE_ONEVPL + res = session.CreateSession(); + DBG(cout << "MFX CreateSession: " << res << endl); +#else + mfxIMPL impl = getImpl(); + res = session.Init(impl, &ver); DBG(cout << "MFX SessionInit: " << res << endl); res = session.QueryIMPL(&impl); DBG(cout << "MFX QueryIMPL: " << res << " => " << asHex(impl) << endl); +#endif res = session.QueryVersion(&ver); DBG(cout << "MFX QueryVersion: " << res << " => " << ver.Major << "." << ver.Minor << endl); @@ -77,7 +106,7 @@ VAHandle::~VAHandle() { } } -bool VAHandle::initDeviceSession(MFXVideoSession &session) { +bool VAHandle::initDeviceSession(MFXVideoSession_WRAP &session) { int majorVer = 0, minorVer = 0; VAStatus va_res = vaInitialize(display, &majorVer, &minorVer); DBG(cout << "vaInitialize: " << va_res << endl << majorVer << '.' << minorVer << endl); diff --git a/modules/videoio/src/cap_mfx_common.hpp b/modules/videoio/src/cap_mfx_common.hpp index 2830592163..9824e89dc5 100644 --- a/modules/videoio/src/cap_mfx_common.hpp +++ b/modules/videoio/src/cap_mfx_common.hpp @@ -12,12 +12,18 @@ #include #include +CV_SUPPRESS_DEPRECATED_START +# if defined(_MSC_VER) +# pragma warning(push) +# pragma warning(disable:4201) // nonstandard extension used: nameless struct/union +# endif #ifdef HAVE_ONEVPL # include # include # include # include # include +# include #else # include # include @@ -28,6 +34,10 @@ # include # endif #endif +# if defined(_MSC_VER) +# pragma warning(pop) +# endif +CV_SUPPRESS_DEPRECATED_END // // // Debug helpers // @@ -176,10 +186,29 @@ inline void cleanup(T * &ptr) //================================================================================================== +#ifdef HAVE_ONEVPL +mfxLoader getVPLLoaderInstance(); +#endif + +//================================================================================================== + +class MFXVideoSession_WRAP : public MFXVideoSession +{ +#ifdef HAVE_ONEVPL +public: + mfxStatus CreateSession() + { + return MFXCreateSession(getVPLLoaderInstance(), 0, &m_session); + } +#endif +}; + +//================================================================================================== + class Plugin { public: - static Plugin * loadEncoderPlugin(MFXVideoSession &session, mfxU32 codecId) + static Plugin * loadEncoderPlugin(MFXVideoSession_WRAP &session, mfxU32 codecId) { #ifdef HAVE_MFX_PLUGIN static const mfxPluginUID hevc_enc_uid = { 0x6f, 0xad, 0xc7, 0x91, 0xa0, 0xc2, 0xeb, 0x47, 0x9a, 0xb6, 0xdc, 0xd5, 0xea, 0x9d, 0xa3, 0x47 }; @@ -190,7 +219,7 @@ public: #endif return 0; } - static Plugin * loadDecoderPlugin(MFXVideoSession &session, mfxU32 codecId) + static Plugin * loadDecoderPlugin(MFXVideoSession_WRAP &session, mfxU32 codecId) { #ifdef HAVE_MFX_PLUGIN static const mfxPluginUID hevc_dec_uid = { 0x33, 0xa6, 0x1c, 0x0b, 0x4c, 0x27, 0x45, 0x4c, 0xa8, 0xd8, 0x5d, 0xde, 0x75, 0x7c, 0x6f, 0x8e }; @@ -213,9 +242,9 @@ private: mfxStatus res; private: #ifdef HAVE_MFX_PLUGIN - MFXVideoSession &session; + MFXVideoSession_WRAP &session; mfxPluginUID uid; - Plugin(MFXVideoSession &_session, mfxPluginUID _uid) : session(_session), uid(_uid) + Plugin(MFXVideoSession_WRAP &_session, mfxPluginUID _uid) : session(_session), uid(_uid) { res = MFXVideoUSER_Load(session, &uid, 1); } @@ -298,9 +327,9 @@ public: class DeviceHandler { public: virtual ~DeviceHandler() {} - bool init(MFXVideoSession &session); + bool init(MFXVideoSession_WRAP &session); protected: - virtual bool initDeviceSession(MFXVideoSession &session) = 0; + virtual bool initDeviceSession(MFXVideoSession_WRAP &session) = 0; }; @@ -340,7 +369,7 @@ public: private: VAHandle(const VAHandle &); VAHandle &operator=(const VAHandle &); - bool initDeviceSession(MFXVideoSession &session) CV_OVERRIDE; + bool initDeviceSession(MFXVideoSession_WRAP &session) CV_OVERRIDE; private: VADisplay display; int file; @@ -360,7 +389,7 @@ public: private: DXHandle(const DXHandle &); DXHandle &operator=(const DXHandle &); - bool initDeviceSession(MFXVideoSession &) CV_OVERRIDE { return true; } + bool initDeviceSession(MFXVideoSession_WRAP &) CV_OVERRIDE { return true; } }; #endif // _WIN32 diff --git a/modules/videoio/src/cap_mfx_reader.cpp b/modules/videoio/src/cap_mfx_reader.cpp index 330f9bd340..3c9d6327f8 100644 --- a/modules/videoio/src/cap_mfx_reader.cpp +++ b/modules/videoio/src/cap_mfx_reader.cpp @@ -41,7 +41,7 @@ VideoCapture_IntelMFX::VideoCapture_IntelMFX(const cv::String &filename) // Init device and session deviceHandler = createDeviceHandler(); - session = new MFXVideoSession(); + session = new MFXVideoSession_WRAP(); if (!deviceHandler->init(*session)) { MSG(cerr << "MFX: Can't initialize session" << endl); @@ -87,11 +87,11 @@ VideoCapture_IntelMFX::VideoCapture_IntelMFX(const cv::String &filename) return; } - // Adjust parameters + // Adjust parameters - COMMENTED: h265 decoder resets crop size to 0 (oneVPL/Win) - res = decoder->Query(¶ms, ¶ms); - DBG(cout << "MFX Query: " << res << endl << params.mfx << params.mfx.FrameInfo); - CV_Assert(res >= MFX_ERR_NONE); + //res = decoder->Query(¶ms, ¶ms); + //DBG(cout << "MFX Query: " << res << endl << params.mfx << params.mfx.FrameInfo); + //CV_Assert(res >= MFX_ERR_NONE); // Init surface pool @@ -105,7 +105,7 @@ VideoCapture_IntelMFX::VideoCapture_IntelMFX(const cv::String &filename) // Init decoder res = decoder->Init(¶ms); - DBG(cout << "MFX Init: " << res << endl << params.mfx.FrameInfo); + DBG(cout << "MFX decoder Init: " << res << endl << params.mfx.FrameInfo); if (res < MFX_ERR_NONE) { MSG(cerr << "MFX: Failed to init decoder: " << res << endl); @@ -113,6 +113,10 @@ VideoCapture_IntelMFX::VideoCapture_IntelMFX(const cv::String &filename) } frameSize = Size(params.mfx.FrameInfo.CropW, params.mfx.FrameInfo.CropH); + if (frameSize == Size(0, 0)) // sometimes Crop size is 0 + { + frameSize = Size(params.mfx.FrameInfo.Width, params.mfx.FrameInfo.Height); + } good = true; } diff --git a/modules/videoio/src/cap_mfx_reader.hpp b/modules/videoio/src/cap_mfx_reader.hpp index 122bc50763..6350c690ac 100644 --- a/modules/videoio/src/cap_mfx_reader.hpp +++ b/modules/videoio/src/cap_mfx_reader.hpp @@ -8,7 +8,7 @@ #include "precomp.hpp" -class MFXVideoSession; +class MFXVideoSession_WRAP; class Plugin; class DeviceHandler; class ReadBitstream; @@ -27,7 +27,7 @@ public: bool isOpened() const CV_OVERRIDE; int getCaptureDomain() CV_OVERRIDE; private: - MFXVideoSession *session; + MFXVideoSession_WRAP *session; Plugin *plugin; DeviceHandler *deviceHandler; ReadBitstream *bs; diff --git a/modules/videoio/src/cap_mfx_writer.cpp b/modules/videoio/src/cap_mfx_writer.cpp index 3bcca33d09..51157e9ba1 100644 --- a/modules/videoio/src/cap_mfx_writer.cpp +++ b/modules/videoio/src/cap_mfx_writer.cpp @@ -11,6 +11,31 @@ using namespace std; using namespace cv; +static float estimateBitrate(int codecId, size_t pixelNum, float fps) +{ + float bitrate = 0.f; + const float mp = pixelNum / 1000000.f; + if (codecId == MFX_CODEC_MPEG2) + { + bitrate = (mp * 43) * fps + 360; + } + else if (codecId == MFX_CODEC_AVC) + { + bitrate = (mp * 140 + 19) * pow(fps, 0.60f); + } + else if (codecId == MFX_CODEC_HEVC) + { + bitrate = (mp * 63 + 45) * pow(fps, 0.60f); + } + else + { + MSG(cerr << "MFX encoder Bitrate estimation FAILED" << endl); + } + DBG(cout << "MFX encoder Bitrate estimation (" << mp << " MP x " << fps << " fps): " << bitrate << endl); + return bitrate; + +} + static size_t getBitrateDivisor() { static const size_t res = utils::getConfigurationParameterSizeT("OPENCV_VIDEOIO_MFX_BITRATE_DIVISOR", 300); @@ -61,7 +86,7 @@ VideoWriter_IntelMFX::VideoWriter_IntelMFX(const String &filename, int _fourcc, // Init device and session deviceHandler = createDeviceHandler(); - session = new MFXVideoSession(); + session = new MFXVideoSession_WRAP(); if (!deviceHandler->init(*session)) { MSG(cerr << "MFX: Can't initialize session" << endl); @@ -90,7 +115,7 @@ VideoWriter_IntelMFX::VideoWriter_IntelMFX(const String &filename, int _fourcc, memset(¶ms, 0, sizeof(params)); params.mfx.CodecId = codecId; params.mfx.TargetUsage = MFX_TARGETUSAGE_BALANCED; - params.mfx.TargetKbps = saturate_cast((frameSize.area() * fps) / (42.6666 * getBitrateDivisor())); // TODO: set in options + params.mfx.TargetKbps = saturate_cast(estimateBitrate(codecId, frameSize.area(), (float)fps) * 300 / getBitrateDivisor()); // TODO: set in options params.mfx.RateControlMethod = MFX_RATECONTROL_VBR; params.mfx.FrameInfo.FrameRateExtN = cvRound(fps * 1000); params.mfx.FrameInfo.FrameRateExtD = 1000; @@ -122,7 +147,7 @@ VideoWriter_IntelMFX::VideoWriter_IntelMFX(const String &filename, int _fourcc, // Init encoder res = encoder->Init(¶ms); - DBG(cout << "MFX Init: " << res << endl << params.mfx.FrameInfo); + DBG(cout << "MFX encoder Init: " << res << endl << params.mfx.FrameInfo); if (res < MFX_ERR_NONE) { MSG(cerr << "MFX: Failed to init encoder: " << res << endl); diff --git a/modules/videoio/src/cap_mfx_writer.hpp b/modules/videoio/src/cap_mfx_writer.hpp index f4913a8f31..fff2a6387f 100644 --- a/modules/videoio/src/cap_mfx_writer.hpp +++ b/modules/videoio/src/cap_mfx_writer.hpp @@ -7,7 +7,7 @@ #include "precomp.hpp" -class MFXVideoSession; +class MFXVideoSession_WRAP; class Plugin; class DeviceHandler; class WriteBitstream; @@ -33,7 +33,7 @@ private: VideoWriter_IntelMFX & operator=(const VideoWriter_IntelMFX &); private: - MFXVideoSession *session; + MFXVideoSession_WRAP *session; Plugin *plugin; DeviceHandler *deviceHandler; WriteBitstream *bs; diff --git a/modules/videoio/test/test_mfx.cpp b/modules/videoio/test/test_mfx.cpp index 2048fe5af9..032d5a96e0 100644 --- a/modules/videoio/test/test_mfx.cpp +++ b/modules/videoio/test/test_mfx.cpp @@ -97,6 +97,11 @@ TEST_P(videoio_mfx, read_write_raw) const String filename = cv::tempfile(ext); const int fourcc = fourccByExt(ext); + // For some reason MPEG2 codec does not work well with this particular videostream at 1 FPS + // even with large bitrate values. Thus skipping this case. + if (FPS == 1. && fourcc == VideoWriter::fourcc('M', 'P', 'G', '2')) + throw SkipTestException("This configuration is not supported"); + bool isColor = true; std::queue goodFrames; @@ -120,24 +125,29 @@ TEST_P(videoio_mfx, read_write_raw) ASSERT_TRUE(cap.isOpened()); EXPECT_EQ(FRAME_SIZE.width, cap.get(CAP_PROP_FRAME_WIDTH)); EXPECT_EQ(FRAME_SIZE.height, cap.get(CAP_PROP_FRAME_HEIGHT)); + double psnrThreshold = (fourcc == VideoWriter::fourcc('M', 'P', 'G', '2')) ? 27.0 : 29.5; // experimentally chosen value for (int i = 0; i < FRAME_COUNT; ++i) { + SCOPED_TRACE(i); ASSERT_TRUE(cap.read(frame)); ASSERT_FALSE(frame.empty()); ASSERT_EQ(FRAME_SIZE.width, frame.cols); ASSERT_EQ(FRAME_SIZE.height, frame.rows); // verify ASSERT_NE(goodFrames.size(), 0u); - const Mat &goodFrame = goodFrames.front(); + const Mat goodFrame = goodFrames.front(); goodFrames.pop(); EXPECT_EQ(goodFrame.depth(), frame.depth()); EXPECT_EQ(goodFrame.channels(), frame.channels()); EXPECT_EQ(goodFrame.type(), frame.type()); double psnr = cvtest::PSNR(goodFrame, frame); - if (fourcc == VideoWriter::fourcc('M', 'P', 'G', '2')) - EXPECT_GT(psnr, 31); // experimentally chosen value - else - EXPECT_GT(psnr, 33); // experimentally chosen value - goodFrames.pop(); + if ((i == 1 || i == 4) && fourcc == VideoWriter::fourcc('H', '2', '6', '5')) + { + // ignore bugs of some HW/SW configurations: + // - (added 2021-10) i7-11700K, Win10, oneVPL 2021.4.0 / 2021.6.0 + std::cout << "SKIP: bypass frame content check: i=" << i << " psnr=" << psnr << ", expected to be >= " << psnrThreshold << std::endl; + continue; + } + EXPECT_GE(psnr, psnrThreshold); } EXPECT_FALSE(cap.read(frame)); EXPECT_TRUE(frame.empty()); From d21622bef4bff56f322f56f7da3af28a8a2d7104 Mon Sep 17 00:00:00 2001 From: AleksandrPanov Date: Thu, 21 Oct 2021 18:12:51 +0300 Subject: [PATCH 020/226] fix findMinEnclosingTriangle and add tests --- .../imgproc/src/min_enclosing_triangle.cpp | 3 +- modules/imgproc/test/test_convhull.cpp | 33 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/modules/imgproc/src/min_enclosing_triangle.cpp b/modules/imgproc/src/min_enclosing_triangle.cpp index 2651871412..7bd15fced0 100644 --- a/modules/imgproc/src/min_enclosing_triangle.cpp +++ b/modules/imgproc/src/min_enclosing_triangle.cpp @@ -317,8 +317,9 @@ namespace minEnclosingTriangle { */ static void findMinEnclosingTriangle(cv::InputArray points, CV_OUT cv::OutputArray triangle, CV_OUT double &area) { - std::vector resultingTriangle, polygon; CV_Assert(!points.empty()); + std::vector resultingTriangle; + cv::Mat polygon; convexHull(points, polygon, true, true); findMinEnclosingTriangle(polygon, resultingTriangle, area); cv::Mat(resultingTriangle).copyTo(triangle); diff --git a/modules/imgproc/test/test_convhull.cpp b/modules/imgproc/test/test_convhull.cpp index dee3769762..bc5c940827 100644 --- a/modules/imgproc/test/test_convhull.cpp +++ b/modules/imgproc/test/test_convhull.cpp @@ -2457,5 +2457,38 @@ TEST(Imgproc_minAreaRect, reproducer_19769) EXPECT_TRUE(checkMinAreaRect(rr, contour)) << rr.center << " " << rr.size << " " << rr.angle; } +TEST(Imgproc_minEnclosingTriangle, regression_17585) +{ + const int N = 3; + float pts_[N][2] = { {0, 0}, {0, 1}, {1, 1} }; + cv::Mat points(N, 2, CV_32FC1, static_cast(pts_)); + vector triangle; + + EXPECT_NO_THROW(minEnclosingTriangle(points, triangle)); +} + +TEST(Imgproc_minEnclosingTriangle, regression_20890) +{ + vector points; + points.push_back(Point(0, 0)); + points.push_back(Point(0, 1)); + points.push_back(Point(1, 1)); + vector triangle; + + EXPECT_NO_THROW(minEnclosingTriangle(points, triangle)); +} + +TEST(Imgproc_minEnclosingTriangle, regression_mat_with_diff_channels) +{ + const int N = 3; + float pts_[N][2] = { {0, 0}, {0, 1}, {1, 1} }; + cv::Mat points1xN(1, N, CV_32FC2, static_cast(pts_)); + cv::Mat pointsNx1(N, 1, CV_32FC2, static_cast(pts_)); + vector triangle; + + EXPECT_NO_THROW(minEnclosingTriangle(points1xN, triangle)); + EXPECT_NO_THROW(minEnclosingTriangle(pointsNx1, triangle)); +} + }} // namespace /* End of file. */ From d376fe9e173379630d869faa8e21c07bf78cd199 Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Thu, 21 Oct 2021 19:12:03 +0300 Subject: [PATCH 021/226] Merge pull request #20921 from sivanov-work:atl_patch G-API: FIX OpenVINO build - Add CComPtr RAII replacement into G-API sample * Add CComPtr RAII replacement into G-API sample * Remove completely `atlbase.h' --- .../gapi/samples/onevpl_infer_single_roi.cpp | 47 +++++++++++++------ .../onevpl/cfg_param_device_selector.cpp | 13 +++-- modules/gapi/src/streaming/onevpl/utils.hpp | 21 +++++++++ 3 files changed, 60 insertions(+), 21 deletions(-) diff --git a/modules/gapi/samples/onevpl_infer_single_roi.cpp b/modules/gapi/samples/onevpl_infer_single_roi.cpp index deca86f1ca..9da64c99d3 100644 --- a/modules/gapi/samples/onevpl_infer_single_roi.cpp +++ b/modules/gapi/samples/onevpl_infer_single_roi.cpp @@ -25,7 +25,6 @@ #define D3D11_NO_HELPERS #define NOMINMAX #include -#include #include #pragma comment(lib, "dxgi") #undef NOMINMAX @@ -64,9 +63,29 @@ std::string get_weights_path(const std::string &model_path) { #ifdef HAVE_DIRECTX #ifdef HAVE_D3D11 -using AccelParamsType = std::tuple, CComPtr>; +// Since ATL headers might not be available on specific MSVS Build Tools +// we use simple `CComPtr` implementation like as `ComPtrGuard` +// which is not supposed to be the full functional replacement of `CComPtr` +// and it uses as RAII to make sure utilization is correct +template +void release(COMNonManageableType *ptr) { + if (ptr) { + ptr->Release(); + } +} -AccelParamsType create_device_with_ctx(CComPtr adapter) { +template +using ComPtrGuard = std::unique_ptr)>; + +template +ComPtrGuard createCOMPtrGuard(COMNonManageableType *ptr = nullptr) { + return ComPtrGuard {ptr, &release}; +} + + +using AccelParamsType = std::tuple, ComPtrGuard>; + +AccelParamsType create_device_with_ctx(IDXGIAdapter* adapter) { UINT flags = 0; D3D_FEATURE_LEVEL feature_levels[] = { D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0, @@ -85,7 +104,8 @@ AccelParamsType create_device_with_ctx(CComPtr adapter) { std::to_string(HRESULT_CODE(err))); } - return std::make_tuple(ret_device_ptr, ret_ctx_ptr); + return std::make_tuple(createCOMPtrGuard(ret_device_ptr), + createCOMPtrGuard(ret_ctx_ptr)); } #endif // HAVE_D3D11 #endif // HAVE_DIRECTX @@ -270,12 +290,11 @@ int main(int argc, char *argv[]) { #ifdef HAVE_INF_ENGINE #ifdef HAVE_DIRECTX #ifdef HAVE_D3D11 - CComPtr dx11_dev; - CComPtr dx11_ctx; + auto dx11_dev = createCOMPtrGuard(); + auto dx11_ctx = createCOMPtrGuard(); if (device_id.find("GPU") != std::string::npos) { - CComPtr adapter_factory; - CComPtr intel_adapters; + auto adapter_factory = createCOMPtrGuard(); { IDXGIFactory* out_factory = nullptr; HRESULT err = CreateDXGIFactory(__uuidof(IDXGIFactory), @@ -284,10 +303,10 @@ int main(int argc, char *argv[]) { std::cerr << "Cannot create CreateDXGIFactory, error: " << HRESULT_CODE(err) << std::endl; return -1; } - adapter_factory.Attach(out_factory); + adapter_factory = createCOMPtrGuard(out_factory); } - CComPtr intel_adapter; + auto intel_adapter = createCOMPtrGuard(); UINT adapter_index = 0; const unsigned int refIntelVendorID = 0x8086; IDXGIAdapter* out_adapter = nullptr; @@ -296,7 +315,7 @@ int main(int argc, char *argv[]) { DXGI_ADAPTER_DESC desc{}; out_adapter->GetDesc(&desc); if (desc.VendorId == refIntelVendorID) { - intel_adapter.Attach(out_adapter); + intel_adapter = createCOMPtrGuard(out_adapter); break; } ++adapter_index; @@ -307,9 +326,9 @@ int main(int argc, char *argv[]) { return -1; } - std::tie(dx11_dev, dx11_ctx) = create_device_with_ctx(intel_adapter); - accel_device_ptr = reinterpret_cast(dx11_dev.p); - accel_ctx_ptr = reinterpret_cast(dx11_ctx.p); + std::tie(dx11_dev, dx11_ctx) = create_device_with_ctx(intel_adapter.get()); + accel_device_ptr = reinterpret_cast(dx11_dev.get()); + accel_ctx_ptr = reinterpret_cast(dx11_ctx.get()); // put accel type description for VPL source source_cfgs.push_back(cfg::create_from_string( diff --git a/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp b/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp index c8fd49c1ad..0bdec70986 100644 --- a/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp +++ b/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp @@ -10,6 +10,7 @@ #include #include "streaming/onevpl/cfg_param_device_selector.hpp" +#include "streaming/onevpl/utils.hpp" #include "logger.hpp" #ifdef HAVE_DIRECTX @@ -19,7 +20,6 @@ // get rid of generate macro max/min/etc from DX side #define D3D11_NO_HELPERS #define NOMINMAX -#include #include #include #pragma comment(lib, "dxgi") @@ -113,8 +113,7 @@ CfgParamDeviceSelector::CfgParamDeviceSelector(const CfgParams& cfg_params) : }; D3D_FEATURE_LEVEL featureLevel; - CComPtr adapter_factory; - CComPtr intel_adapters; + auto adapter_factory = createCOMPtrGuard(); { IDXGIFactory* out_factory = nullptr; HRESULT err = CreateDXGIFactory(__uuidof(IDXGIFactory), @@ -122,10 +121,10 @@ CfgParamDeviceSelector::CfgParamDeviceSelector(const CfgParams& cfg_params) : if (FAILED(err)) { throw std::runtime_error("Cannot create CreateDXGIFactory, error: " + std::to_string(HRESULT_CODE(err))); } - adapter_factory.Attach(out_factory); + adapter_factory = createCOMPtrGuard(out_factory); } - CComPtr intel_adapter; + auto intel_adapter = createCOMPtrGuard(); UINT adapter_index = 0; const unsigned int refIntelVendorID = 0x8086; IDXGIAdapter* out_adapter = nullptr; @@ -134,7 +133,7 @@ CfgParamDeviceSelector::CfgParamDeviceSelector(const CfgParams& cfg_params) : DXGI_ADAPTER_DESC desc{}; out_adapter->GetDesc(&desc); if (desc.VendorId == refIntelVendorID) { - intel_adapter.Attach(out_adapter); + intel_adapter = createCOMPtrGuard(out_adapter); break; } ++adapter_index; @@ -145,7 +144,7 @@ CfgParamDeviceSelector::CfgParamDeviceSelector(const CfgParams& cfg_params) : } // Create the Direct3D 11 API device object and a corresponding context. - HRESULT err = D3D11CreateDevice(intel_adapter, + HRESULT err = D3D11CreateDevice(intel_adapter.get(), D3D_DRIVER_TYPE_UNKNOWN, nullptr, creationFlags, featureLevels, ARRAYSIZE(featureLevels), diff --git a/modules/gapi/src/streaming/onevpl/utils.hpp b/modules/gapi/src/streaming/onevpl/utils.hpp index 94f5a249e8..19c1b135dc 100644 --- a/modules/gapi/src/streaming/onevpl/utils.hpp +++ b/modules/gapi/src/streaming/onevpl/utils.hpp @@ -16,6 +16,7 @@ #include #include +#include #include #include @@ -26,6 +27,26 @@ namespace gapi { namespace wip { namespace onevpl { +// Since ATL headers might not be available on specific MSVS Build Tools +// we use simple `CComPtr` implementation like as `ComPtrGuard` +// which is not supposed to be the full functional replacement of `CComPtr` +// and it uses as RAII to make sure utilization is correct +template +void release(COMNonManageableType *ptr) { + if (ptr) { + ptr->Release(); + } +} + +template +using ComPtrGuard = std::unique_ptr)>; + +template +ComPtrGuard createCOMPtrGuard(COMNonManageableType *ptr = nullptr) { + return ComPtrGuard {ptr, &release}; +} + + const char* mfx_impl_to_cstr(const mfxIMPL impl); mfxIMPL cstr_to_mfx_impl(const char* cstr); From 0a58d6812ac053d5f52142f6f6f3684d2beec800 Mon Sep 17 00:00:00 2001 From: atrutnev Date: Fri, 22 Oct 2021 11:06:09 +0300 Subject: [PATCH 022/226] fix gkernel Doxygen documentation --- modules/gapi/include/opencv2/gapi/gkernel.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/gapi/include/opencv2/gapi/gkernel.hpp b/modules/gapi/include/opencv2/gapi/gkernel.hpp index cfac552bfe..ca942495ab 100644 --- a/modules/gapi/include/opencv2/gapi/gkernel.hpp +++ b/modules/gapi/include/opencv2/gapi/gkernel.hpp @@ -719,7 +719,7 @@ namespace gapi { * @{ */ /** - * @brief cv::use_only() is a special combinator which hints G-API to use only + * @brief cv::gapi::use_only() is a special combinator which hints G-API to use only * kernels specified in cv::GComputation::compile() (and not to extend kernels available by * default with that package). */ From a6f57175679f230fb9e83ee8703594210e2e5017 Mon Sep 17 00:00:00 2001 From: berak Date: Thu, 21 Oct 2021 11:38:17 +0200 Subject: [PATCH 023/226] resolves #20913 imgproc: remove asserts for circles_ in HoughCircles --- modules/imgproc/src/hough.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/imgproc/src/hough.cpp b/modules/imgproc/src/hough.cpp index e66be8b4b7..b48b7ea137 100644 --- a/modules/imgproc/src/hough.cpp +++ b/modules/imgproc/src/hough.cpp @@ -1728,7 +1728,6 @@ static void HoughCircles( InputArray _image, OutputArray _circles, } CV_Assert(!_image.empty() && _image.type() == CV_8UC1 && (_image.isMat() || _image.isUMat())); - CV_Assert(_circles.isMat() || _circles.isVector()); if( dp <= 0 || minDist <= 0 || param1 <= 0 || param2 <= 0) CV_Error( Error::StsOutOfRange, "dp, min_dist, canny_threshold and acc_threshold must be all positive numbers" ); From 9267536feefa0e0c5ca6f1113b6e22debe438ddf Mon Sep 17 00:00:00 2001 From: Harvey <619328684@qq.com> Date: Fri, 22 Oct 2021 22:04:19 +0800 Subject: [PATCH 024/226] Merge pull request #20875 from Harvey-Huang:master * bmp specified BI_BITFIELDS should take care RGBA bit mask * change the name * support xrgb bmp file * support xrgb bmp file(add test case) * update testing code --- modules/imgcodecs/src/grfmt_bmp.cpp | 56 +++++++++++++++++++++++++-- modules/imgcodecs/src/grfmt_bmp.hpp | 5 +++ modules/imgcodecs/test/test_grfmt.cpp | 26 +++++++++++++ 3 files changed, 84 insertions(+), 3 deletions(-) diff --git a/modules/imgcodecs/src/grfmt_bmp.cpp b/modules/imgcodecs/src/grfmt_bmp.cpp index 880a8cd105..c838e819f6 100644 --- a/modules/imgcodecs/src/grfmt_bmp.cpp +++ b/modules/imgcodecs/src/grfmt_bmp.cpp @@ -58,6 +58,7 @@ BmpDecoder::BmpDecoder() m_origin = ORIGIN_TL; m_bpp = 0; m_rle_code = BMP_RGB; + initMask(); } @@ -97,6 +98,7 @@ bool BmpDecoder::readHeader() int size = m_strm.getDWord(); CV_Assert(size > 0); // overflow, 2Gb limit + initMask(); if( size >= 36 ) { m_width = m_strm.getDWord(); @@ -107,7 +109,30 @@ bool BmpDecoder::readHeader() m_rle_code = (BmpCompression)m_rle_code_; m_strm.skip(12); int clrused = m_strm.getDWord(); - m_strm.skip( size - 36 ); + + if( m_bpp == 32 && m_rle_code == BMP_BITFIELDS && size >= 56 ) + { + m_strm.skip(4); //important colors + //0 is Red channel bit mask, 1 is Green channel bit mask, 2 is Blue channel bit mask, 3 is Alpha channel bit mask + for( int index_rgba = 0; index_rgba < 4; ++index_rgba ) + { + uint mask = m_strm.getDWord(); + m_rgba_mask[index_rgba] = mask; + if(mask != 0) + { + int bit_count = 0; + while(!(mask & 1)) + { + mask >>= 1; + ++bit_count; + } + m_rgba_bit_offset[index_rgba] = bit_count; + } + } + m_strm.skip( size - 56 ); + } + else + m_strm.skip( size - 36 ); if( m_width > 0 && m_height != 0 && (((m_bpp == 1 || m_bpp == 4 || m_bpp == 8 || @@ -486,8 +511,14 @@ decode_rle8_bad: ; icvCvt_BGRA2Gray_8u_C4C1R( src, 0, data, 0, Size(m_width,1) ); else if( img.channels() == 3 ) icvCvt_BGRA2BGR_8u_C4C3R(src, 0, data, 0, Size(m_width, 1)); - else if( img.channels() == 4 ) - memcpy(data, src, m_width * 4); + else if ( img.channels() == 4 ) + { + bool has_bit_mask = (m_rgba_bit_offset[0] >= 0) && (m_rgba_bit_offset[1] >= 0) && (m_rgba_bit_offset[2] >= 0); + if ( has_bit_mask ) + maskBGRA(data, src, m_width); + else + memcpy(data, src, m_width * 4); + } } result = true; break; @@ -503,7 +534,26 @@ decode_rle8_bad: ; return result; } +void BmpDecoder::initMask() +{ + memset(m_rgba_mask, 0, sizeof(m_rgba_mask)); + memset(m_rgba_bit_offset, -1, sizeof(m_rgba_bit_offset)); +} +void BmpDecoder::maskBGRA(uchar* des, uchar* src, int num) +{ + for( int i = 0; i < num; i++, des += 4, src += 4 ) + { + uint data = *((uint*)src); + des[0] = (uchar)((m_rgba_mask[2] & data) >> m_rgba_bit_offset[2]); + des[1] = (uchar)((m_rgba_mask[1] & data) >> m_rgba_bit_offset[1]); + des[2] = (uchar)((m_rgba_mask[0] & data) >> m_rgba_bit_offset[0]); + if (m_rgba_bit_offset[3] >= 0) + des[3] = (uchar)((m_rgba_mask[3] & data) >> m_rgba_bit_offset[3]); + else + des[3] = 255; + } +} ////////////////////////////////////////////////////////////////////////////////////////// BmpEncoder::BmpEncoder() diff --git a/modules/imgcodecs/src/grfmt_bmp.hpp b/modules/imgcodecs/src/grfmt_bmp.hpp index e94b8ff8b4..4b34124bf7 100644 --- a/modules/imgcodecs/src/grfmt_bmp.hpp +++ b/modules/imgcodecs/src/grfmt_bmp.hpp @@ -73,6 +73,9 @@ public: protected: + void initMask(); + void maskBGRA(uchar* des, uchar* src, int num); + enum Origin { ORIGIN_TL = 0, @@ -85,6 +88,8 @@ protected: int m_bpp; int m_offset; BmpCompression m_rle_code; + uint m_rgba_mask[4]; + int m_rgba_bit_offset[4]; }; diff --git a/modules/imgcodecs/test/test_grfmt.cpp b/modules/imgcodecs/test/test_grfmt.cpp index 0d28688004..03a8d408d6 100644 --- a/modules/imgcodecs/test/test_grfmt.cpp +++ b/modules/imgcodecs/test/test_grfmt.cpp @@ -295,6 +295,32 @@ TEST(Imgcodecs_Bmp, read_rle8) EXPECT_PRED_FORMAT2(cvtest::MatComparator(0, 0), rle, ord); } +TEST(Imgcodecs_Bmp, rgba_bit_mask) +{ + const string root = cvtest::TS::ptr()->get_data_path(); + const string filenameInput = root + "readwrite/test_rgba_mask.bmp"; + + const Mat img = cv::imread(filenameInput, IMREAD_UNCHANGED); + ASSERT_FALSE(img.empty()); + ASSERT_EQ(CV_8UC4, img.type()); + + const uchar* data = img.ptr(); + ASSERT_EQ(data[3], 255); +} + +TEST(Imgcodecs_Bmp, read_32bit_xrgb) +{ + const string root = cvtest::TS::ptr()->get_data_path(); + const string filenameInput = root + "readwrite/test_32bit_xrgb.bmp"; + + const Mat img = cv::imread(filenameInput, IMREAD_UNCHANGED); + ASSERT_FALSE(img.empty()); + ASSERT_EQ(CV_8UC4, img.type()); + + const uchar* data = img.ptr(); + ASSERT_EQ(data[3], 255); +} + #ifdef HAVE_IMGCODEC_HDR TEST(Imgcodecs_Hdr, regression) { From 3891f18e7115c9090706825d5bad67ea7db4cde5 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 18 Oct 2021 20:48:24 +0000 Subject: [PATCH 025/226] samples: update parallel_backend examples - use find_package(TBB) --- samples/cpp/CMakeLists.txt | 4 +- .../core/parallel_backend/CMakeLists.txt | 49 ++++++++++++------- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/samples/cpp/CMakeLists.txt b/samples/cpp/CMakeLists.txt index b378998022..050397ee91 100644 --- a/samples/cpp/CMakeLists.txt +++ b/samples/cpp/CMakeLists.txt @@ -72,6 +72,8 @@ include("tutorial_code/calib3d/real_time_pose_estimation/CMakeLists.txt" OPTIONA if(OpenCV_FOUND AND NOT CMAKE_VERSION VERSION_LESS "3.1") add_subdirectory("example_cmake") endif() -if(OpenCV_FOUND AND NOT CMAKE_VERSION VERSION_LESS "3.9") +if(OpenCV_FOUND AND NOT CMAKE_VERSION VERSION_LESS "3.9" + AND NOT OPENCV_EXAMPLES_SKIP_PARALLEL_BACKEND +) add_subdirectory("tutorial_code/core/parallel_backend") endif() diff --git a/samples/cpp/tutorial_code/core/parallel_backend/CMakeLists.txt b/samples/cpp/tutorial_code/core/parallel_backend/CMakeLists.txt index 0e67dc29e2..9d4f91d4a5 100644 --- a/samples/cpp/tutorial_code/core/parallel_backend/CMakeLists.txt +++ b/samples/cpp/tutorial_code/core/parallel_backend/CMakeLists.txt @@ -2,25 +2,36 @@ cmake_minimum_required(VERSION 3.9) find_package(OpenCV REQUIRED COMPONENTS opencv_core) -find_package(OpenMP) -if(OpenMP_FOUND) - project(opencv_example_openmp_backend) - add_executable(opencv_example_openmp_backend example-openmp.cpp) - target_link_libraries(opencv_example_openmp_backend PRIVATE - opencv_core - OpenMP::OpenMP_CXX - ) +if(NOT OPENCV_EXAMPLES_SKIP_PARALLEL_BACKEND_OPENMP + AND NOT OPENCV_EXAMPLES_SKIP_OPENMP +) + find_package(OpenMP) + if(OpenMP_FOUND) + project(opencv_example_openmp_backend) + add_executable(opencv_example_openmp_backend example-openmp.cpp) + target_link_libraries(opencv_example_openmp_backend PRIVATE + opencv_core + OpenMP::OpenMP_CXX + ) + endif() endif() -# TODO: find_package(TBB) -find_path(TBB_INCLUDE_DIR NAMES "tbb/tbb.h") -find_library(TBB_LIBRARY NAMES "tbb") -if(TBB_INCLUDE_DIR AND TBB_LIBRARY AND NOT OPENCV_EXAMPLE_SKIP_TBB) - project(opencv_example_tbb_backend) - add_executable(opencv_example_tbb_backend example-tbb.cpp) - target_include_directories(opencv_example_tbb_backend SYSTEM PRIVATE ${TBB_INCLUDE_DIR}) - target_link_libraries(opencv_example_tbb_backend PRIVATE - opencv_core - ${TBB_LIBRARY} - ) +if(NOT OPENCV_EXAMPLES_SKIP_PARALLEL_BACKEND_TBB + AND NOT OPENCV_EXAMPLES_SKIP_TBB + AND NOT OPENCV_EXAMPLE_SKIP_TBB # deprecated (to be removed in OpenCV 5.0) +) + find_package(TBB) + if(NOT TBB_FOUND) + find_path(TBB_INCLUDE_DIR NAMES "tbb/tbb.h") + find_library(TBB_LIBRARY NAMES "tbb") + endif() + if(TBB_INCLUDE_DIR AND TBB_LIBRARY) + project(opencv_example_tbb_backend) + add_executable(opencv_example_tbb_backend example-tbb.cpp) + target_include_directories(opencv_example_tbb_backend SYSTEM PRIVATE ${TBB_INCLUDE_DIR}) + target_link_libraries(opencv_example_tbb_backend PRIVATE + opencv_core + ${TBB_LIBRARY} + ) + endif() endif() From 0b1ae11498022c290950043b38a2f707ff676be7 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 26 Oct 2021 14:29:57 +0000 Subject: [PATCH 026/226] cmake: find_package with QUIET --- samples/cpp/tutorial_code/core/parallel_backend/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/cpp/tutorial_code/core/parallel_backend/CMakeLists.txt b/samples/cpp/tutorial_code/core/parallel_backend/CMakeLists.txt index 9d4f91d4a5..857d9b76ef 100644 --- a/samples/cpp/tutorial_code/core/parallel_backend/CMakeLists.txt +++ b/samples/cpp/tutorial_code/core/parallel_backend/CMakeLists.txt @@ -20,7 +20,7 @@ if(NOT OPENCV_EXAMPLES_SKIP_PARALLEL_BACKEND_TBB AND NOT OPENCV_EXAMPLES_SKIP_TBB AND NOT OPENCV_EXAMPLE_SKIP_TBB # deprecated (to be removed in OpenCV 5.0) ) - find_package(TBB) + find_package(TBB QUIET) if(NOT TBB_FOUND) find_path(TBB_INCLUDE_DIR NAMES "tbb/tbb.h") find_library(TBB_LIBRARY NAMES "tbb") From 0a71063530c215e8ad90da0c35b2e5cc829d64b5 Mon Sep 17 00:00:00 2001 From: MaximMilashchenko <67949029+MaximMilashchenko@users.noreply.github.com> Date: Tue, 26 Oct 2021 17:33:53 +0300 Subject: [PATCH 027/226] Merge pull request #20942 from MaximMilashchenko:AudioPatch Audio patch * fixed microphone, audio position * fixed docs * changed AudioOpenCheck --- modules/videoio/src/cap_msmf.cpp | 168 +++++++++++++++------------- modules/videoio/test/test_audio.cpp | 11 ++ 2 files changed, 104 insertions(+), 75 deletions(-) diff --git a/modules/videoio/src/cap_msmf.cpp b/modules/videoio/src/cap_msmf.cpp index 0fa064dfb8..39f191e642 100644 --- a/modules/videoio/src/cap_msmf.cpp +++ b/modules/videoio/src/cap_msmf.cpp @@ -709,6 +709,7 @@ public: virtual void close(); virtual double getProperty(int) const CV_OVERRIDE; virtual bool setProperty(int, double) CV_OVERRIDE; + bool configureAudioFrame(); bool grabAudioFrame(); bool grabVideoFrame(); virtual bool grabFrame() CV_OVERRIDE; @@ -1038,6 +1039,7 @@ bool CvCapture_MSMF::configureAudioOutput(MediaType newType) if (bestMatch.second.isEmpty(true)) { CV_LOG_DEBUG(NULL, "Can not find audio stream with requested parameters"); + isOpen = false; return false; } dwAudioStreamIndex = bestMatch.first.stream; @@ -1439,6 +1441,91 @@ bool CvCapture_MSMF::grabVideoFrame() return returnFlag; } +bool CvCapture_MSMF::configureAudioFrame() +{ + if (!audioSamples.empty() || !bufferAudioData.empty() && aEOS) + { + _ComPtr buf = NULL; + std::vector audioDataInUse; + BYTE* ptr = NULL; + DWORD maxsize = 0, cursize = 0; + CV_TRACE_REGION("get_contiguous_buffer"); + for (auto item : audioSamples) + { + if (!SUCCEEDED(item->ConvertToContiguousBuffer(&buf))) + { + CV_TRACE_REGION("get_buffer"); + DWORD bcnt = 0; + if (!SUCCEEDED(item->GetBufferCount(&bcnt))) + break; + if (bcnt == 0) + break; + if (!SUCCEEDED(item->GetBufferByIndex(0, &buf))) + break; + } + if (!SUCCEEDED(buf->Lock(&ptr, &maxsize, &cursize))) + break; + size_t lastSize = bufferAudioData.size(); + bufferAudioData.resize(lastSize+cursize); + for (unsigned int i = 0; i < cursize; i++) + { + bufferAudioData[lastSize+i]=*(ptr+i); + } + CV_TRACE_REGION_NEXT("unlock"); + buf->Unlock(); + buf = NULL; + } + audioSamples.clear(); + + audioSamplePos += chunkLengthOfBytes/((captureAudioFormat.bit_per_sample/8)*captureAudioFormat.nChannels); + chunkLengthOfBytes = (videoStream != -1) ? (LONGLONG)((requiredAudioTime*captureAudioFormat.nSamplesPerSec*captureAudioFormat.nChannels*(captureAudioFormat.bit_per_sample)/8)/1e7) : cursize; + if ((videoStream != -1) && (chunkLengthOfBytes % ((int)(captureAudioFormat.bit_per_sample)/8* (int)captureAudioFormat.nChannels) != 0)) + { + if ( (double)audioSamplePos/captureAudioFormat.nSamplesPerSec + audioStartOffset * 1e-7 - usedVideoSampleTime * 1e-7 >= 0 ) + chunkLengthOfBytes -= numberOfAdditionalAudioBytes; + numberOfAdditionalAudioBytes = ((int)(captureAudioFormat.bit_per_sample)/8* (int)captureAudioFormat.nChannels) + - chunkLengthOfBytes % ((int)(captureAudioFormat.bit_per_sample)/8* (int)captureAudioFormat.nChannels); + chunkLengthOfBytes += numberOfAdditionalAudioBytes; + } + if (lastFrame && !syncLastFrame || aEOS && !vEOS) + { + chunkLengthOfBytes = bufferAudioData.size(); + audioSamplePos += chunkLengthOfBytes/((captureAudioFormat.bit_per_sample/8)*captureAudioFormat.nChannels); + } + CV_Check((double)chunkLengthOfBytes, chunkLengthOfBytes >= INT_MIN || chunkLengthOfBytes <= INT_MAX, "MSMF: The chunkLengthOfBytes is out of the allowed range"); + copy(bufferAudioData.begin(), bufferAudioData.begin() + (int)chunkLengthOfBytes, std::back_inserter(audioDataInUse)); + bufferAudioData.erase(bufferAudioData.begin(), bufferAudioData.begin() + (int)chunkLengthOfBytes); + if (audioFrame.empty()) + { + switch (outputAudioFormat) + { + case CV_8S: + cv::Mat((int)chunkLengthOfBytes/(captureAudioFormat.nChannels), captureAudioFormat.nChannels, CV_8S, audioDataInUse.data()).copyTo(audioFrame); + break; + case CV_16S: + cv::Mat((int)chunkLengthOfBytes/(2*captureAudioFormat.nChannels), captureAudioFormat.nChannels, CV_16S, audioDataInUse.data()).copyTo(audioFrame); + break; + case CV_32S: + cv::Mat((int)chunkLengthOfBytes/(4*captureAudioFormat.nChannels), captureAudioFormat.nChannels, CV_32S, audioDataInUse.data()).copyTo(audioFrame); + break; + case CV_32F: + cv::Mat((int)chunkLengthOfBytes/(4*captureAudioFormat.nChannels), captureAudioFormat.nChannels, CV_32F, audioDataInUse.data()).copyTo(audioFrame); + break; + default: + break; + } + } + audioDataInUse.clear(); + audioDataInUse.shrink_to_fit(); + + return true; + } + else + { + return false; + } +} + bool CvCapture_MSMF::grabAudioFrame() { DWORD streamIndex, flags; @@ -1500,6 +1587,8 @@ bool CvCapture_MSMF::grabAudioFrame() aEOS = true; if (videoStream != -1 && !vEOS) returnFlag = true; + if (videoStream == -1) + audioSamplePos += chunkLengthOfBytes/((captureAudioFormat.bit_per_sample/8)*captureAudioFormat.nChannels); CV_LOG_DEBUG(NULL, "videoio(MSMF): End of audio stream detected"); break; } @@ -1538,81 +1627,7 @@ bool CvCapture_MSMF::grabAudioFrame() } } - if (!audioSamples.empty() || !bufferAudioData.empty() && aEOS) - { - _ComPtr buf = NULL; - std::vector audioDataInUse; - BYTE* ptr = NULL; - DWORD maxsize = 0, cursize = 0; - CV_TRACE_REGION("get_contiguous_buffer"); - for (auto item : audioSamples) - { - if (!SUCCEEDED(item->ConvertToContiguousBuffer(&buf))) - { - CV_TRACE_REGION("get_buffer"); - DWORD bcnt = 0; - if (!SUCCEEDED(item->GetBufferCount(&bcnt))) - break; - if (bcnt == 0) - break; - if (!SUCCEEDED(item->GetBufferByIndex(0, &buf))) - break; - } - if (!SUCCEEDED(buf->Lock(&ptr, &maxsize, &cursize))) - break; - size_t lastSize = bufferAudioData.size(); - bufferAudioData.resize(lastSize+cursize); - for (unsigned int i = 0; i < cursize; i++) - { - bufferAudioData[lastSize+i]=*(ptr+i); - } - CV_TRACE_REGION_NEXT("unlock"); - buf->Unlock(); - buf = NULL; - } - audioSamples.clear(); - - audioSamplePos += chunkLengthOfBytes/((captureAudioFormat.bit_per_sample/8)*captureAudioFormat.nChannels); - chunkLengthOfBytes = (videoStream != -1) ? (LONGLONG)((requiredAudioTime*captureAudioFormat.nSamplesPerSec*captureAudioFormat.nChannels*(captureAudioFormat.bit_per_sample)/8)/1e7) : cursize; - if ((videoStream != -1) && (chunkLengthOfBytes % ((int)(captureAudioFormat.bit_per_sample)/8* (int)captureAudioFormat.nChannels) != 0)) - { - if ( (double)audioSamplePos/captureAudioFormat.nSamplesPerSec + audioStartOffset * 1e-7 - usedVideoSampleTime * 1e-7 >= 0 ) - chunkLengthOfBytes -= numberOfAdditionalAudioBytes; - numberOfAdditionalAudioBytes = ((int)(captureAudioFormat.bit_per_sample)/8* (int)captureAudioFormat.nChannels) - - chunkLengthOfBytes % ((int)(captureAudioFormat.bit_per_sample)/8* (int)captureAudioFormat.nChannels); - chunkLengthOfBytes += numberOfAdditionalAudioBytes; - } - if (lastFrame && !syncLastFrame|| aEOS && !vEOS) - { - chunkLengthOfBytes = bufferAudioData.size(); - } - CV_Check((double)chunkLengthOfBytes, chunkLengthOfBytes >= INT_MIN || chunkLengthOfBytes <= INT_MAX, "MSMF: The chunkLengthOfBytes is out of the allowed range"); - copy(bufferAudioData.begin(), bufferAudioData.begin() + (int)chunkLengthOfBytes, std::back_inserter(audioDataInUse)); - bufferAudioData.erase(bufferAudioData.begin(), bufferAudioData.begin() + (int)chunkLengthOfBytes); - if (audioFrame.empty()) - { - switch (outputAudioFormat) - { - case CV_8S: - cv::Mat((int)chunkLengthOfBytes/(captureAudioFormat.nChannels), captureAudioFormat.nChannels, CV_8S, audioDataInUse.data()).copyTo(audioFrame); - break; - case CV_16S: - cv::Mat((int)chunkLengthOfBytes/(2*captureAudioFormat.nChannels), captureAudioFormat.nChannels, CV_16S, audioDataInUse.data()).copyTo(audioFrame); - break; - case CV_32S: - cv::Mat((int)chunkLengthOfBytes/(4*captureAudioFormat.nChannels), captureAudioFormat.nChannels, CV_32S, audioDataInUse.data()).copyTo(audioFrame); - break; - case CV_32F: - cv::Mat((int)chunkLengthOfBytes/(4*captureAudioFormat.nChannels), captureAudioFormat.nChannels, CV_32F, audioDataInUse.data()).copyTo(audioFrame); - break; - default: - break; - } - } - audioDataInUse.clear(); - audioDataInUse.shrink_to_fit(); - } - + returnFlag &= configureAudioFrame(); return returnFlag; } @@ -1662,6 +1677,9 @@ bool CvCapture_MSMF::grabFrame() } if (videoStream != -1) usedVideoSampleTime = reader->m_lastSampleTimestamp; + if (audioStream != -1) + return configureAudioFrame(); + return true; } else if (isOpen) diff --git a/modules/videoio/test/test_audio.cpp b/modules/videoio/test/test_audio.cpp index 3ff51e2613..0b637aeabd 100644 --- a/modules/videoio/test/test_audio.cpp +++ b/modules/videoio/test/test_audio.cpp @@ -270,4 +270,15 @@ TEST_P(Media, audio) INSTANTIATE_TEST_CASE_P(/**/, Media, testing::ValuesIn(mediaParams)); +TEST(AudioOpenCheck, bad_arg_invalid_audio_stream) +{ + std::string fileName = "audio/test_audio.mp4"; + std::vector params { CAP_PROP_AUDIO_STREAM, 1, + CAP_PROP_VIDEO_STREAM, 0, + CAP_PROP_AUDIO_DATA_DEPTH, CV_16S }; + VideoCapture cap; + cap.open(findDataFile(fileName), cv::CAP_MSMF, params); + ASSERT_FALSE(cap.isOpened()); +} + }} //namespace From af154e30539988bf658a31b555a48669db1e0890 Mon Sep 17 00:00:00 2001 From: Harvey Date: Wed, 27 Oct 2021 13:52:54 +0800 Subject: [PATCH 028/226] fixed bug: opencv read tif file convert Palette color image as grayscale image --- modules/imgcodecs/src/grfmt_tiff.cpp | 7 ++++++- modules/imgcodecs/test/test_tiff.cpp | 10 ++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/modules/imgcodecs/src/grfmt_tiff.cpp b/modules/imgcodecs/src/grfmt_tiff.cpp index a46ed3ac90..5e7523b203 100644 --- a/modules/imgcodecs/src/grfmt_tiff.cpp +++ b/modules/imgcodecs/src/grfmt_tiff.cpp @@ -303,7 +303,12 @@ bool TiffDecoder::readHeader() result = true; break; case 8: - m_type = CV_MAKETYPE(CV_8U, !isGrayScale ? wanted_channels : 1); + //Palette color, the value of the component is used as an index into the red, + //green and blue curves in the ColorMap field to retrieve an RGB triplet that defines the color. + if(photometric == PHOTOMETRIC_PALETTE) + m_type = CV_MAKETYPE(CV_8U, 3); + else + m_type = CV_MAKETYPE(CV_8U, !isGrayScale ? wanted_channels : 1); result = true; break; case 16: diff --git a/modules/imgcodecs/test/test_tiff.cpp b/modules/imgcodecs/test/test_tiff.cpp index dec38014aa..a2f9655c73 100644 --- a/modules/imgcodecs/test/test_tiff.cpp +++ b/modules/imgcodecs/test/test_tiff.cpp @@ -219,6 +219,16 @@ TEST(Imgcodecs_Tiff, readWrite_32FC3_RAW) EXPECT_EQ(0, remove(filenameOutput.c_str())); } +TEST(Imgcodecs_Tiff, read_palette_color_image) +{ + const string root = cvtest::TS::ptr()->get_data_path(); + const string filenameInput = root + "readwrite/test_palette_color_image.tif"; + + const Mat img = cv::imread(filenameInput, IMREAD_UNCHANGED); + ASSERT_FALSE(img.empty()); + ASSERT_EQ(CV_8UC3, img.type()); +} + //================================================================================================== From 9dadc06e64ef5e08fd3b403cf4c55e36345f9315 Mon Sep 17 00:00:00 2001 From: shengyu Date: Wed, 27 Oct 2021 20:19:05 +0800 Subject: [PATCH 029/226] remove redundant semicolons --- modules/gapi/include/opencv2/gapi/fluid/gfluidbuffer.hpp | 2 +- modules/gapi/include/opencv2/gapi/gtransform.hpp | 2 +- modules/gapi/include/opencv2/gapi/plaidml/gplaidmlkernel.hpp | 2 +- .../opencv2/gapi/streaming/onevpl/data_provider_interface.hpp | 4 ++-- modules/gapi/include/opencv2/gapi/util/optional.hpp | 2 +- modules/gapi/src/compiler/gislandmodel.hpp | 2 +- modules/gapi/src/compiler/transactions.hpp | 4 ++-- modules/gapi/src/executor/gtbbexecutor.hpp | 4 ++-- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/fluid/gfluidbuffer.hpp b/modules/gapi/include/opencv2/gapi/fluid/gfluidbuffer.hpp index daa9d4153e..551f0a398f 100644 --- a/modules/gapi/include/opencv2/gapi/fluid/gfluidbuffer.hpp +++ b/modules/gapi/include/opencv2/gapi/fluid/gfluidbuffer.hpp @@ -25,7 +25,7 @@ namespace fluid { struct Border { // This constructor is required to support existing kernels which are part of G-API - Border(int _type, cv::Scalar _val) : type(_type), value(_val) {}; + Border(int _type, cv::Scalar _val) : type(_type), value(_val) {} int type; cv::Scalar value; diff --git a/modules/gapi/include/opencv2/gapi/gtransform.hpp b/modules/gapi/include/opencv2/gapi/gtransform.hpp index 5d1b91b517..109bc87b7f 100644 --- a/modules/gapi/include/opencv2/gapi/gtransform.hpp +++ b/modules/gapi/include/opencv2/gapi/gtransform.hpp @@ -31,7 +31,7 @@ struct GAPI_EXPORTS GTransform F pattern; F substitute; - GTransform(const std::string& d, const F &p, const F &s) : description(d), pattern(p), substitute(s){}; + GTransform(const std::string& d, const F &p, const F &s) : description(d), pattern(p), substitute(s) {} }; namespace detail diff --git a/modules/gapi/include/opencv2/gapi/plaidml/gplaidmlkernel.hpp b/modules/gapi/include/opencv2/gapi/plaidml/gplaidmlkernel.hpp index 7ce00cfa35..e22ecc7211 100644 --- a/modules/gapi/include/opencv2/gapi/plaidml/gplaidmlkernel.hpp +++ b/modules/gapi/include/opencv2/gapi/plaidml/gplaidmlkernel.hpp @@ -59,7 +59,7 @@ public: using F = std::function; GPlaidMLKernel() = default; - explicit GPlaidMLKernel(const F& f) : m_f(f) {}; + explicit GPlaidMLKernel(const F& f) : m_f(f) {} void apply(GPlaidMLContext &ctx) const { diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/data_provider_interface.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/data_provider_interface.hpp index 2c299520f7..ac3444757d 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/onevpl/data_provider_interface.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/data_provider_interface.hpp @@ -17,7 +17,7 @@ namespace wip { namespace onevpl { struct GAPI_EXPORTS DataProviderException : public std::exception { - virtual ~DataProviderException() {}; + virtual ~DataProviderException() {} }; struct GAPI_EXPORTS DataProviderSystemErrorException : public DataProviderException { @@ -42,7 +42,7 @@ private: struct GAPI_EXPORTS IDataProvider { using Ptr = std::shared_ptr; - virtual ~IDataProvider() {}; + virtual ~IDataProvider() {} /** * The function is used by onevpl::GSource to extract binary data stream from @ref IDataProvider diff --git a/modules/gapi/include/opencv2/gapi/util/optional.hpp b/modules/gapi/include/opencv2/gapi/util/optional.hpp index 6c8ceebbda..dca03cadad 100644 --- a/modules/gapi/include/opencv2/gapi/util/optional.hpp +++ b/modules/gapi/include/opencv2/gapi/util/optional.hpp @@ -33,7 +33,7 @@ namespace util // Constructors // NB.: there were issues with Clang 3.8 when =default() was used // instead {} - optional() {}; + optional() {} optional(const optional&) = default; explicit optional(T&&) noexcept; explicit optional(const T&) noexcept; diff --git a/modules/gapi/src/compiler/gislandmodel.hpp b/modules/gapi/src/compiler/gislandmodel.hpp index 2cdd10346c..063504a922 100644 --- a/modules/gapi/src/compiler/gislandmodel.hpp +++ b/modules/gapi/src/compiler/gislandmodel.hpp @@ -140,7 +140,7 @@ public: // FIXME: This thing will likely break stuff once we introduce // "multi-source streaming", a better design needs to be proposed // at that stage. - virtual void handleNewStream() {}; // do nothing here by default + virtual void handleNewStream() {} // do nothing here by default // This method is called for every IslandExecutable when // the stream-based execution is stopped. diff --git a/modules/gapi/src/compiler/transactions.hpp b/modules/gapi/src/compiler/transactions.hpp index bdc1723e19..9092c66291 100644 --- a/modules/gapi/src/compiler/transactions.hpp +++ b/modules/gapi/src/compiler/transactions.hpp @@ -69,8 +69,8 @@ struct ChangeT { struct Base { - virtual void commit (ade::Graph & ) {}; - virtual void rollback(ade::Graph & ) {}; + virtual void commit (ade::Graph & ) {} + virtual void rollback(ade::Graph & ) {} virtual ~Base() = default; }; diff --git a/modules/gapi/src/executor/gtbbexecutor.hpp b/modules/gapi/src/executor/gtbbexecutor.hpp index db3ee5c133..3c2bf1ff98 100644 --- a/modules/gapi/src/executor/gtbbexecutor.hpp +++ b/modules/gapi/src/executor/gtbbexecutor.hpp @@ -83,8 +83,8 @@ struct tile_node { std::vector dependants; - tile_node(decltype(sync_task_body::body)&& f) : task_body(sync_task_body{std::move(f)}) {}; - tile_node(async_tag, decltype(async_task_body::body)&& f) : task_body(async_task_body{std::move(f)}) {}; + tile_node(decltype(sync_task_body::body)&& f) : task_body(sync_task_body{std::move(f)}) {} + tile_node(async_tag, decltype(async_task_body::body)&& f) : task_body(async_task_body{std::move(f)}) {} }; std::ostream& operator<<(std::ostream& o, tile_node const& n); From 244ba1a61a8a514dbd6014766dc17dfde4560487 Mon Sep 17 00:00:00 2001 From: Chengrui Wang <80876977+crywang@users.noreply.github.com> Date: Wed, 27 Oct 2021 20:23:42 +0800 Subject: [PATCH 030/226] Merge pull request #20935 from crywang:dnn_face Fix problems in tutorial and python sample of dnn_face. * Update dnn_face.markdown * Update face_match.py --- doc/tutorials/dnn/dnn_face/dnn_face.markdown | 6 +++--- samples/dnn/face_match.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/tutorials/dnn/dnn_face/dnn_face.markdown b/doc/tutorials/dnn/dnn_face/dnn_face.markdown index e5092b8b92..202be3e0e3 100644 --- a/doc/tutorials/dnn/dnn_face/dnn_face.markdown +++ b/doc/tutorials/dnn/dnn_face/dnn_face.markdown @@ -12,7 +12,7 @@ ## Introduction -In this section, we introduce the DNN-based module for face detection and face recognition. Models can be obtained in [Models](#Models). The usage of `FaceDetectorYN` and `FaceRecognizer` are presented in [Usage](#Usage). +In this section, we introduce the DNN-based module for face detection and face recognition. Models can be obtained in [Models](#Models). The usage of `FaceDetectorYN` and `FaceRecognizerSF` are presented in [Usage](#Usage). ## Models @@ -58,8 +58,8 @@ x1, y1, w, h, x_re, y_re, x_le, y_le, x_nt, y_nt, x_rcm, y_rcm, x_lcm, y_lcm Following Face Detection, run codes below to extract face feature from facial image. ```cpp -// Initialize FaceRecognizer with model path (cv::String) -Ptr faceRecognizer = FaceRecognizer::create(model_path, ""); +// Initialize FaceRecognizerSF with model path (cv::String) +Ptr faceRecognizer = FaceRecognizerSF::create(model_path, ""); // Aligning and cropping facial image through the first face of faces detected by dnn_face::DNNFaceDetector Mat aligned_face; diff --git a/samples/dnn/face_match.py b/samples/dnn/face_match.py index b36c9f6367..916c76abf1 100644 --- a/samples/dnn/face_match.py +++ b/samples/dnn/face_match.py @@ -38,20 +38,20 @@ face1_align = recognizer.alignCrop(img1, face1[1][0]) face2_align = recognizer.alignCrop(img2, face2[1][0]) # Extract features -face1_feature = recognizer.faceFeature(face1_align) -face2_feature = recognizer.faceFeature(face2_align) +face1_feature = recognizer.feature(face1_align) +face2_feature = recognizer.feature(face2_align) # Calculate distance (0: cosine, 1: L2) cosine_similarity_threshold = 0.363 -cosine_score = recognizer.faceMatch(face1_feature, face2_feature, 0) +cosine_score = recognizer.match(face1_feature, face2_feature, 0) msg = 'different identities' if cosine_score >= cosine_similarity_threshold: msg = 'the same identity' print('They have {}. Cosine Similarity: {}, threshold: {} (higher value means higher similarity, max 1.0).'.format(msg, cosine_score, cosine_similarity_threshold)) l2_similarity_threshold = 1.128 -l2_score = recognizer.faceMatch(face1_feature, face2_feature, 1) +l2_score = recognizer.match(face1_feature, face2_feature, 1) msg = 'different identities' if l2_score <= l2_similarity_threshold: msg = 'the same identity' -print('They have {}. NormL2 Distance: {}, threshold: {} (lower value means higher similarity, min 0.0).'.format(msg, l2_score, l2_similarity_threshold)) \ No newline at end of file +print('They have {}. NormL2 Distance: {}, threshold: {} (lower value means higher similarity, min 0.0).'.format(msg, l2_score, l2_similarity_threshold)) From 84a81579ba839c4224d3b1fba806e1d64a475597 Mon Sep 17 00:00:00 2001 From: Noah Stier Date: Wed, 27 Oct 2021 12:01:53 -0700 Subject: [PATCH 031/226] tvl1 cuda optflow optimization --- modules/cudaoptflow/src/tvl1flow.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/modules/cudaoptflow/src/tvl1flow.cpp b/modules/cudaoptflow/src/tvl1flow.cpp index 5f28d4c617..3ea2d365a6 100644 --- a/modules/cudaoptflow/src/tvl1flow.cpp +++ b/modules/cudaoptflow/src/tvl1flow.cpp @@ -162,7 +162,9 @@ namespace GpuMat p32_buf; GpuMat diff_buf; - GpuMat norm_buf; + + GpuMat diff_sum_dev; + Mat diff_sum_host; }; void OpticalFlowDual_TVL1_Impl::calc(InputArray _frame0, InputArray _frame1, InputOutputArray _flow, Stream& stream) @@ -361,8 +363,11 @@ namespace estimateU(I1wx, I1wy, grad, rho_c, p11, p12, p21, p22, p31, p32, u1, u2, u3, diff, l_t, static_cast(theta_), gamma_, calcError, stream); if (calcError) { + cuda::calcSum(diff, diff_sum_dev, cv::noArray(), _stream); + diff_sum_dev.download(diff_sum_host, _stream); _stream.waitForCompletion(); - error = cuda::sum(diff, norm_buf)[0]; + + error = diff_sum_host.at(0,0); prevError = error; } else From 5dfe65d53ab3d1ac19a23f04b1fa274306615b65 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 28 Oct 2021 05:20:23 +0000 Subject: [PATCH 032/226] cmake: fix popcnt detection with Intel Compiler --- cmake/OpenCVCompilerOptimizations.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/OpenCVCompilerOptimizations.cmake b/cmake/OpenCVCompilerOptimizations.cmake index 970dd28903..058443821a 100644 --- a/cmake/OpenCVCompilerOptimizations.cmake +++ b/cmake/OpenCVCompilerOptimizations.cmake @@ -238,7 +238,7 @@ if(X86 OR X86_64) ocv_intel_compiler_optimization_option(FP16 "-mavx" "/arch:AVX") ocv_intel_compiler_optimization_option(AVX "-mavx" "/arch:AVX") ocv_intel_compiler_optimization_option(FMA3 "" "") - ocv_intel_compiler_optimization_option(POPCNT "" "") + ocv_intel_compiler_optimization_option(POPCNT "-mpopcnt" "") # -mpopcnt is available since ICC 19.0.0 ocv_intel_compiler_optimization_option(SSE4_2 "-msse4.2" "/arch:SSE4.2") ocv_intel_compiler_optimization_option(SSE4_1 "-msse4.1" "/arch:SSE4.1") ocv_intel_compiler_optimization_option(SSE3 "-msse3" "/arch:SSE3") From 1726bb6c0decddb0fd6f6e2fc70777fcd2c59e68 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 28 Oct 2021 05:49:05 +0000 Subject: [PATCH 033/226] build(icc): fix nodiscard attribute handling --- modules/core/include/opencv2/core/cvdef.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/core/include/opencv2/core/cvdef.h b/modules/core/include/opencv2/core/cvdef.h index 6011b2a931..c2cdcad075 100644 --- a/modules/core/include/opencv2/core/cvdef.h +++ b/modules/core/include/opencv2/core/cvdef.h @@ -589,6 +589,8 @@ Cv64suf; # elif __cplusplus >= 201703L // available when compiler is C++17 compliant # define CV_NODISCARD_STD [[nodiscard]] +# elif defined(__INTEL_COMPILER) + // see above, available when C++17 is enabled # elif defined(_MSC_VER) && _MSC_VER >= 1911 && _MSVC_LANG >= 201703L // available with VS2017 v15.3+ with /std:c++17 or higher; works on functions and classes # define CV_NODISCARD_STD [[nodiscard]] From 75e2ba5af3adc26b08c60a81fbd6931ced0977b3 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 28 Oct 2021 11:25:00 +0000 Subject: [PATCH 034/226] core(simd): fix compilation with MSVC-Clang --- modules/core/include/opencv2/core/hal/intrin_sse.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core/include/opencv2/core/hal/intrin_sse.hpp b/modules/core/include/opencv2/core/hal/intrin_sse.hpp index 2244717e19..443ee16097 100644 --- a/modules/core/include/opencv2/core/hal/intrin_sse.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_sse.hpp @@ -244,7 +244,7 @@ struct v_uint64x2 explicit v_uint64x2(__m128i v) : val(v) {} v_uint64x2(uint64 v0, uint64 v1) { -#if defined(_MSC_VER) && _MSC_VER >= 1920/*MSVS 2019*/ && defined(_M_X64) +#if defined(_MSC_VER) && _MSC_VER >= 1920/*MSVS 2019*/ && defined(_M_X64) && !defined(__clang__) val = _mm_setr_epi64x((int64_t)v0, (int64_t)v1); #elif defined(__GNUC__) val = _mm_setr_epi64((__m64)v0, (__m64)v1); @@ -278,7 +278,7 @@ struct v_int64x2 explicit v_int64x2(__m128i v) : val(v) {} v_int64x2(int64 v0, int64 v1) { -#if defined(_MSC_VER) && _MSC_VER >= 1920/*MSVS 2019*/ && defined(_M_X64) +#if defined(_MSC_VER) && _MSC_VER >= 1920/*MSVS 2019*/ && defined(_M_X64) && !defined(__clang__) val = _mm_setr_epi64x((int64_t)v0, (int64_t)v1); #elif defined(__GNUC__) val = _mm_setr_epi64((__m64)v0, (__m64)v1); From eb152d7431462579efd6873e94eae7c20453611f Mon Sep 17 00:00:00 2001 From: Maxim Pashchenkov Date: Thu, 28 Oct 2021 21:19:46 +0300 Subject: [PATCH 035/226] Merge pull request #20937 from mpashchenkov:mp/ocv-gapi-warnings G-API: Disable Windows warnings with 4996 code * Windows warnings 4503 and 4996 are disabled with dnn style * Applying comments to review * Reproducing * Added check MSVC_VERSION for both warnings --- modules/gapi/CMakeLists.txt | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 61ab5397d7..185f5c6639 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -29,12 +29,11 @@ ocv_add_module(gapi ) if(MSVC) - # Disable obsollete warning C4503 popping up on MSVC <<2017 - # https://docs.microsoft.com/en-us/cpp/error-messages/compiler-warnings/compiler-warning-level-1-c4503?view=vs-2019 - ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4503) - if (OPENCV_GAPI_INF_ENGINE AND NOT INF_ENGINE_RELEASE VERSION_GREATER "2021000000") - # Disable IE deprecated code warning C4996 for releases < 2021.1 - ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4996) + if(MSVC_VERSION LESS 1910) + # Disable obsolete warning C4503 popping up on MSVC << 15 2017 + # https://docs.microsoft.com/en-us/cpp/error-messages/compiler-warnings/compiler-warning-level-1-c4503?view=vs-2019 + # and IE deprecated code warning C4996 + ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4503 /wd4996) endif() endif() From d612c72405c69becc0b0b4af3a0942ea9ee86279 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 28 Oct 2021 21:08:36 +0000 Subject: [PATCH 036/226] build: fix MSVC-Clang warnings about unused parameters in stubs --- modules/core/src/hal_replacement.hpp | 25 +++++++++++++--------- modules/features2d/src/hal_replacement.hpp | 25 +++++++++++++--------- modules/imgproc/src/hal_replacement.hpp | 25 +++++++++++++--------- 3 files changed, 45 insertions(+), 30 deletions(-) diff --git a/modules/core/src/hal_replacement.hpp b/modules/core/src/hal_replacement.hpp index 1a558f2532..6ed795b5e1 100644 --- a/modules/core/src/hal_replacement.hpp +++ b/modules/core/src/hal_replacement.hpp @@ -47,12 +47,15 @@ #include "opencv2/core/hal/interface.h" -#if defined __GNUC__ -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wunused-parameter" -#elif defined _MSC_VER -# pragma warning( push ) -# pragma warning( disable: 4100 ) +#if defined(__clang__) // clang or MSVC clang +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunused-parameter" +#elif defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4100) +#elif defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" #endif //! @addtogroup core_hal_interface @@ -731,10 +734,12 @@ inline int hal_ni_minMaxIdx(const uchar* src_data, size_t src_step, int width, i //! @} -#if defined __GNUC__ -# pragma GCC diagnostic pop -#elif defined _MSC_VER -# pragma warning( pop ) +#if defined(__clang__) +#pragma clang diagnostic pop +#elif defined(_MSC_VER) +#pragma warning(pop) +#elif defined(__GNUC__) +#pragma GCC diagnostic pop #endif #include "hal_internal.hpp" diff --git a/modules/features2d/src/hal_replacement.hpp b/modules/features2d/src/hal_replacement.hpp index 7780aa4a24..977cef1e32 100644 --- a/modules/features2d/src/hal_replacement.hpp +++ b/modules/features2d/src/hal_replacement.hpp @@ -44,12 +44,15 @@ #include "opencv2/core/hal/interface.h" -#if defined __GNUC__ -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wunused-parameter" -#elif defined _MSC_VER -# pragma warning( push ) -# pragma warning( disable: 4100 ) +#if defined(__clang__) // clang or MSVC clang +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunused-parameter" +#elif defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4100) +#elif defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" #endif //! @addtogroup features2d_hal_interface @@ -103,10 +106,12 @@ inline int hal_ni_FAST(const uchar* src_data, size_t src_step, int width, int he //! @} -#if defined __GNUC__ -# pragma GCC diagnostic pop -#elif defined _MSC_VER -# pragma warning( pop ) +#if defined(__clang__) +#pragma clang diagnostic pop +#elif defined(_MSC_VER) +#pragma warning(pop) +#elif defined(__GNUC__) +#pragma GCC diagnostic pop #endif #include "custom_hal.hpp" diff --git a/modules/imgproc/src/hal_replacement.hpp b/modules/imgproc/src/hal_replacement.hpp index 3368093c56..d9ef7530f9 100644 --- a/modules/imgproc/src/hal_replacement.hpp +++ b/modules/imgproc/src/hal_replacement.hpp @@ -48,12 +48,15 @@ #include "opencv2/core/hal/interface.h" #include "opencv2/imgproc/hal/interface.h" -#if defined __GNUC__ -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wunused-parameter" -#elif defined _MSC_VER -# pragma warning( push ) -# pragma warning( disable: 4100 ) +#if defined(__clang__) // clang or MSVC clang +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunused-parameter" +#elif defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4100) +#elif defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" #endif //! @addtogroup imgproc_hal_interface @@ -809,10 +812,12 @@ inline int hal_ni_canny(const uchar* src_data, size_t src_step, uchar* dst_data, //! @} -#if defined __GNUC__ -# pragma GCC diagnostic pop -#elif defined _MSC_VER -# pragma warning( pop ) +#if defined(__clang__) +#pragma clang diagnostic pop +#elif defined(_MSC_VER) +#pragma warning(pop) +#elif defined(__GNUC__) +#pragma GCC diagnostic pop #endif #include "custom_hal.hpp" From a49cda65230c0094148fc7dddd70eac2c3ed418e Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 28 Oct 2021 21:32:47 +0000 Subject: [PATCH 037/226] core: eliminate Winvalid-noreturn in base.hpp --- modules/core/include/opencv2/core/base.hpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/core/include/opencv2/core/base.hpp b/modules/core/include/opencv2/core/base.hpp index 12504974d9..19d496080c 100644 --- a/modules/core/include/opencv2/core/base.hpp +++ b/modules/core/include/opencv2/core/base.hpp @@ -297,7 +297,10 @@ It is possible to alternate error processing by using redirectError(). */ CV_EXPORTS void error(int _code, const String& _err, const char* _func, const char* _file, int _line); -#ifdef __GNUC__ +#if defined(__clang__) && defined(_MSC_VER) // MSVC-Clang +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Winvalid-noreturn" +#elif defined(__GNUC__) # if defined __clang__ || defined __APPLE__ # pragma GCC diagnostic push # pragma GCC diagnostic ignored "-Winvalid-noreturn" @@ -316,7 +319,10 @@ CV_INLINE CV_NORETURN void errorNoReturn(int _code, const String& _err, const ch # endif #endif } -#ifdef __GNUC__ + +#if defined(__clang__) && defined(_MSC_VER) // MSVC-Clang +# pragma clang diagnostic pop +#elif defined(__GNUC__) # if defined __clang__ || defined __APPLE__ # pragma GCC diagnostic pop # endif From 9637cf05741e022f83549f5d7284d45c6a68cf71 Mon Sep 17 00:00:00 2001 From: Ihsan Soydemir Date: Fri, 29 Oct 2021 13:30:51 +0300 Subject: [PATCH 038/226] Correct drive links for DB_IC15 and DB_TD500 --- .../dnn/dnn_text_spotting/dnn_text_spotting.markdown | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown b/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown index 5c465941ca..0a22f272b5 100644 --- a/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown +++ b/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown @@ -102,7 +102,7 @@ recommended parameter setting: -inputHeight=736, -inputWidth=1280; description: This model is trained on ICDAR2015, so it can only detect English text instances. - DB_IC15_resnet18.onnx: -url: https://drive.google.com/uc?export=dowload&id=1sZszH3pEt8hliyBlTmB-iulxHP1dCQWV +url: https://drive.google.com/uc?export=dowload&id=1vY_KsDZZZb_svd5RT6pjyI8BS1nPbBSX sha: 19543ce09b2efd35f49705c235cc46d0e22df30b recommended parameter setting: -inputHeight=736, -inputWidth=1280; description: This model is trained on ICDAR2015, so it can only detect English text instances. @@ -114,7 +114,7 @@ recommended parameter setting: -inputHeight=736, -inputWidth=736; description: This model is trained on MSRA-TD500, so it can detect both English and Chinese text instances. - DB_TD500_resnet18.onnx: -url: https://drive.google.com/uc?export=dowload&id=1vY_KsDZZZb_svd5RT6pjyI8BS1nPbBSX +url: https://drive.google.com/uc?export=dowload&id=1sZszH3pEt8hliyBlTmB-iulxHP1dCQWV sha: 8a3700bdc13e00336a815fc7afff5dcc1ce08546 recommended parameter setting: -inputHeight=736, -inputWidth=736; description: This model is trained on MSRA-TD500, so it can detect both English and Chinese text instances. From e5647cf70d5d207b2147163a5ba842dfab7f3641 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 29 Oct 2021 02:02:32 +0300 Subject: [PATCH 039/226] cmake: use CMAKE_BUILD_TYPE=Release by default --- CMakeLists.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d9f98bf23e..b660cdd33b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -98,6 +98,12 @@ ocv_cmake_hook(CMAKE_INIT) # must go before the project()/enable_language() commands ocv_update(CMAKE_CONFIGURATION_TYPES "Debug;Release" CACHE STRING "Configs" FORCE) +if(NOT DEFINED CMAKE_BUILD_TYPE + AND NOT OPENCV_SKIP_DEFAULT_BUILD_TYPE +) + message(STATUS "'Release' build type is used by default. Use CMAKE_BUILD_TYPE to specify build type (Release or Debug)") + set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build") +endif() if(DEFINED CMAKE_BUILD_TYPE) set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "${CMAKE_CONFIGURATION_TYPES}") endif() @@ -599,10 +605,6 @@ endif() # ---------------------------------------------------------------------------- # OpenCV compiler and linker options # ---------------------------------------------------------------------------- -# In case of Makefiles if the user does not setup CMAKE_BUILD_TYPE, assume it's Release: -if(CMAKE_GENERATOR MATCHES "Makefiles|Ninja" AND "${CMAKE_BUILD_TYPE}" STREQUAL "") - set(CMAKE_BUILD_TYPE Release) -endif() ocv_cmake_hook(POST_CMAKE_BUILD_OPTIONS) From 6a73e5a7207393adfc93c0084de3ace15d772df2 Mon Sep 17 00:00:00 2001 From: Trutnev Aleksei Date: Fri, 29 Oct 2021 19:30:35 +0300 Subject: [PATCH 040/226] Merge pull request #20922 from alexgiving:atrutnev/align_expect_assert_macros GAPI: Align EXPECT/ASSERT macros * Align TEST macros * restart CI * Fix ASSERT_GT in gapi_async_test --- .../gapi/test/common/gapi_core_tests_inl.hpp | 56 ++++++------- .../test/common/gapi_imgproc_tests_inl.hpp | 80 +++++++++---------- .../test/common/gapi_operators_tests_inl.hpp | 6 +- modules/gapi/test/gapi_opaque_tests.cpp | 16 ++-- .../gapi/test/infer/gapi_infer_onnx_test.cpp | 2 +- .../test/internal/gapi_int_gmetaarg_test.cpp | 2 +- modules/gapi/test/own/conc_queue_tests.cpp | 2 +- .../test/own/last_written_value_tests.cpp | 2 +- modules/gapi/test/own/mat_tests.cpp | 78 +++++++++--------- modules/gapi/test/own/scalar_tests.cpp | 18 ++--- .../test/streaming/gapi_streaming_tests.cpp | 4 +- .../gapi_streaming_vpl_core_test.cpp | 70 ++++++++-------- modules/gapi/test/util/any_tests.cpp | 30 +++---- modules/gapi/test/util/variant_tests.cpp | 42 +++++----- 14 files changed, 204 insertions(+), 204 deletions(-) diff --git a/modules/gapi/test/common/gapi_core_tests_inl.hpp b/modules/gapi/test/common/gapi_core_tests_inl.hpp index d9287a176c..09b61e2876 100644 --- a/modules/gapi/test/common/gapi_core_tests_inl.hpp +++ b/modules/gapi/test/common/gapi_core_tests_inl.hpp @@ -138,7 +138,7 @@ TEST_P(MathOpTest, MatricesAccuracyTest) #else EXPECT_EQ(0, cvtest::norm(out_mat_gapi, out_mat_ocv, NORM_INF)); #endif - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -173,7 +173,7 @@ TEST_P(MulDoubleTest, AccuracyTest) #else EXPECT_EQ(0, cvtest::norm(out_mat_gapi, out_mat_ocv, NORM_INF)); #endif - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } TEST_P(DivTest, DISABLED_DivByZeroTest) // https://github.com/opencv/opencv/pull/12826 @@ -195,7 +195,7 @@ TEST_P(DivTest, DISABLED_DivByZeroTest) // https://github.com/opencv/opencv/pul // Comparison ////////////////////////////////////////////////////////////// { EXPECT_EQ(0, cvtest::norm(out_mat_gapi, out_mat_ocv, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -304,7 +304,7 @@ TEST_P(Polar2CartTest, AccuracyTest) // // TODO: Make threshold a configurable parameter of this test (ADE-221) - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); cv::Mat &outx = out_mat_gapi, &outy = out_mat2; @@ -347,7 +347,7 @@ TEST_P(Cart2PolarTest, AccuracyTest) // // TODO: Make threshold a configurable parameter of this test (ADE-221) - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); cv::Mat &outm = out_mat_gapi, &outa = out_mat2; @@ -406,7 +406,7 @@ TEST_P(CmpTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); } } @@ -464,7 +464,7 @@ TEST_P(BitwiseTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); EXPECT_EQ(0, cvtest::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); } } @@ -484,7 +484,7 @@ TEST_P(NotTest, AccuracyTest) } // Comparison ////////////////////////////////////////////////////////////// { - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); EXPECT_EQ(0, cvtest::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); } } @@ -508,7 +508,7 @@ TEST_P(SelectTest, AccuracyTest) } // Comparison ////////////////////////////////////////////////////////////// { - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); EXPECT_EQ(0, cvtest::norm(out_mat_gapi, out_mat_ocv, NORM_INF)); } } @@ -528,7 +528,7 @@ TEST_P(MinTest, AccuracyTest) } // Comparison ////////////////////////////////////////////////////////////// { - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); EXPECT_EQ(0, cvtest::norm(out_mat_gapi, out_mat_ocv, NORM_INF)); } } @@ -548,7 +548,7 @@ TEST_P(MaxTest, AccuracyTest) } // Comparison ////////////////////////////////////////////////////////////// { - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); EXPECT_EQ(0, cvtest::norm(out_mat_gapi, out_mat_ocv, NORM_INF)); } } @@ -568,7 +568,7 @@ TEST_P(AbsDiffTest, AccuracyTest) } // Comparison ////////////////////////////////////////////////////////////// { - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); EXPECT_EQ(0, cvtest::norm(out_mat_gapi, out_mat_ocv, NORM_INF)); } } @@ -589,7 +589,7 @@ TEST_P(AbsDiffCTest, AccuracyTest) } // Comparison ////////////////////////////////////////////////////////////// { - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); EXPECT_EQ(0, cvtest::norm(out_mat_gapi, out_mat_ocv, NORM_INF)); } } @@ -659,7 +659,7 @@ TEST_P(AddWeightedTest, AccuracyTest) } // Comparison ////////////////////////////////////////////////////////////// EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } TEST_P(NormTest, AccuracyTest) @@ -738,7 +738,7 @@ TEST_P(ThresholdTest, AccuracyTestBinary) } // Comparison ////////////////////////////////////////////////////////////// { - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); EXPECT_EQ(0, cv::norm(out_mat_ocv, out_mat_gapi, NORM_L1)); } } @@ -764,7 +764,7 @@ TEST_P(ThresholdOTTest, AccuracyTestOtsu) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_EQ(0, cvtest::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); EXPECT_EQ(ocv_res, out_gapi_scalar.val[0]); } } @@ -788,7 +788,7 @@ TEST_P(InRangeTest, AccuracyTest) } // Comparison ////////////////////////////////////////////////////////////// { - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); EXPECT_EQ(0, cvtest::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); } } @@ -992,7 +992,7 @@ TEST_P(RemapTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_EQ(0, cvtest::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -1011,7 +1011,7 @@ TEST_P(FlipTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_EQ(0, cvtest::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -1037,7 +1037,7 @@ TEST_P(CropTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_EQ(0, cvtest::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz_out); + EXPECT_EQ(sz_out, out_mat_gapi.size()); } } @@ -1063,7 +1063,7 @@ TEST_P(CopyTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_EQ(0, cvtest::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz_out); + EXPECT_EQ(sz_out, out_mat_gapi.size()); } } @@ -1245,7 +1245,7 @@ TEST_P(LUTTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_EQ(0, cvtest::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -1269,7 +1269,7 @@ TEST_P(ConvertToTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -1378,7 +1378,7 @@ TEST_P(NormalizeTest, Test) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -1654,7 +1654,7 @@ TEST_P(ReInitOutTest, TestWithAdd) // Comparison ////////////////////////////////////////////////////////////// EXPECT_EQ(0, cvtest::norm(out_mat_gapi, out_mat_ocv, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); }; // run for uninitialized output @@ -1738,7 +1738,7 @@ TEST_P(SizeTest, ParseTest) cv::GComputation c(cv::GIn(in), cv::GOut(out)); c.apply(cv::gin(in_mat1), cv::gout(out_sz), getCompileArgs()); - EXPECT_EQ(out_sz, sz); + EXPECT_EQ(sz, out_sz); } TEST_P(SizeRTest, ParseTest) @@ -1751,7 +1751,7 @@ TEST_P(SizeRTest, ParseTest) cv::GComputation c(cv::GIn(op_rect), cv::GOut(out)); c.apply(cv::gin(rect), cv::gout(out_sz), getCompileArgs()); - EXPECT_EQ(out_sz, sz); + EXPECT_EQ(sz, out_sz); } namespace { @@ -1784,7 +1784,7 @@ TEST_P(SizeMFTest, ParseTest) cv::GComputation c(cv::GIn(in), cv::GOut(out)); c.apply(cv::gin(frame), cv::gout(out_sz), getCompileArgs()); - EXPECT_EQ(out_sz, sz); + EXPECT_EQ(sz, out_sz); } } // opencv_test diff --git a/modules/gapi/test/common/gapi_imgproc_tests_inl.hpp b/modules/gapi/test/common/gapi_imgproc_tests_inl.hpp index 755d09a6e4..6500c7853d 100644 --- a/modules/gapi/test/common/gapi_imgproc_tests_inl.hpp +++ b/modules/gapi/test/common/gapi_imgproc_tests_inl.hpp @@ -92,7 +92,7 @@ TEST_P(Filter2DTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -116,7 +116,7 @@ TEST_P(BoxFilterTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -142,7 +142,7 @@ TEST_P(SepFilterTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -163,7 +163,7 @@ TEST_P(BlurTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -185,7 +185,7 @@ TEST_P(GaussianBlurTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -204,7 +204,7 @@ TEST_P(MedianBlurTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -225,7 +225,7 @@ TEST_P(ErodeTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -246,7 +246,7 @@ TEST_P(Erode3x3Test, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -267,7 +267,7 @@ TEST_P(DilateTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -288,7 +288,7 @@ TEST_P(Dilate3x3Test, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -311,7 +311,7 @@ TEST_P(MorphologyExTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -330,7 +330,7 @@ TEST_P(SobelTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -362,8 +362,8 @@ TEST_P(SobelXYTest, AccuracyTest) { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); EXPECT_TRUE(cmpF(out_mat_gapi2, out_mat_ocv2)); - EXPECT_EQ(out_mat_gapi.size(), sz); - EXPECT_EQ(out_mat_gapi2.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); + EXPECT_EQ(sz, out_mat_gapi2.size()); } } @@ -383,7 +383,7 @@ TEST_P(LaplacianTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -402,7 +402,7 @@ TEST_P(BilateralFilterTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -421,7 +421,7 @@ TEST_P(EqHistTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -440,7 +440,7 @@ TEST_P(CannyTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -611,7 +611,7 @@ TEST_P(BGR2RGBTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -630,7 +630,7 @@ TEST_P(RGB2GrayTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -649,7 +649,7 @@ TEST_P(BGR2GrayTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -668,7 +668,7 @@ TEST_P(RGB2YUVTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -687,7 +687,7 @@ TEST_P(YUV2RGBTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -706,7 +706,7 @@ TEST_P(BGR2I420Test, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), Size(sz.width, sz.height * 3 / 2)); + EXPECT_EQ(Size(sz.width, sz.height * 3 / 2), out_mat_gapi.size()); } } @@ -725,7 +725,7 @@ TEST_P(RGB2I420Test, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), Size(sz.width, sz.height * 3 / 2)); + EXPECT_EQ(Size(sz.width, sz.height * 3 / 2), out_mat_gapi.size()); } } @@ -744,7 +744,7 @@ TEST_P(I4202BGRTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), Size(sz.width, sz.height * 2 / 3)); + EXPECT_EQ(Size(sz.width, sz.height * 2 / 3), out_mat_gapi.size()); } } @@ -763,7 +763,7 @@ TEST_P(I4202RGBTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), Size(sz.width, sz.height * 2 / 3)); + EXPECT_EQ(Size(sz.width, sz.height * 2 / 3), out_mat_gapi.size()); } } @@ -787,7 +787,7 @@ TEST_P(NV12toRGBTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -811,7 +811,7 @@ TEST_P(NV12toBGRTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -839,7 +839,7 @@ TEST_P(NV12toGrayTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -882,7 +882,7 @@ TEST_P(NV12toRGBpTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi_planar, out_mat_ocv_planar)); - EXPECT_EQ(out_mat_gapi_planar.size(), sz_p); + EXPECT_EQ(sz_p, out_mat_gapi_planar.size()); } } @@ -912,7 +912,7 @@ TEST_P(NV12toBGRpTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi_planar, out_mat_ocv_planar)); - EXPECT_EQ(out_mat_gapi_planar.size(), sz_p); + EXPECT_EQ(sz_p, out_mat_gapi_planar.size()); } } @@ -931,7 +931,7 @@ TEST_P(RGB2LabTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -950,7 +950,7 @@ TEST_P(BGR2LUVTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -969,7 +969,7 @@ TEST_P(LUV2BGRTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -988,7 +988,7 @@ TEST_P(BGR2YUVTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -1007,7 +1007,7 @@ TEST_P(YUV2BGRTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -1026,7 +1026,7 @@ TEST_P(RGB2HSVTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -1045,7 +1045,7 @@ TEST_P(BayerGR2RGBTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -1064,7 +1064,7 @@ TEST_P(RGB2YUV422Test, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } } // opencv_test diff --git a/modules/gapi/test/common/gapi_operators_tests_inl.hpp b/modules/gapi/test/common/gapi_operators_tests_inl.hpp index ad579290d5..9f739428e0 100644 --- a/modules/gapi/test/common/gapi_operators_tests_inl.hpp +++ b/modules/gapi/test/common/gapi_operators_tests_inl.hpp @@ -36,7 +36,7 @@ TEST_P(MathOperatorMatScalarTest, OperatorAccuracyTest ) // Comparison ////////////////////////////////////////////////////////////// { - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); } } @@ -63,7 +63,7 @@ TEST_P(MathOperatorMatMatTest, OperatorAccuracyTest ) // Comparison ////////////////////////////////////////////////////////////// { - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); } } @@ -83,7 +83,7 @@ TEST_P(NotOperatorTest, OperatorAccuracyTest) } // Comparison ////////////////////////////////////////////////////////////// { - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); EXPECT_EQ(0, cvtest::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); } } diff --git a/modules/gapi/test/gapi_opaque_tests.cpp b/modules/gapi/test/gapi_opaque_tests.cpp index de3572c4bd..3af0a68909 100644 --- a/modules/gapi/test/gapi_opaque_tests.cpp +++ b/modules/gapi/test/gapi_opaque_tests.cpp @@ -161,7 +161,7 @@ TEST(GOpaque, TestOpaqueBetween) c.apply(cv::gin(mat_in), cv::gout(mat_out), cv::compile_args(cv::gapi::kernels())); int painted = mat_out.at(42, 42); - EXPECT_EQ(painted, 77); + EXPECT_EQ(77, painted); } TEST(GOpaque, TestOpaqueBetweenIslands) @@ -181,7 +181,7 @@ TEST(GOpaque, TestOpaqueBetweenIslands) c.apply(cv::gin(mat_in), cv::gout(mat_out), cv::compile_args(cv::gapi::kernels())); int painted = mat_out.at(42, 42); - EXPECT_EQ(painted, 77); + EXPECT_EQ(77, painted); } TEST(GOpaque, TestOpaqueCustomOut2) @@ -200,11 +200,11 @@ TEST(GOpaque, TestOpaqueCustomOut2) cv::GComputation c(cv::GIn(in1, in2), cv::GOut(std::get<0>(out), std::get<1>(out))); c.apply(cv::gin(input1, input2), cv::gout(out1, out2), cv::compile_args(cv::gapi::kernels())); - EXPECT_EQ(out1.num, input1.size().width * input1.size().height); - EXPECT_EQ(out1.s, str); + EXPECT_EQ(input1.size().width * input1.size().height, out1.num); + EXPECT_EQ(str, out1.s); - EXPECT_EQ(out2.num, input2.size().width * input2.size().height); - EXPECT_EQ(out2.s, str2); + EXPECT_EQ(input2.size().width * input2.size().height, out2.num); + EXPECT_EQ(str2, out2.s); } TEST(GOpaque, TestOpaqueOCLBackendIn) @@ -220,7 +220,7 @@ TEST(GOpaque, TestOpaqueOCLBackendIn) cv::compile_args(cv::gapi::kernels())); int painted = mat_out.at(42, 42); - EXPECT_EQ(painted, 77); + EXPECT_EQ(77, painted); } TEST(GOpaque, TestOpaqueOCLBackendBetween) @@ -240,7 +240,7 @@ TEST(GOpaque, TestOpaqueOCLBackendBetween) cv::compile_args(cv::gapi::kernels())); int painted = mat_out.at(42, 42); - EXPECT_EQ(painted, 77); + EXPECT_EQ(77, painted); } TEST(GOpaque, TestOpaqueOCLBackendOut) diff --git a/modules/gapi/test/infer/gapi_infer_onnx_test.cpp b/modules/gapi/test/infer/gapi_infer_onnx_test.cpp index b1bf9c9356..1f21f95ce8 100644 --- a/modules/gapi/test/infer/gapi_infer_onnx_test.cpp +++ b/modules/gapi/test/infer/gapi_infer_onnx_test.cpp @@ -231,7 +231,7 @@ void remapToIESSDOut(const std::vector &detections, } // SSD-MobilenetV1 structure check - ASSERT_EQ(detections[0].total(), 1u); + ASSERT_EQ(1u, detections[0].total()); ASSERT_EQ(detections[2].total(), detections[0].total() * 100); ASSERT_EQ(detections[2].total(), detections[3].total()); ASSERT_EQ((detections[2].total() * 4), detections[1].total()); diff --git a/modules/gapi/test/internal/gapi_int_gmetaarg_test.cpp b/modules/gapi/test/internal/gapi_int_gmetaarg_test.cpp index 81f1ad4e2d..78048cfa3f 100644 --- a/modules/gapi/test/internal/gapi_int_gmetaarg_test.cpp +++ b/modules/gapi/test/internal/gapi_int_gmetaarg_test.cpp @@ -102,7 +102,7 @@ TEST(GMetaArg, Can_Get_Metas_From_Input_Run_Args) GMatDesc m_desc; GMetaArgs meta_args = descr_of(cv::gin(m, s, v)); - EXPECT_EQ(meta_args.size(), 3u); + EXPECT_EQ(3u, meta_args.size()); EXPECT_NO_THROW(m_desc = util::get(meta_args[0])); EXPECT_NO_THROW(util::get(meta_args[1])); EXPECT_NO_THROW(util::get(meta_args[2])); diff --git a/modules/gapi/test/own/conc_queue_tests.cpp b/modules/gapi/test/own/conc_queue_tests.cpp index 6e268f318c..6234a42719 100644 --- a/modules/gapi/test/own/conc_queue_tests.cpp +++ b/modules/gapi/test/own/conc_queue_tests.cpp @@ -27,7 +27,7 @@ TEST(ConcQueue, PushPop) { int x; q.pop(x); - EXPECT_EQ(x, i); + EXPECT_EQ(i, x); } } diff --git a/modules/gapi/test/own/last_written_value_tests.cpp b/modules/gapi/test/own/last_written_value_tests.cpp index 4bfb27f15f..be9f13e47b 100644 --- a/modules/gapi/test/own/last_written_value_tests.cpp +++ b/modules/gapi/test/own/last_written_value_tests.cpp @@ -21,7 +21,7 @@ TEST(LastValue, PushPop) { int x = 1; v.pop(x); - EXPECT_EQ(x, i); + EXPECT_EQ(i, x); } } diff --git a/modules/gapi/test/own/mat_tests.cpp b/modules/gapi/test/own/mat_tests.cpp index 0ddc201f9f..22da4ef718 100644 --- a/modules/gapi/test/own/mat_tests.cpp +++ b/modules/gapi/test/own/mat_tests.cpp @@ -24,12 +24,12 @@ inline std::size_t multiply_dims(Dims const& dims){ TEST(OwnMat, DefaultConstruction) { Mat m; - ASSERT_EQ(m.data, nullptr); - ASSERT_EQ(m.cols, 0); - ASSERT_EQ(m.rows, 0); - ASSERT_EQ(m.cols, 0); - ASSERT_EQ(m.type(), 0); - ASSERT_EQ(m.depth(), 0); + ASSERT_EQ(nullptr, m.data); + ASSERT_EQ(0, m.cols); + ASSERT_EQ(0, m.rows); + ASSERT_EQ(0, m.cols); + ASSERT_EQ(0, m.type()); + ASSERT_EQ(0, m.depth()); ASSERT_TRUE(m.dims.empty()); ASSERT_TRUE(m.empty()); } @@ -40,15 +40,15 @@ TEST(OwnMat, Create) Mat m; m.create(size, CV_8UC1); - ASSERT_NE(m.data, nullptr); - ASSERT_EQ((cv::gapi::own::Size{m.cols, m.rows}), size); + ASSERT_NE(nullptr, m.data); + ASSERT_EQ(size, (cv::gapi::own::Size{m.cols, m.rows})); - ASSERT_EQ(m.total(), static_cast(size.height) * size.width); - ASSERT_EQ(m.type(), CV_8UC1); - ASSERT_EQ(m.depth(), CV_8U); - ASSERT_EQ(m.channels(), 1); - ASSERT_EQ(m.elemSize(), sizeof(uint8_t)); - ASSERT_EQ(m.step, sizeof(uint8_t) * m.cols); + ASSERT_EQ(static_cast(size.height) * size.width, m.total()); + ASSERT_EQ(CV_8UC1, m.type()); + ASSERT_EQ(CV_8U, m.depth()); + ASSERT_EQ(1, m.channels()); + ASSERT_EQ(sizeof(uint8_t), m.elemSize()); + ASSERT_EQ(sizeof(uint8_t) * m.cols, m.step); ASSERT_TRUE(m.dims.empty()); ASSERT_FALSE(m.empty()); } @@ -59,16 +59,16 @@ TEST(OwnMat, CreateND) Mat m; m.create(dims, CV_32F); - ASSERT_NE(nullptr , m.data ); + ASSERT_NE(nullptr , m.data); ASSERT_EQ((cv::gapi::own::Size{0,0}), (cv::gapi::own::Size{m.cols, m.rows})); ASSERT_EQ(multiply_dims(dims), m.total()); - ASSERT_EQ(CV_32F , m.type() ); - ASSERT_EQ(CV_32F , m.depth() ); - ASSERT_EQ(-1 , m.channels()); - ASSERT_EQ(sizeof(float) , m.elemSize()); - ASSERT_EQ(0u , m.step ); - ASSERT_EQ(dims , m.dims ); + ASSERT_EQ(CV_32F, m.type()); + ASSERT_EQ(CV_32F, m.depth()); + ASSERT_EQ(-1, m.channels()); + ASSERT_EQ(sizeof(float), m.elemSize()); + ASSERT_EQ(0u, m.step); + ASSERT_EQ(dims, m.dims); ASSERT_FALSE(m.empty()); } @@ -78,15 +78,15 @@ TEST(OwnMat, CreateOverload) Mat m; m.create(size.height,size.width, CV_8UC1); - ASSERT_NE(m.data, nullptr); - ASSERT_EQ((cv::Size{m.cols, m.rows}), size); + ASSERT_NE(nullptr, m.data); + ASSERT_EQ(size, (cv::Size{m.cols, m.rows})); - ASSERT_EQ(m.total(), static_cast(size.height) * size.width); - ASSERT_EQ(m.type(), CV_8UC1); - ASSERT_EQ(m.depth(), CV_8U); - ASSERT_EQ(m.channels(), 1); - ASSERT_EQ(m.elemSize(), sizeof(uint8_t)); - ASSERT_EQ(m.step, sizeof(uint8_t) * m.cols); + ASSERT_EQ(static_cast(size.height) * size.width, m.total()); + ASSERT_EQ(CV_8UC1, m.type()); + ASSERT_EQ(CV_8U, m.depth()); + ASSERT_EQ(1, m.channels()); + ASSERT_EQ(sizeof(uint8_t), m.elemSize()); + ASSERT_EQ(sizeof(uint8_t) * m.cols, m.step); ASSERT_TRUE(m.dims.empty()); ASSERT_FALSE(m.empty()); } @@ -97,14 +97,14 @@ TEST(OwnMat, Create3chan) Mat m; m.create(size, CV_8UC3); - ASSERT_NE(m.data, nullptr); - ASSERT_EQ((cv::Size{m.cols, m.rows}), size); + ASSERT_NE(nullptr, m.data); + ASSERT_EQ(size, (cv::Size{m.cols, m.rows})); - ASSERT_EQ(m.type(), CV_8UC3); - ASSERT_EQ(m.depth(), CV_8U); - ASSERT_EQ(m.channels(), 3); - ASSERT_EQ(m.elemSize(), 3 * sizeof(uint8_t)); - ASSERT_EQ(m.step, 3* sizeof(uint8_t) * m.cols); + ASSERT_EQ(CV_8UC3, m.type()); + ASSERT_EQ(CV_8U, m.depth()); + ASSERT_EQ(3, m.channels()); + ASSERT_EQ(3 * sizeof(uint8_t), m.elemSize()); + ASSERT_EQ(3 * sizeof(uint8_t) * m.cols, m.step); ASSERT_TRUE(m.dims.empty()); ASSERT_FALSE(m.empty()); } @@ -134,7 +134,7 @@ namespace { }; void ensure_mats_are_same(Mat const& copy, Mat const& m){ - EXPECT_NE(copy.data, nullptr); + EXPECT_NE(nullptr, copy.data); EXPECT_EQ(state_of(copy), state_of(m)); } } @@ -157,8 +157,8 @@ struct OwnMatMoveSemantics : NonEmptyMat, ::testing::Test { void ensure_state_moved_to(Mat const& moved_to) { - EXPECT_EQ(state_of(moved_to), initial_state); - EXPECT_EQ(state_of(moved_from), state_of(Mat{})); + EXPECT_EQ(state_of(moved_to), initial_state); + EXPECT_EQ(state_of(moved_from), state_of(Mat{})); } }; diff --git a/modules/gapi/test/own/scalar_tests.cpp b/modules/gapi/test/own/scalar_tests.cpp index 09fec67ab9..05f97be8da 100644 --- a/modules/gapi/test/own/scalar_tests.cpp +++ b/modules/gapi/test/own/scalar_tests.cpp @@ -17,7 +17,7 @@ TEST(Scalar, CreateEmpty) for (int i = 0; i < 4; ++i) { - EXPECT_EQ(s[i], 0.0); + EXPECT_EQ(0.0, s[i]); } } @@ -25,20 +25,20 @@ TEST(Scalar, CreateFromVal) { cv::gapi::own::Scalar s(5.0); - EXPECT_EQ(s[0], 5.0); - EXPECT_EQ(s[1], 0.0); - EXPECT_EQ(s[2], 0.0); - EXPECT_EQ(s[3], 0.0); + EXPECT_EQ(5.0, s[0]); + EXPECT_EQ(0.0, s[1]); + EXPECT_EQ(0.0, s[2]); + EXPECT_EQ(0.0, s[3]); } TEST(Scalar, CreateFromVals) { cv::gapi::own::Scalar s(5.3, 3.3, 4.1, -2.0); - EXPECT_EQ(s[0], 5.3); - EXPECT_EQ(s[1], 3.3); - EXPECT_EQ(s[2], 4.1); - EXPECT_EQ(s[3], -2.0); + EXPECT_EQ(5.3, s[0]); + EXPECT_EQ(3.3, s[1]); + EXPECT_EQ(4.1, s[2]); + EXPECT_EQ(-2.0, s[3]); } } // namespace opencv_test diff --git a/modules/gapi/test/streaming/gapi_streaming_tests.cpp b/modules/gapi/test/streaming/gapi_streaming_tests.cpp index 4f8b2563ac..57e061861c 100644 --- a/modules/gapi/test/streaming/gapi_streaming_tests.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_tests.cpp @@ -1130,7 +1130,7 @@ TEST_F(GAPI_Streaming_TemplateTypes, UnusedVectorIsOK) } GAPI_Assert(out_mat || out_int); if (out_int) { - EXPECT_EQ( 3, out_int.value()); + EXPECT_EQ(3, out_int.value()); } } } @@ -1748,7 +1748,7 @@ TEST(GAPI_Streaming_Desync, MultipleDesyncOutputs_1) { if (out_vec || out_int) { EXPECT_EQ(320, out_vec.value()[0]); EXPECT_EQ(240, out_vec.value()[1]); - EXPECT_EQ( 3, out_int.value()); + EXPECT_EQ(3, out_int.value()); } } } diff --git a/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp b/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp index 0b8822b366..3ffdaf0fc9 100644 --- a/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp @@ -124,11 +124,11 @@ TEST(OneVPL_Source_Surface, InitSurface) // check self consistency EXPECT_EQ(reinterpret_cast(surf->get_handle()), reinterpret_cast(mfx_core_handle)); - EXPECT_EQ(surf->get_locks_count(), 0); - EXPECT_EQ(surf->obtain_lock(), 0); - EXPECT_EQ(surf->get_locks_count(), 1); - EXPECT_EQ(surf->release_lock(), 1); - EXPECT_EQ(surf->get_locks_count(), 0); + EXPECT_EQ(0, surf->get_locks_count()); + EXPECT_EQ(0, surf->obtain_lock()); + EXPECT_EQ(1, surf->get_locks_count()); + EXPECT_EQ(1, surf->release_lock()); + EXPECT_EQ(0, surf->get_locks_count()); } TEST(OneVPL_Source_Surface, ConcurrentLock) @@ -143,7 +143,7 @@ TEST(OneVPL_Source_Surface, ConcurrentLock) auto surf = Surface::create_surface(std::move(handle), associated_memory); // check self consistency - EXPECT_EQ(surf->get_locks_count(), 0); + EXPECT_EQ(0, surf->get_locks_count()); // MFX internal limitation: do not exceede U16 range // so I16 is using here @@ -168,7 +168,7 @@ TEST(OneVPL_Source_Surface, ConcurrentLock) } worker_thread.join(); - EXPECT_EQ(surf->get_locks_count(), lock_counter * 2); + EXPECT_EQ(lock_counter * 2, surf->get_locks_count()); } TEST(OneVPL_Source_Surface, MemoryLifeTime) @@ -180,7 +180,7 @@ TEST(OneVPL_Source_Surface, MemoryLifeTime) std::shared_ptr associated_memory (preallocated_memory_ptr.get(), [&preallocated_memory_ptr] (void* ptr) { EXPECT_TRUE(preallocated_memory_ptr); - EXPECT_EQ(preallocated_memory_ptr.get(), ptr); + EXPECT_EQ(ptr, preallocated_memory_ptr.get()); preallocated_memory_ptr.reset(); }); @@ -201,7 +201,7 @@ TEST(OneVPL_Source_Surface, MemoryLifeTime) } // workspace memory must be alive - EXPECT_EQ(surfaces.size(), 0); + EXPECT_EQ(0, surfaces.size()); EXPECT_TRUE(associated_memory != nullptr); EXPECT_TRUE(preallocated_memory_ptr.get() != nullptr); @@ -223,7 +223,7 @@ TEST(OneVPL_Source_Surface, MemoryLifeTime) associated_memory.reset(); // workspace memory must be still alive - EXPECT_EQ(surfaces.size(), 0); + EXPECT_EQ(0, surfaces.size()); EXPECT_TRUE(associated_memory == nullptr); EXPECT_TRUE(preallocated_memory_ptr.get() != nullptr); @@ -246,13 +246,13 @@ TEST(OneVPL_Source_CPU_FrameAdapter, InitFrameAdapter) auto surf = Surface::create_surface(std::move(handle), associated_memory); // check consistency - EXPECT_EQ(surf->get_locks_count(), 0); + EXPECT_EQ(0, surf->get_locks_count()); { VPLMediaFrameCPUAdapter adapter(surf); - EXPECT_EQ(surf->get_locks_count(), 1); + EXPECT_EQ(1, surf->get_locks_count()); } - EXPECT_EQ(surf->get_locks_count(), 0); + EXPECT_EQ(0, surf->get_locks_count()); } TEST(OneVPL_Source_CPU_Accelerator, InitDestroy) @@ -276,8 +276,8 @@ TEST(OneVPL_Source_CPU_Accelerator, InitDestroy) surface_size_bytes, create_test_surface); // check consistency - EXPECT_EQ(acceleration_policy->get_surface_count(key), surface_count); - EXPECT_EQ(acceleration_policy->get_free_surface_count(key), surface_count); + EXPECT_EQ(surface_count, acceleration_policy->get_surface_count(key)); + EXPECT_EQ(surface_count, acceleration_policy->get_free_surface_count(key)); pool_export_keys.push_back(key); } @@ -301,8 +301,8 @@ TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConsume) surface_size_bytes, create_test_surface); // check consistency - EXPECT_EQ(acceleration_policy->get_surface_count(key), surface_count); - EXPECT_EQ(acceleration_policy->get_free_surface_count(key), surface_count); + EXPECT_EQ(surface_count, acceleration_policy->get_surface_count(key)); + EXPECT_EQ(surface_count, acceleration_policy->get_free_surface_count(key)); // consume available surfaces std::vector> surfaces; @@ -310,13 +310,13 @@ TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConsume) for (size_t i = 0; i < surface_count; i++) { std::shared_ptr surf = acceleration_policy->get_free_surface(key).lock(); EXPECT_TRUE(surf.get() != nullptr); - EXPECT_EQ(surf->obtain_lock(), 0); + EXPECT_EQ(0, surf->obtain_lock()); surfaces.push_back(std::move(surf)); } // check consistency (no free surfaces) EXPECT_EQ(acceleration_policy->get_surface_count(key), surface_count); - EXPECT_EQ(acceleration_policy->get_free_surface_count(key), 0); + EXPECT_EQ(0, acceleration_policy->get_free_surface_count(key)); // fail consume non-free surfaces for (size_t i = 0; i < surface_count; i++) { @@ -325,19 +325,19 @@ TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConsume) // release surfaces for (auto& surf : surfaces) { - EXPECT_EQ(surf->release_lock(), 1); + EXPECT_EQ(1, surf->release_lock()); } surfaces.clear(); // check consistency - EXPECT_EQ(acceleration_policy->get_surface_count(key), surface_count); - EXPECT_EQ(acceleration_policy->get_free_surface_count(key), surface_count); + EXPECT_EQ(surface_count, acceleration_policy->get_surface_count(key)); + EXPECT_EQ(surface_count, acceleration_policy->get_free_surface_count(key)); //check availability after release for (size_t i = 0; i < surface_count; i++) { std::shared_ptr surf = acceleration_policy->get_free_surface(key).lock(); EXPECT_TRUE(surf.get() != nullptr); - EXPECT_EQ(surf->obtain_lock(), 0); + EXPECT_EQ(0, surf->obtain_lock()); } } @@ -358,8 +358,8 @@ TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConcurrentConsume) create_test_surface); // check consistency - EXPECT_EQ(acceleration_policy->get_surface_count(key), surface_count); - EXPECT_EQ(acceleration_policy->get_free_surface_count(key), surface_count); + EXPECT_EQ(surface_count, acceleration_policy->get_surface_count(key)); + EXPECT_EQ(surface_count, acceleration_policy->get_free_surface_count(key)); // consume available surfaces std::vector> surfaces; @@ -367,7 +367,7 @@ TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConcurrentConsume) for (size_t i = 0; i < surface_count; i++) { std::shared_ptr surf = acceleration_policy->get_free_surface(key).lock(); EXPECT_TRUE(surf.get() != nullptr); - EXPECT_EQ(surf->obtain_lock(), 0); + EXPECT_EQ(0, surf->obtain_lock()); surfaces.push_back(std::move(surf)); } @@ -381,7 +381,7 @@ TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConcurrentConsume) // concurrent release surfaces size_t surfaces_count = surfaces.size(); for (auto& surf : surfaces) { - EXPECT_EQ(surf->release_lock(), 1); + EXPECT_EQ(1, surf->release_lock()); std::this_thread::sleep_for(std::chrono::seconds(1)); } surfaces.clear(); @@ -415,28 +415,28 @@ TEST(OneVPL_Source_ProcessingEngine, Init) mfxSession mfx_session{}; engine.initialize_session(mfx_session, DecoderParams{}, std::shared_ptr{}); - EXPECT_EQ(engine.get_ready_frames_count(), 0); + EXPECT_EQ(0, engine.get_ready_frames_count()); ProcessingEngineBase::ExecutionStatus ret = engine.process(mfx_session); EXPECT_EQ(ret, ProcessingEngineBase::ExecutionStatus::Continue); - EXPECT_EQ(engine.pipeline_stage_num, 0); + EXPECT_EQ(0, engine.pipeline_stage_num); ret = engine.process(mfx_session); EXPECT_EQ(ret, ProcessingEngineBase::ExecutionStatus::Continue); - EXPECT_EQ(engine.pipeline_stage_num, 1); + EXPECT_EQ(1, engine.pipeline_stage_num); ret = engine.process(mfx_session); EXPECT_EQ(ret, ProcessingEngineBase::ExecutionStatus::Continue); - EXPECT_EQ(engine.pipeline_stage_num, 2); + EXPECT_EQ(2, engine.pipeline_stage_num); ret = engine.process(mfx_session); EXPECT_EQ(ret, ProcessingEngineBase::ExecutionStatus::Processed); - EXPECT_EQ(engine.pipeline_stage_num, 3); - EXPECT_EQ(engine.get_ready_frames_count(), 1); + EXPECT_EQ(3, engine.pipeline_stage_num); + EXPECT_EQ(1, engine.get_ready_frames_count()); ret = engine.process(mfx_session); EXPECT_EQ(ret, ProcessingEngineBase::ExecutionStatus::SessionNotFound); - EXPECT_EQ(engine.pipeline_stage_num, 3); - EXPECT_EQ(engine.get_ready_frames_count(), 1); + EXPECT_EQ(3, engine.pipeline_stage_num); + EXPECT_EQ(1, engine.get_ready_frames_count()); cv::gapi::wip::Data frame; engine.get_frame(frame); diff --git a/modules/gapi/test/util/any_tests.cpp b/modules/gapi/test/util/any_tests.cpp index 238c6dbd7b..890aabc72e 100644 --- a/modules/gapi/test/util/any_tests.cpp +++ b/modules/gapi/test/util/any_tests.cpp @@ -17,10 +17,10 @@ TEST(Any, basic) any a(8); auto casted_pointer = any_cast(&a); ASSERT_NE(nullptr, casted_pointer); - ASSERT_EQ(*casted_pointer, 8); + ASSERT_EQ(8, *casted_pointer); *casted_pointer = 7; - ASSERT_EQ(any_cast(a), 7); + ASSERT_EQ(7, any_cast(a)); } TEST(Any, any_cast_ref_throws_on_empty) @@ -36,13 +36,13 @@ TEST(Any, copy) using namespace util; any a(8); - ASSERT_EQ(any_cast(a), 8); + ASSERT_EQ(8, any_cast(a)); any b (a); ASSERT_NE(nullptr, any_cast(&b)); - ASSERT_EQ(8 , any_cast(b)); - ASSERT_EQ(8 , any_cast(a)); + ASSERT_EQ(8, any_cast(b)); + ASSERT_EQ(8, any_cast(a)); } TEST(Any, copy_empty) @@ -63,12 +63,12 @@ TEST(Any, move) using namespace util; any a(8); - ASSERT_EQ(any_cast(a), 8); + ASSERT_EQ(8, any_cast(a)); any b (std::move(a)); ASSERT_NE(nullptr, any_cast(&b)); - ASSERT_EQ(8 , any_cast(b)); + ASSERT_EQ(8, any_cast(b)); ASSERT_EQ(nullptr, any_cast(&a)); } @@ -93,12 +93,12 @@ TEST(Any, move_assign) any a(8); any b; - ASSERT_EQ(any_cast(a), 8); + ASSERT_EQ(8, any_cast(a)); b = (std::move(a)); ASSERT_NE(nullptr, any_cast(&b)); - ASSERT_EQ(8 , any_cast(b)); + ASSERT_EQ(8, any_cast(b)); ASSERT_EQ(nullptr, any_cast(&a)); } @@ -108,14 +108,14 @@ TEST(Any, copy_assign) any a(8); any b; - ASSERT_EQ(any_cast(a), 8); + ASSERT_EQ(8, any_cast(a)); ASSERT_EQ(nullptr, any_cast(&b)); b = a; ASSERT_NE(nullptr, any_cast(&b)); - ASSERT_EQ(8 , any_cast(b)); - ASSERT_EQ(8 , any_cast(a)); + ASSERT_EQ(8, any_cast(b)); + ASSERT_EQ(8, any_cast(a)); } TEST(Any, get_ref_to_val_from_any) @@ -125,7 +125,7 @@ TEST(Any, get_ref_to_val_from_any) any a(x); int& casted_ref = any_cast(a); - ASSERT_EQ(casted_ref, 8); + ASSERT_EQ(8, casted_ref); } TEST(Any, update_val_via_ref) @@ -134,9 +134,9 @@ TEST(Any, update_val_via_ref) int x = 8; any a(x); int& casted_ref = any_cast(a); - ASSERT_EQ(casted_ref, 8); + ASSERT_EQ(8, casted_ref); casted_ref = 7; - ASSERT_EQ(any_cast(a), 7); + ASSERT_EQ(7, any_cast(a)); } } // namespace opencv_test diff --git a/modules/gapi/test/util/variant_tests.cpp b/modules/gapi/test/util/variant_tests.cpp index 7725f9a702..aa096e711c 100644 --- a/modules/gapi/test/util/variant_tests.cpp +++ b/modules/gapi/test/util/variant_tests.cpp @@ -587,34 +587,34 @@ TEST(Variant, DynamicVisitor) test_validation::MyBoolParamIndexedVisitor visitor(ss); EXPECT_TRUE(cv::util::visit(visitor, var, int{42})); - EXPECT_EQ(ss.str(), std::string("0:42,")); + EXPECT_EQ(std::string("0:42,"), ss.str()); } std::stringstream ss; test_validation::MyBoolNoParamNonIndexedVisitor visitor(ss); cv::util::visit(visitor, var); - EXPECT_EQ(ss.str(), std::string("0:42,")); + EXPECT_EQ(std::string("0:42,"), ss.str()); var = double{1.0}; EXPECT_TRUE(cv::util::visit(visitor, var)); - EXPECT_EQ(ss.str(), std::string("0:42,1:1,")); + EXPECT_EQ(std::string("0:42,1:1,"), ss.str()); var = char{'a'}; EXPECT_TRUE(cv::util::visit(visitor, var)); - EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,")); + EXPECT_EQ(std::string("0:42,1:1,2:a,"), ss.str()); var = float{6.0}; EXPECT_TRUE(cv::util::visit(visitor, var)); - EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,")); + EXPECT_EQ(std::string("0:42,1:1,2:a,3:6,"), ss.str()); var = test_validation::MyType{}; EXPECT_TRUE(cv::util::visit(visitor, var)); - EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,4:MyType,")); + EXPECT_EQ(std::string("0:42,1:1,2:a,3:6,4:MyType,"), ss.str()); var = test_validation::MyClass{}; EXPECT_TRUE(cv::util::visit(visitor, var)); - EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,4:MyType,5:MyClass,")); + EXPECT_EQ(std::string("0:42,1:1,2:a,3:6,4:MyType,5:MyClass,"), ss.str()); } TEST(Variant, StaticVisitor) @@ -625,27 +625,27 @@ TEST(Variant, StaticVisitor) test_validation::MyVoidNoParamNonIndexedVisitor visitor(ss); cv::util::visit(visitor, var); - EXPECT_EQ(ss.str(), std::string("42,")); + EXPECT_EQ(std::string("42,"), ss.str()); var = double{1.0}; cv::util::visit(visitor, var); - EXPECT_EQ(ss.str(), std::string("42,1,")); + EXPECT_EQ(std::string("42,1,"), ss.str()); var = char{'a'}; cv::util::visit(visitor, var); - EXPECT_EQ(ss.str(), std::string("42,1,a,")); + EXPECT_EQ(std::string("42,1,a,"), ss.str()); var = float{6.0}; cv::util::visit(visitor, var); - EXPECT_EQ(ss.str(), std::string("42,1,a,6,")); + EXPECT_EQ(std::string("42,1,a,6,"), ss.str()); var = test_validation::MyType{}; cv::util::visit(visitor, var); - EXPECT_EQ(ss.str(), std::string("42,1,a,6,MyType,")); + EXPECT_EQ(std::string("42,1,a,6,MyType,"), ss.str()); var = test_validation::MyClass{}; cv::util::visit(visitor, var); - EXPECT_EQ(ss.str(), std::string("42,1,a,6,MyType,MyClass,")); + EXPECT_EQ(std::string("42,1,a,6,MyType,MyClass,"), ss.str()); } TEST(Variant, StaticIndexedVisitor) @@ -655,27 +655,27 @@ TEST(Variant, StaticIndexedVisitor) std::stringstream ss; cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor {ss}, var); - EXPECT_EQ(ss.str(), std::string("0:42,")); + EXPECT_EQ(std::string("0:42,"), ss.str()); var = double{1.0}; cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor (ss), var); - EXPECT_EQ(ss.str(), std::string("0:42,1:1,")); + EXPECT_EQ(std::string("0:42,1:1,"), ss.str()); var = char{'a'}; cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor (ss), var); - EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,")); + EXPECT_EQ(std::string("0:42,1:1,2:a,"), ss.str()); var = float{6.0}; cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor (ss), var); - EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,")); + EXPECT_EQ(std::string("0:42,1:1,2:a,3:6,"), ss.str()); var = test_validation::MyType{}; cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor (ss), var); - EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,4:MyType,")); + EXPECT_EQ(std::string("0:42,1:1,2:a,3:6,4:MyType,"), ss.str()); var = test_validation::MyClass{}; cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor (ss), var); - EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,4:MyType,5:MyClass,")); + EXPECT_EQ(std::string("0:42,1:1,2:a,3:6,4:MyType,5:MyClass,"), ss.str()); } @@ -686,7 +686,7 @@ TEST(Variant, LambdaVisitor) { cv::util::visit(cv::util::overload_lambdas( [](int value) { - EXPECT_EQ(value, 42); + EXPECT_EQ(42, value); }, [](double) { ADD_FAILURE() << "can't be called for `double`"; @@ -719,7 +719,7 @@ TEST(Variant, LambdaVisitor) ADD_FAILURE() << "can't be called for `double`"; }, [](char value) { - EXPECT_EQ(value, 'c'); + EXPECT_EQ('c', value); }, [](float) { ADD_FAILURE() << "can't be called for `float`"; From 69d0bc8fd56d6287e7851b2b9ee16e5ec4e61e69 Mon Sep 17 00:00:00 2001 From: Julie Bareeva Date: Fri, 29 Oct 2021 19:46:11 +0300 Subject: [PATCH 041/226] Added overflow handling during conversion from float to int for LinearFilter --- modules/core/include/opencv2/core/cuda/filters.hpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modules/core/include/opencv2/core/cuda/filters.hpp b/modules/core/include/opencv2/core/cuda/filters.hpp index bb94212299..bf3147edb9 100644 --- a/modules/core/include/opencv2/core/cuda/filters.hpp +++ b/modules/core/include/opencv2/core/cuda/filters.hpp @@ -47,6 +47,7 @@ #include "vec_traits.hpp" #include "vec_math.hpp" #include "type_traits.hpp" +#include "nppdefs.h" /** @file * @deprecated Use @ref cudev instead. @@ -95,6 +96,12 @@ namespace cv { namespace cuda { namespace device const int x1 = __float2int_rd(x); const int y1 = __float2int_rd(y); + if (x1 <= NPP_MIN_32S || x1 >= NPP_MAX_32S || y1 <= NPP_MIN_32S || y1 >= NPP_MAX_32S) + { + elem_type src_reg = src(y1, x1); + out = out + src_reg * 1.0f; + return saturate_cast(out); + } const int x2 = x1 + 1; const int y2 = y1 + 1; From 40c748a2aeca327c11c5daac2fa0f387e24b7676 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 29 Oct 2021 16:08:57 +0000 Subject: [PATCH 042/226] python: properly handle step for multichannel case --- modules/core/src/matrix.cpp | 2 +- modules/python/src2/cv2.cpp | 2 ++ modules/python/test/test_misc.py | 4 ++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/modules/core/src/matrix.cpp b/modules/core/src/matrix.cpp index 4642d3b623..1729862cb7 100644 --- a/modules/core/src/matrix.cpp +++ b/modules/core/src/matrix.cpp @@ -463,7 +463,7 @@ Mat::Mat(Size _sz, int _type, void* _data, size_t _step) } else { - CV_Assert(_step >= minstep); + CV_CheckGE(_step, minstep, ""); if (_step % esz1 != 0) { diff --git a/modules/python/src2/cv2.cpp b/modules/python/src2/cv2.cpp index 6231fde67f..58b1357eff 100644 --- a/modules/python/src2/cv2.cpp +++ b/modules/python/src2/cv2.cpp @@ -788,6 +788,8 @@ static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) if (ndims >= 1 && _strides[ndims - 1] != (npy_intp)elemsize*_sizes[ndims]) needcopy = true; + + elemsize = CV_ELEM_SIZE(type); } if (needcopy) diff --git a/modules/python/test/test_misc.py b/modules/python/test/test_misc.py index 76803992dc..fe484089f6 100644 --- a/modules/python/test/test_misc.py +++ b/modules/python/test/test_misc.py @@ -187,6 +187,10 @@ class Arguments(NewOpenCVTests): #res6 = cv.utils.dumpInputArray([a, b]) #self.assertEqual(res6, "InputArrayOfArrays: empty()=false kind=0x00050000 flags=0x01050000 total(-1)=2 dims(-1)=1 size(-1)=2x1 type(0)=CV_32FC1 dims(0)=4 size(0)=[2 3 4 5]") + def test_20968(self): + pixel = np.uint8([[[40, 50, 200]]]) + _ = cv.cvtColor(pixel, cv.COLOR_RGB2BGR) # should not raise exception + def test_parse_to_bool_convertible(self): try_to_convert = partial(self._try_to_convert, cv.utils.dumpBool) for convertible_true in (True, 1, 64, np.bool(1), np.int8(123), np.int16(11), np.int32(2), From 0ee61d178f79d9d679af6b34eda1009cdf1415c1 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 30 Oct 2021 10:51:52 +0000 Subject: [PATCH 043/226] highgui: drop invalid cvGetWindowImageRect - return type is C++ template - removal from 'extern "C"' scope broke ABI anyway, so this symbols is removed completelly --- modules/highgui/include/opencv2/highgui/highgui_c.h | 5 ----- modules/highgui/src/window.cpp | 1 + 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/modules/highgui/include/opencv2/highgui/highgui_c.h b/modules/highgui/include/opencv2/highgui/highgui_c.h index 35413139c7..d8323d0d93 100644 --- a/modules/highgui/include/opencv2/highgui/highgui_c.h +++ b/modules/highgui/include/opencv2/highgui/highgui_c.h @@ -135,11 +135,6 @@ CVAPI(int) cvNamedWindow( const char* name, int flags CV_DEFAULT(CV_WINDOW_AUTOS CVAPI(void) cvSetWindowProperty(const char* name, int prop_id, double prop_value); CVAPI(double) cvGetWindowProperty(const char* name, int prop_id); -#ifdef __cplusplus // FIXIT remove in OpenCV 4.0 -/* Get window image rectangle coordinates, width and height */ -CVAPI(cv::Rect)cvGetWindowImageRect(const char* name); -#endif - /* display image within window (highgui windows remember their content) */ CVAPI(void) cvShowImage( const char* name, const CvArr* image ); diff --git a/modules/highgui/src/window.cpp b/modules/highgui/src/window.cpp index 877d6751c9..cfe58a0277 100644 --- a/modules/highgui/src/window.cpp +++ b/modules/highgui/src/window.cpp @@ -191,6 +191,7 @@ CV_IMPL double cvGetWindowProperty(const char* name, int prop_id) } } +static cv::Rect cvGetWindowImageRect(const char* name) { if (!name) From 66f3e9745724d0a4b984d764ccf3298a13e69302 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 30 Oct 2021 10:57:47 +0000 Subject: [PATCH 044/226] highgui: drop invalid cvGetWindowImageRect --- .../include/opencv2/highgui/highgui_c.h | 5 ----- modules/highgui/src/window.cpp | 22 +++++++------------ 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/modules/highgui/include/opencv2/highgui/highgui_c.h b/modules/highgui/include/opencv2/highgui/highgui_c.h index 5d20b9501e..e508e14975 100644 --- a/modules/highgui/include/opencv2/highgui/highgui_c.h +++ b/modules/highgui/include/opencv2/highgui/highgui_c.h @@ -129,11 +129,6 @@ CVAPI(int) cvNamedWindow( const char* name, int flags CV_DEFAULT(CV_WINDOW_AUTOS CVAPI(void) cvSetWindowProperty(const char* name, int prop_id, double prop_value); CVAPI(double) cvGetWindowProperty(const char* name, int prop_id); -#ifdef __cplusplus // FIXIT remove in OpenCV 4.0 -/* Get window image rectangle coordinates, width and height */ -CVAPI(cv::Rect)cvGetWindowImageRect(const char* name); -#endif - /* display image within window (highgui windows remember their content) */ CVAPI(void) cvShowImage( const char* name, const CvArr* image ); diff --git a/modules/highgui/src/window.cpp b/modules/highgui/src/window.cpp index d9481de6da..481fee9fbd 100644 --- a/modules/highgui/src/window.cpp +++ b/modules/highgui/src/window.cpp @@ -399,13 +399,13 @@ CV_IMPL double cvGetWindowProperty(const char* name, int prop_id) #endif } -cv::Rect cvGetWindowImageRect(const char* name) +cv::Rect cv::getWindowImageRect(const String& winname) { CV_TRACE_FUNCTION(); - CV_Assert(name); + CV_Assert(!winname.empty()); { - auto window = findWindow_(name); + auto window = findWindow_(winname); if (window) { return window->getImageRect(); @@ -416,7 +416,7 @@ cv::Rect cvGetWindowImageRect(const char* name) auto backend = getCurrentUIBackend(); if (backend) { - CV_LOG_WARNING(NULL, "Can't find window with name: '" << name << "'. Do nothing"); + CV_LOG_WARNING(NULL, "Can't find window with name: '" << winname << "'. Do nothing"); CV_NOT_FOUND_DEPRECATION; } else @@ -427,13 +427,13 @@ cv::Rect cvGetWindowImageRect(const char* name) #else #if defined (HAVE_QT) - return cvGetWindowRect_QT(name); + return cvGetWindowRect_QT(winname.c_str()); #elif defined(HAVE_WIN32UI) - return cvGetWindowRect_W32(name); + return cvGetWindowRect_W32(winname.c_str()); #elif defined (HAVE_GTK) - return cvGetWindowRect_GTK(name); + return cvGetWindowRect_GTK(winname.c_str()); #elif defined (HAVE_COCOA) - return cvGetWindowRect_COCOA(name); + return cvGetWindowRect_COCOA(winname.c_str()); #else return Rect(-1, -1, -1, -1); #endif @@ -441,12 +441,6 @@ cv::Rect cvGetWindowImageRect(const char* name) #endif } -cv::Rect cv::getWindowImageRect(const String& winname) -{ - CV_TRACE_FUNCTION(); - return cvGetWindowImageRect(winname.c_str()); -} - void cv::namedWindow( const String& winname, int flags ) { CV_TRACE_FUNCTION(); From 17234f82d025e3bbfbf611089637e5aa2038e7b8 Mon Sep 17 00:00:00 2001 From: Trutnev Aleksei Date: Mon, 1 Nov 2021 15:43:27 +0300 Subject: [PATCH 045/226] Merge pull request #20836 from alexgiving:atrutnev/rename_Adapter_to_IAdapter * Rename RMat::Adapter to RMat::IAdapter * Add comments --- modules/gapi/include/opencv2/gapi/rmat.hpp | 18 ++++++++++-------- modules/gapi/include/opencv2/gapi/s11n.hpp | 2 +- modules/gapi/src/backends/common/gbackend.hpp | 4 ++-- modules/gapi/test/rmat/rmat_test_common.hpp | 4 ++-- modules/gapi/test/rmat/rmat_tests.cpp | 4 ++-- modules/gapi/test/s11n/gapi_s11n_tests.cpp | 2 +- 6 files changed, 18 insertions(+), 16 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/rmat.hpp b/modules/gapi/include/opencv2/gapi/rmat.hpp index 6b289001e7..38668d67a5 100644 --- a/modules/gapi/include/opencv2/gapi/rmat.hpp +++ b/modules/gapi/include/opencv2/gapi/rmat.hpp @@ -25,11 +25,11 @@ namespace cv { // "Remote Mat", a general class which provides an abstraction layer over the data // storage and placement (host, remote device etc) and allows to access this data. // -// The device specific implementation is hidden in the RMat::Adapter class +// The device specific implementation is hidden in the RMat::IAdapter class // // The basic flow is the following: // * Backend which is aware of the remote device: -// - Implements own AdapterT class which is derived from RMat::Adapter +// - Implements own AdapterT class which is derived from RMat::IAdapter // - Wraps device memory into RMat via make_rmat utility function: // cv::RMat rmat = cv::make_rmat(args); // @@ -101,25 +101,27 @@ public: }; enum class Access { R, W }; - class Adapter + class IAdapter + // Adapter class is going to be deleted and renamed as IAdapter { public: - virtual ~Adapter() = default; + virtual ~IAdapter() = default; virtual GMatDesc desc() const = 0; // Implementation is responsible for setting the appropriate callback to // the view when accessed for writing, to ensure that the data from the view // is transferred to the device when the view is destroyed virtual View access(Access) = 0; virtual void serialize(cv::gapi::s11n::IOStream&) { - GAPI_Assert(false && "Generic serialize method of RMat::Adapter does nothing by default. " + GAPI_Assert(false && "Generic serialize method of RMat::IAdapter does nothing by default. " "Please, implement it in derived class to properly serialize the object."); } virtual void deserialize(cv::gapi::s11n::IIStream&) { - GAPI_Assert(false && "Generic deserialize method of RMat::Adapter does nothing by default. " + GAPI_Assert(false && "Generic deserialize method of RMat::IAdapter does nothing by default. " "Please, implement it in derived class to properly deserialize the object."); } }; - using AdapterP = std::shared_ptr; + using Adapter = IAdapter; // Keep backward compatibility + using AdapterP = std::shared_ptr; RMat() = default; RMat(AdapterP&& a) : m_adapter(std::move(a)) {} @@ -136,7 +138,7 @@ public: // return nullptr if underlying type is different template T* get() const { - static_assert(std::is_base_of::value, "T is not derived from Adapter!"); + static_assert(std::is_base_of::value, "T is not derived from IAdapter!"); GAPI_Assert(m_adapter != nullptr); return dynamic_cast(m_adapter.get()); } diff --git a/modules/gapi/include/opencv2/gapi/s11n.hpp b/modules/gapi/include/opencv2/gapi/s11n.hpp index 53800970d1..6863a5ecab 100644 --- a/modules/gapi/include/opencv2/gapi/s11n.hpp +++ b/modules/gapi/include/opencv2/gapi/s11n.hpp @@ -431,7 +431,7 @@ struct deserialize_runarg { static GRunArg exec(cv::gapi::s11n::IIStream& is, uint32_t idx) { if (idx == GRunArg::index_of()) { // Type or void (if not found) - using TA = typename cv::util::find_adapter_impl::type; + using TA = typename cv::util::find_adapter_impl::type; return deserialize_arg_with_adapter::exec(is); } else if (idx == GRunArg::index_of()) { // Type or void (if not found) diff --git a/modules/gapi/src/backends/common/gbackend.hpp b/modules/gapi/src/backends/common/gbackend.hpp index b22ec5e177..f005135d87 100644 --- a/modules/gapi/src/backends/common/gbackend.hpp +++ b/modules/gapi/src/backends/common/gbackend.hpp @@ -45,7 +45,7 @@ namespace gimpl { #endif } - class RMatAdapter : public RMat::Adapter { + class RMatAdapter : public RMat::IAdapter { cv::Mat m_mat; public: const void* data() const { return m_mat.data; } @@ -58,7 +58,7 @@ namespace gimpl { struct Data; struct RcDesc; - struct GAPI_EXPORTS RMatMediaFrameAdapter final: public cv::RMat::Adapter + struct GAPI_EXPORTS RMatMediaFrameAdapter final: public cv::RMat::IAdapter { using MapDescF = std::function; using MapDataF = std::function; diff --git a/modules/gapi/test/rmat/rmat_test_common.hpp b/modules/gapi/test/rmat/rmat_test_common.hpp index 5685d06253..218b21b632 100644 --- a/modules/gapi/test/rmat/rmat_test_common.hpp +++ b/modules/gapi/test/rmat/rmat_test_common.hpp @@ -11,7 +11,7 @@ #include namespace opencv_test { -class RMatAdapterRef : public RMat::Adapter { +class RMatAdapterRef : public RMat::IAdapter { cv::Mat& m_mat; bool& m_callbackCalled; public: @@ -36,7 +36,7 @@ public: virtual cv::GMatDesc desc() const override { return cv::descr_of(m_mat); } }; -class RMatAdapterCopy : public RMat::Adapter { +class RMatAdapterCopy : public RMat::IAdapter { cv::Mat& m_deviceMat; cv::Mat m_hostMat; bool& m_callbackCalled; diff --git a/modules/gapi/test/rmat/rmat_tests.cpp b/modules/gapi/test/rmat/rmat_tests.cpp index 52c3806c5b..ea2f9040e5 100644 --- a/modules/gapi/test/rmat/rmat_tests.cpp +++ b/modules/gapi/test/rmat/rmat_tests.cpp @@ -94,7 +94,7 @@ TYPED_TEST(RMatTypedTest, CorrectAdapterCast) { EXPECT_NE(nullptr, this->rmat().template get()); } -class DummyAdapter : public RMat::Adapter { +class DummyAdapter : public RMat::IAdapter { virtual RMat::View access(RMat::Access) override { return {}; } virtual cv::GMatDesc desc() const override { return {}; } }; @@ -103,7 +103,7 @@ TYPED_TEST(RMatTypedTest, IncorrectAdapterCast) { EXPECT_EQ(nullptr, this->rmat().template get()); } -class RMatAdapterForBackend : public RMat::Adapter { +class RMatAdapterForBackend : public RMat::IAdapter { int m_i; public: RMatAdapterForBackend(int i) : m_i(i) {} diff --git a/modules/gapi/test/s11n/gapi_s11n_tests.cpp b/modules/gapi/test/s11n/gapi_s11n_tests.cpp index 4c6e63b552..94b99f877a 100644 --- a/modules/gapi/test/s11n/gapi_s11n_tests.cpp +++ b/modules/gapi/test/s11n/gapi_s11n_tests.cpp @@ -127,7 +127,7 @@ template<> struct CompileArgTag { } // namespace cv namespace { -class MyRMatAdapter : public cv::RMat::Adapter { +class MyRMatAdapter : public cv::RMat::IAdapter { cv::Mat m_mat; int m_value; std::string m_str; From 30d6766db48a7b8265e2c0e08e4dab0e3b778ee8 Mon Sep 17 00:00:00 2001 From: Souriya Trinh Date: Thu, 28 Oct 2021 23:46:13 +0200 Subject: [PATCH 046/226] Add conventional Bayer naming. --- modules/imgproc/doc/colors.markdown | 15 ++- modules/imgproc/doc/pics/Bayer_patterns.png | Bin 0 -> 4298 bytes modules/imgproc/include/opencv2/imgproc.hpp | 119 ++++++++++++++------ 3 files changed, 93 insertions(+), 41 deletions(-) create mode 100755 modules/imgproc/doc/pics/Bayer_patterns.png diff --git a/modules/imgproc/doc/colors.markdown b/modules/imgproc/doc/colors.markdown index 47d97a7263..c1c2abfce7 100644 --- a/modules/imgproc/doc/colors.markdown +++ b/modules/imgproc/doc/colors.markdown @@ -151,16 +151,23 @@ sources on the web, primarily from the Charles Poynton site _0e!SSq;fs+m5LFu}wcb`I}|v zg&*ZbDjcl)p29kiz4j># zd`W}1^&=a9mEe*^{^k8Bn4xxNVz*%I$+3}*Md4Xfr1e}Ng2bJr3IYW_;{q9@c(L#m zwj`BHeb6|gosSE+d*AjXFP9SexHB*a0*Uy8s{2L#f$KmKkUdpn5M-aFGkDKjctjmk zK?wQH2Pkr7W)Zko-D@M>^3gpNBynm~p8IA&r}>O&(;T zG@kJN-a;;A37jwICW-EKND8z>C4R=t@IS^k?eX%-72~ECxG81ac;%x!{X@sr0|wD4 z5pt_=zQs_7WgTV>L(6R&6X42Jxy}u)jmC4uXz+~T8pFsoB9j-LmD=B=phFt>^JT5b z+hI2rQUVx$c!PF1WkVtkPL>{)^G@lA_qAbtwTj%+>9sMaU`65-_<9y~4(IDFaTkYz zQO{wkP7<}~DbZ#4Wi*cbo{pDDp2d&=AMFRaZc*lX3mN>`GDfk@`w7L)vh;kJkmH$C z%3{}da)y&Ha>Eo@F|-I|WBdgw)dD`;VDm-;Z#L%DT|^-u4%V$VYrrn}j$3C27HIyo z3He51N!{DkX-K|D@&5W=-wf)@DgCKztAYRr|m z+lq0Y-EytwX(^Q^Ku(Exb~`OX5sP~nG+X8irGGFFNi)Z^^)|J~7w#HuFCE;Kqc^VU zUm+|zLdHas)!37$(z!5K7w~YXGrz36mZ0TVLg~wNcP9~lAgOw%0>W$YMKiaFhD%41IkdX$cg^+;|hY!`M_yzuhB{1N!^KeJhdin*i<

hpZM`_scsj= z!@f>XjFh|g;+FzdgkUSB>k{1<7RiC9U#~sst75J>Fr8Jy)1pWXG5T(!0Rhn(#P$dI zizZa$*SOsO`Vnrsf57LCuxhl+tD_yj$DbmQq7O3%SxMoF!sXhB-K`x)nEev+k^a(7 zgW)!)wN|gJU}HBIY-C3*nWQ5UO;$2cyPuI!X;Un@Mtvl&;@ZL8N^wQZFzf)0hhbXt z8OcVUVJ=IlK%(Zx<%cI-lXURsR2xQv&kV4K*y7rbZ?!$SQi7b=&k zrD3E6qs;P_lj>;FQ7l!2dUuNlrk8O_PS~D5(1F0x@F{@HvP(ikutmL1c1eN3D6B7p zdffwk*(%lWss0wxPe^!r&sBA2?~Z5Yr0k{6f2q8_A}sc0-cYqxDr3$=(V5>K{gy0b znOb#t$U2K|vnii?D}ztExY8z-P3zeY>?bcC&wECiIs6&fRea4qGImgWHmgHp+xj8! z9Q5a!Ice@sDGzEI(TI7^=Ha}vT4@g6OVjRMAA+*jHbW~C>$pZZGGxU+Yl5A>7m9Cl zcT)5+iEPo$<};Gjta3=+_(Fh4>^X+XO;zTW&aD%X`kUBVX=v4;JB=&&W)g2@;`7I8 zd|BGH{36RoY?cvXAY~GhH1Vpd)Pq{)c4)bpkYXKUnEOPG`sSB+(kAmOf%oH$hazx0 z%+1EjGdluqQ`+@eemx0=@dn#Vg`R!Kjt%gRpInaQPfOKfq%+LQaaY*PLqh8(iP*NU zvn{G_-K)ekW${aG(omZ!@%M2E@OdYFNuOP$z`EV$_&Dpd#rAWhtcZzXSZXQ~-94EN zpZw`(ibB3D+-S7$wq;X&sNT|K6zzTqH3e!(J@Z}!&jKpyz;4r5bOjcHfzodG@tu*JdM1ga^P6`?zo_!H-M>=Fm4_G8!AtAi7030?(KK~ z0NvYhy?sJ`@Td8XM9VoW+{OCzXAP~T1?YQ|57IRpC&U*HbbOu{K0^kMsUx7r+ z1H&L-;0q(Pd8CLAJAPSfe0mcMa>d}~%qSITgLZrW{9 z9Xu}V4+6=fe%A&6(Iej^2WMqY;Qx*6n{@cI2}hp%uWk6Ht*OoSS*p7A@TxyYzW_#V2dU3(GfW%rFHJJMHj@ZH?zQc3abe& zh)kqM*_}uGg6)-#{6$~hKc|dJ$aFIFHj>Y`fge(3s9uN?rX*RG+UJl|4N3Z?(aiB+ zGYn5wzBzZaV9*C_eY_1F89BVlp5G1W3sA6Zv)kLtLC(Jj&2S$(rAjl26pMxKJiHyT zYG~SnuT*Yp_sL@}P3LKAwirgivp4@&GhmME>JOB6YT&hUm0=;@Z&$Eo7h)MkZ&*!~ zvN@zsc^)&uOLQV_b|P)P)FC-sk^K(2Oq%0teML7lS8VbZ`{tCy9V=m_=0E8IVO1Df zZMy>A6)C(eX5nRb`0E(qU>#$CS)0PYP@`Z=gn;&-x&2%(YjZtmIV1XG5v6DXPkSfK zjlq6p_Vdg7@gRBn5jhVVh<&9k<3_y)N>6(fFv8WH47kj|!=+Gz~_K_AFN-Zm5mH&>3yXSyrZ z@T|4&bFb~E48_6D4wz4*vAYWy@0t*lTS^ygUof-B(j}p-DDjr5^NnyQ-KdLoL;_M7 zbR(&NWFGgsCfLZJfprTA76?y3YP9P*^3SdndJf>kMb;14c#gFPU|l2hy8`$->nl}M>4K>kVqB<3Xe z_S0Pu`}(Dk9*tI;--VgM(9+u#V#&->fs~YK`)&9T`v%CMtA`Q zv}>Qad!;)kO`*8QX=el)FkBB7vQF~MrK}^8+Fo|J@xstD+Z9BPNWcRkAwnE>Ln7Eo zVx`@oEY3~p;-n5q>+;1zJE51_?mAH6{7XrE0_RO2a5>jnJR59y(j~*t9PupjD+$K%mmda`6ysXOqMaLyK)!v>UuZ zyI%QR5U}`Y6O(G%QwslSh|~LMrb2X6xXAMTM|hXZHtWeR)ZQU{-DhGhVPs0Jq!`wl zrl!)Iy4`B4)gl4sBA1kUF+VBJYadS=d#kDaFcYEIa1FORia9#(4=+R5<%P}(=R6i@ z7jfc`H~p^|IKvSGO$eZ)wSDVoDes;t4BKfc+Ql%e#FLYE?7X$=tR^p8UiNYX2c9hP zZeFVs=n8Y17i~-G-|4zoM$YV*$crBdwNkd$3@toR932UdSm^g{y8mXf1#QCVl#?94 zTQZjB7P28~Kc(Ev z{l~zL?ni*z{|_YnwFw9Gf5z8e+y3X={eNHsJ_Q7@yal6`^MdprSa7}!Sen^hZTR8# GgMS08YI(;1 literal 0 HcmV?d00001 diff --git a/modules/imgproc/include/opencv2/imgproc.hpp b/modules/imgproc/include/opencv2/imgproc.hpp index 576bebc21b..8db6cc5d8d 100644 --- a/modules/imgproc/include/opencv2/imgproc.hpp +++ b/modules/imgproc/include/opencv2/imgproc.hpp @@ -744,54 +744,99 @@ enum ColorConversionCodes { COLOR_RGBA2YUV_YV12 = 133, COLOR_BGRA2YUV_YV12 = 134, - //! Demosaicing - COLOR_BayerBG2BGR = 46, - COLOR_BayerGB2BGR = 47, - COLOR_BayerRG2BGR = 48, - COLOR_BayerGR2BGR = 49, + //! Demosaicing, see @ref color_convert_bayer "color conversions" for additional information + COLOR_BayerBG2BGR = 46, //!< equivalent to RGGB Bayer pattern + COLOR_BayerGB2BGR = 47, //!< equivalent to GRBG Bayer pattern + COLOR_BayerRG2BGR = 48, //!< equivalent to BGGR Bayer pattern + COLOR_BayerGR2BGR = 49, //!< equivalent to GBRG Bayer pattern - COLOR_BayerBG2RGB = COLOR_BayerRG2BGR, - COLOR_BayerGB2RGB = COLOR_BayerGR2BGR, - COLOR_BayerRG2RGB = COLOR_BayerBG2BGR, - COLOR_BayerGR2RGB = COLOR_BayerGB2BGR, + COLOR_BayerRGGB2BGR = COLOR_BayerBG2BGR, + COLOR_BayerGRBG2BGR = COLOR_BayerGB2BGR, + COLOR_BayerBGGR2BGR = COLOR_BayerRG2BGR, + COLOR_BayerGBRG2BGR = COLOR_BayerGR2BGR, - COLOR_BayerBG2GRAY = 86, - COLOR_BayerGB2GRAY = 87, - COLOR_BayerRG2GRAY = 88, - COLOR_BayerGR2GRAY = 89, + COLOR_BayerRGGB2RGB = COLOR_BayerBGGR2BGR, + COLOR_BayerGRBG2RGB = COLOR_BayerGBRG2BGR, + COLOR_BayerBGGR2RGB = COLOR_BayerRGGB2BGR, + COLOR_BayerGBRG2RGB = COLOR_BayerGRBG2BGR, + + COLOR_BayerBG2RGB = COLOR_BayerRG2BGR, //!< equivalent to RGGB Bayer pattern + COLOR_BayerGB2RGB = COLOR_BayerGR2BGR, //!< equivalent to GRBG Bayer pattern + COLOR_BayerRG2RGB = COLOR_BayerBG2BGR, //!< equivalent to BGGR Bayer pattern + COLOR_BayerGR2RGB = COLOR_BayerGB2BGR, //!< equivalent to GBRG Bayer pattern + + COLOR_BayerBG2GRAY = 86, //!< equivalent to RGGB Bayer pattern + COLOR_BayerGB2GRAY = 87, //!< equivalent to GRBG Bayer pattern + COLOR_BayerRG2GRAY = 88, //!< equivalent to BGGR Bayer pattern + COLOR_BayerGR2GRAY = 89, //!< equivalent to GBRG Bayer pattern + + COLOR_BayerRGGB2GRAY = COLOR_BayerBG2GRAY, + COLOR_BayerGRBG2GRAY = COLOR_BayerGB2GRAY, + COLOR_BayerBGGR2GRAY = COLOR_BayerRG2GRAY, + COLOR_BayerGBRG2GRAY = COLOR_BayerGR2GRAY, //! Demosaicing using Variable Number of Gradients - COLOR_BayerBG2BGR_VNG = 62, - COLOR_BayerGB2BGR_VNG = 63, - COLOR_BayerRG2BGR_VNG = 64, - COLOR_BayerGR2BGR_VNG = 65, + COLOR_BayerBG2BGR_VNG = 62, //!< equivalent to RGGB Bayer pattern + COLOR_BayerGB2BGR_VNG = 63, //!< equivalent to GRBG Bayer pattern + COLOR_BayerRG2BGR_VNG = 64, //!< equivalent to BGGR Bayer pattern + COLOR_BayerGR2BGR_VNG = 65, //!< equivalent to GBRG Bayer pattern - COLOR_BayerBG2RGB_VNG = COLOR_BayerRG2BGR_VNG, - COLOR_BayerGB2RGB_VNG = COLOR_BayerGR2BGR_VNG, - COLOR_BayerRG2RGB_VNG = COLOR_BayerBG2BGR_VNG, - COLOR_BayerGR2RGB_VNG = COLOR_BayerGB2BGR_VNG, + COLOR_BayerRGGB2BGR_VNG = COLOR_BayerBG2BGR_VNG, + COLOR_BayerGRBG2BGR_VNG = COLOR_BayerGB2BGR_VNG, + COLOR_BayerBGGR2BGR_VNG = COLOR_BayerRG2BGR_VNG, + COLOR_BayerGBRG2BGR_VNG = COLOR_BayerGR2BGR_VNG, + + COLOR_BayerRGGB2RGB_VNG = COLOR_BayerBGGR2BGR_VNG, + COLOR_BayerGRBG2RGB_VNG = COLOR_BayerGBRG2BGR_VNG, + COLOR_BayerBGGR2RGB_VNG = COLOR_BayerRGGB2BGR_VNG, + COLOR_BayerGBRG2RGB_VNG = COLOR_BayerGRBG2BGR_VNG, + + COLOR_BayerBG2RGB_VNG = COLOR_BayerRG2BGR_VNG, //!< equivalent to RGGB Bayer pattern + COLOR_BayerGB2RGB_VNG = COLOR_BayerGR2BGR_VNG, //!< equivalent to GRBG Bayer pattern + COLOR_BayerRG2RGB_VNG = COLOR_BayerBG2BGR_VNG, //!< equivalent to BGGR Bayer pattern + COLOR_BayerGR2RGB_VNG = COLOR_BayerGB2BGR_VNG, //!< equivalent to GBRG Bayer pattern //! Edge-Aware Demosaicing - COLOR_BayerBG2BGR_EA = 135, - COLOR_BayerGB2BGR_EA = 136, - COLOR_BayerRG2BGR_EA = 137, - COLOR_BayerGR2BGR_EA = 138, + COLOR_BayerBG2BGR_EA = 135, //!< equivalent to RGGB Bayer pattern + COLOR_BayerGB2BGR_EA = 136, //!< equivalent to GRBG Bayer pattern + COLOR_BayerRG2BGR_EA = 137, //!< equivalent to BGGR Bayer pattern + COLOR_BayerGR2BGR_EA = 138, //!< equivalent to GBRG Bayer pattern - COLOR_BayerBG2RGB_EA = COLOR_BayerRG2BGR_EA, - COLOR_BayerGB2RGB_EA = COLOR_BayerGR2BGR_EA, - COLOR_BayerRG2RGB_EA = COLOR_BayerBG2BGR_EA, - COLOR_BayerGR2RGB_EA = COLOR_BayerGB2BGR_EA, + COLOR_BayerRGGB2BGR_EA = COLOR_BayerBG2BGR_EA, + COLOR_BayerGRBG2BGR_EA = COLOR_BayerGB2BGR_EA, + COLOR_BayerBGGR2BGR_EA = COLOR_BayerRG2BGR_EA, + COLOR_BayerGBRG2BGR_EA = COLOR_BayerGR2BGR_EA, + + COLOR_BayerRGGB2RGB_EA = COLOR_BayerBGGR2BGR_EA, + COLOR_BayerGRBG2RGB_EA = COLOR_BayerGBRG2BGR_EA, + COLOR_BayerBGGR2RGB_EA = COLOR_BayerRGGB2BGR_EA, + COLOR_BayerGBRG2RGB_EA = COLOR_BayerGRBG2BGR_EA, + + COLOR_BayerBG2RGB_EA = COLOR_BayerRG2BGR_EA, //!< equivalent to RGGB Bayer pattern + COLOR_BayerGB2RGB_EA = COLOR_BayerGR2BGR_EA, //!< equivalent to GRBG Bayer pattern + COLOR_BayerRG2RGB_EA = COLOR_BayerBG2BGR_EA, //!< equivalent to BGGR Bayer pattern + COLOR_BayerGR2RGB_EA = COLOR_BayerGB2BGR_EA, //!< equivalent to GBRG Bayer pattern //! Demosaicing with alpha channel - COLOR_BayerBG2BGRA = 139, - COLOR_BayerGB2BGRA = 140, - COLOR_BayerRG2BGRA = 141, - COLOR_BayerGR2BGRA = 142, + COLOR_BayerBG2BGRA = 139, //!< equivalent to RGGB Bayer pattern + COLOR_BayerGB2BGRA = 140, //!< equivalent to GRBG Bayer pattern + COLOR_BayerRG2BGRA = 141, //!< equivalent to BGGR Bayer pattern + COLOR_BayerGR2BGRA = 142, //!< equivalent to GBRG Bayer pattern - COLOR_BayerBG2RGBA = COLOR_BayerRG2BGRA, - COLOR_BayerGB2RGBA = COLOR_BayerGR2BGRA, - COLOR_BayerRG2RGBA = COLOR_BayerBG2BGRA, - COLOR_BayerGR2RGBA = COLOR_BayerGB2BGRA, + COLOR_BayerRGGB2BGRA = COLOR_BayerBG2BGRA, + COLOR_BayerGRBG2BGRA = COLOR_BayerGB2BGRA, + COLOR_BayerBGGR2BGRA = COLOR_BayerRG2BGRA, + COLOR_BayerGBRG2BGRA = COLOR_BayerGR2BGRA, + + COLOR_BayerRGGB2RGBA = COLOR_BayerBGGR2BGRA, + COLOR_BayerGRBG2RGBA = COLOR_BayerGBRG2BGRA, + COLOR_BayerBGGR2RGBA = COLOR_BayerRGGB2BGRA, + COLOR_BayerGBRG2RGBA = COLOR_BayerGRBG2BGRA, + + COLOR_BayerBG2RGBA = COLOR_BayerRG2BGRA, //!< equivalent to RGGB Bayer pattern + COLOR_BayerGB2RGBA = COLOR_BayerGR2BGRA, //!< equivalent to GRBG Bayer pattern + COLOR_BayerRG2RGBA = COLOR_BayerBG2BGRA, //!< equivalent to BGGR Bayer pattern + COLOR_BayerGR2RGBA = COLOR_BayerGB2BGRA, //!< equivalent to GBRG Bayer pattern COLOR_COLORCVT_MAX = 143 }; From b3e16c6423d34a39a2c1d311ba344f0ceeef6253 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 2 Nov 2021 19:22:47 +0000 Subject: [PATCH 047/226] videoio(dshow): eliminate build warnings from MSVC-Clang --- modules/videoio/src/cap_dshow.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/videoio/src/cap_dshow.cpp b/modules/videoio/src/cap_dshow.cpp index afabe0a194..1dc18e0f3c 100644 --- a/modules/videoio/src/cap_dshow.cpp +++ b/modules/videoio/src/cap_dshow.cpp @@ -93,8 +93,9 @@ Thanks to: #pragma warning(disable: 4995) #endif -#ifdef __MINGW32__ -// MinGW does not understand COM interfaces +#if defined(__clang__) // clang or MSVC clang +#pragma clang diagnostic ignored "-Wnon-virtual-dtor" +#elif defined(__GNUC__) // MinGW #pragma GCC diagnostic ignored "-Wnon-virtual-dtor" #endif From d484939c02eb5247e803365aa91c8312dd4db93d Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 3 Nov 2021 18:59:36 +0300 Subject: [PATCH 048/226] Merge pull request #20999 from alalek:dnn_replace_deprecated_calls dnn(protobuf): replace deprecated calls * dnn: replace deprecated ByteSize() => ByteSizeLong() * dnn: replace deprecated calls, use GetRepeatedFieldRef --- modules/dnn/src/caffe/caffe_importer.cpp | 8 ++++---- modules/dnn/src/tensorflow/tf_importer.cpp | 20 ++++++++++++++++---- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/modules/dnn/src/caffe/caffe_importer.cpp b/modules/dnn/src/caffe/caffe_importer.cpp index 2d615c448a..0b6c0a6e38 100644 --- a/modules/dnn/src/caffe/caffe_importer.cpp +++ b/modules/dnn/src/caffe/caffe_importer.cpp @@ -49,6 +49,7 @@ #include #include #include +#include #include "caffe_io.hpp" #endif @@ -57,8 +58,7 @@ namespace dnn { CV__DNN_EXPERIMENTAL_NS_BEGIN #ifdef HAVE_PROTOBUF -using ::google::protobuf::RepeatedField; -using ::google::protobuf::RepeatedPtrField; +using ::google::protobuf::RepeatedFieldRef; using ::google::protobuf::Message; using ::google::protobuf::Descriptor; using ::google::protobuf::FieldDescriptor; @@ -136,7 +136,7 @@ public: #define SET_UP_FILED(getter, arrayConstr, gtype) \ if (isRepeated) { \ - const RepeatedField &v = refl->GetRepeatedField(msg, field); \ + const RepeatedFieldRef v = refl->GetRepeatedFieldRef(msg, field); \ params.set(name, DictValue::arrayConstr(v.begin(), (int)v.size())); \ } \ else { \ @@ -168,7 +168,7 @@ public: break; case FieldDescriptor::CPPTYPE_STRING: if (isRepeated) { - const RepeatedPtrField &v = refl->GetRepeatedPtrField(msg, field); + const RepeatedFieldRef v = refl->GetRepeatedFieldRef(msg, field); params.set(name, DictValue::arrayString(v.begin(), (int)v.size())); } else { diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index 16c5c16308..1a01ac87d6 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -2739,14 +2739,21 @@ DataLayout TFImporter::predictOutputDataLayout(const tensorflow::NodeDef& layer) void TFImporter::populateNet() { - CV_Assert(netBin.ByteSize() || netTxt.ByteSize()); +#if GOOGLE_PROTOBUF_VERSION < 3005000 + size_t netBinSize = saturate_cast(netBin.ByteSize()); + size_t netTxtSize = saturate_cast(netTxt.ByteSize()); +#else + size_t netBinSize = netBin.ByteSizeLong(); + size_t netTxtSize = netTxt.ByteSizeLong(); +#endif + CV_Assert(netBinSize || netTxtSize); CV_LOG_INFO(NULL, "DNN/TF: parsing model" << (netBin.has_versions() ? cv::format(" produced by TF v%d (min_consumer=%d)", (int)netBin.versions().producer(), (int)netBin.versions().min_consumer()) : cv::String(" (N/A version info)")) << ". Number of nodes = " << netBin.node_size() ); - if (netTxt.ByteSize()) + if (netTxtSize) { CV_LOG_INFO(NULL, "DNN/TF: parsing config" << (netTxt.has_versions() ? cv::format(" produced by TF v%d (min_consumer=%d)", (int)netTxt.versions().producer(), (int)netTxt.versions().min_consumer()) : cv::String(" (N/A version info)")) @@ -2775,7 +2782,7 @@ void TFImporter::populateNet() CV_LOG_DEBUG(NULL, "DNN/TF: sortByExecutionOrder(model) => " << netBin.node_size() << " nodes"); } - tensorflow::GraphDef& net = netTxt.ByteSize() != 0 ? netTxt : netBin; + tensorflow::GraphDef& net = netTxtSize != 0 ? netTxt : netBin; int layersSize = net.node_size(); @@ -2873,7 +2880,12 @@ void TFImporter::addPermuteLayer(const int* order, const std::string& permName, void TFImporter::parseNode(const tensorflow::NodeDef& layer) { - tensorflow::GraphDef& net = netTxt.ByteSize() != 0 ? netTxt : netBin; +#if GOOGLE_PROTOBUF_VERSION < 3005000 + size_t netTxtSize = saturate_cast(netTxt.ByteSize()); +#else + size_t netTxtSize = netTxt.ByteSizeLong(); +#endif + tensorflow::GraphDef& net = netTxtSize != 0 ? netTxt : netBin; const std::string& name = layer.name(); const std::string& type = layer.op(); From 66dd8712886e7e531bc02fcf8ae53427b71e4194 Mon Sep 17 00:00:00 2001 From: cpengu Date: Fri, 15 Oct 2021 11:11:35 +0800 Subject: [PATCH 049/226] Update qrcode.cpp Fixed issue #20880, QRDetect::searchHorizontalLines() boundary condition will skip the matched qrcode near the end --- modules/objdetect/src/qrcode.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/objdetect/src/qrcode.cpp b/modules/objdetect/src/qrcode.cpp index 3cc130ac89..1479aabf08 100644 --- a/modules/objdetect/src/qrcode.cpp +++ b/modules/objdetect/src/qrcode.cpp @@ -200,7 +200,7 @@ vector QRDetect::searchHorizontalLines() } } pixels_position.push_back(width_bin_barcode - 1); - for (size_t i = 2; i < pixels_position.size() - 4; i+=2) + for (size_t i = 2; i < pixels_position.size() - 3; i+=2) { test_lines[0] = static_cast(pixels_position[i - 1] - pixels_position[i - 2]); test_lines[1] = static_cast(pixels_position[i ] - pixels_position[i - 1]); From 8e72e1ed88215b91157393e959e1de3d559fc785 Mon Sep 17 00:00:00 2001 From: APrigarina Date: Wed, 3 Nov 2021 20:04:27 +0300 Subject: [PATCH 050/226] add test case for QR detect fix --- modules/objdetect/test/test_qrcode.cpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/modules/objdetect/test/test_qrcode.cpp b/modules/objdetect/test/test_qrcode.cpp index fc55498740..19a9f76260 100644 --- a/modules/objdetect/test/test_qrcode.cpp +++ b/modules/objdetect/test/test_qrcode.cpp @@ -649,6 +649,26 @@ TEST(Objdetect_QRCode_decodeMulti, check_output_parameters_type_19363) #endif } +TEST(Objdetect_QRCode_detect, detect_regression_20882) +{ + const std::string name_current_image = "qrcode_near_the_end.jpg"; + const std::string root = "qrcode/"; + + std::string image_path = findDataFile(root + name_current_image); + Mat src = imread(image_path); + ASSERT_FALSE(src.empty()) << "Can't read image: " << image_path; + + QRCodeDetector qrcode; + std::vector corners; + Mat straight_barcode; + cv::String decoded_info; + EXPECT_TRUE(qrcode.detect(src, corners)); + EXPECT_TRUE(!corners.empty()); +#ifdef HAVE_QUIRC + EXPECT_NO_THROW(qrcode.decode(src, corners, straight_barcode)); +#endif +} + TEST(Objdetect_QRCode_basic, not_found_qrcode) { std::vector corners; From c1d61c88e9636bb0e7eb8d2ae1af78160af03f83 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 4 Nov 2021 09:59:19 +0000 Subject: [PATCH 051/226] dnn(cmake): don't hijack OpenCL options with Tengine --- modules/dnn/CMakeLists.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/dnn/CMakeLists.txt b/modules/dnn/CMakeLists.txt index 2e97b88663..9dc048f5b6 100644 --- a/modules/dnn/CMakeLists.txt +++ b/modules/dnn/CMakeLists.txt @@ -13,9 +13,6 @@ ocv_add_dispatched_file_force_all("layers/layers_common" AVX AVX2 AVX512_SKX) ocv_add_module(dnn opencv_core opencv_imgproc WRAP python java js) ocv_option(OPENCV_DNN_OPENCL "Build with OpenCL support" HAVE_OPENCL AND NOT APPLE) -if(HAVE_TENGINE) - add_definitions(-DHAVE_TENGINE=1) -endif() if(OPENCV_DNN_OPENCL AND HAVE_OPENCL) add_definitions(-DCV_OCL4DNN=1) @@ -23,6 +20,10 @@ else() ocv_cmake_hook_append(INIT_MODULE_SOURCES_opencv_dnn "${CMAKE_CURRENT_LIST_DIR}/cmake/hooks/INIT_MODULE_SOURCES_opencv_dnn.cmake") endif() +if(HAVE_TENGINE) + add_definitions(-DHAVE_TENGINE=1) +endif() + if(MSVC) add_definitions( -D_CRT_SECURE_NO_WARNINGS=1 ) ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4244 /wd4267 /wd4018 /wd4355 /wd4800 /wd4251 /wd4996 /wd4146 From 562f2375c5d6ce9c3366fbdec774088fbf5f5759 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 4 Nov 2021 13:26:33 +0000 Subject: [PATCH 052/226] dnn(test): skip tests with high memory usage - 32-bit configuration may fail due to memory fragmentation --- modules/dnn/test/test_model.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/modules/dnn/test/test_model.cpp b/modules/dnn/test/test_model.cpp index 6ac9702c69..4c08cc30b9 100644 --- a/modules/dnn/test/test_model.cpp +++ b/modules/dnn/test/test_model.cpp @@ -282,7 +282,10 @@ TEST_P(Test_Model, Classify) TEST_P(Test_Model, DetectRegion) { - applyTestTag(CV_TEST_TAG_LONG, CV_TEST_TAG_MEMORY_1GB); + applyTestTag( + CV_TEST_TAG_LONG, + CV_TEST_TAG_MEMORY_2GB + ); #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) @@ -332,7 +335,10 @@ TEST_P(Test_Model, DetectRegion) TEST_P(Test_Model, DetectRegionWithNmsAcrossClasses) { - applyTestTag(CV_TEST_TAG_LONG, CV_TEST_TAG_MEMORY_1GB); + applyTestTag( + CV_TEST_TAG_LONG, + CV_TEST_TAG_MEMORY_2GB + ); #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) @@ -582,6 +588,10 @@ TEST_P(Test_Model, Detection_normalized) TEST_P(Test_Model, Segmentation) { + applyTestTag( + CV_TEST_TAG_MEMORY_2GB + ); + std::string inp = _tf("dog416.png"); std::string weights_file = _tf("fcn8s-heavy-pascal.prototxt"); std::string config_file = _tf("fcn8s-heavy-pascal.caffemodel", false); From ffd010767fe401f87a747c0e7f935ddb7da4c63b Mon Sep 17 00:00:00 2001 From: Vincent Rabaud Date: Thu, 4 Nov 2021 22:48:10 +0100 Subject: [PATCH 053/226] Only use fma functions when CV_FMA3 is set. In practice, processors offering AVX2/AVX512 also FMA, that is why it got unnoticed. --- .../core/include/opencv2/core/hal/intrin_avx.hpp | 14 +++++++++++++- .../include/opencv2/core/hal/intrin_avx512.hpp | 14 +++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/modules/core/include/opencv2/core/hal/intrin_avx.hpp b/modules/core/include/opencv2/core/hal/intrin_avx.hpp index 54e8927192..7fbf6f1314 100644 --- a/modules/core/include/opencv2/core/hal/intrin_avx.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_avx.hpp @@ -1390,11 +1390,21 @@ OPENCV_HAL_IMPL_AVX_CHECK_SHORT(v_int16x16) ////////// Other math ///////// /** Some frequent operations **/ +#if CV_FMA3 #define OPENCV_HAL_IMPL_AVX_MULADD(_Tpvec, suffix) \ inline _Tpvec v_fma(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ { return _Tpvec(_mm256_fmadd_##suffix(a.val, b.val, c.val)); } \ inline _Tpvec v_muladd(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ - { return _Tpvec(_mm256_fmadd_##suffix(a.val, b.val, c.val)); } \ + { return _Tpvec(_mm256_fmadd_##suffix(a.val, b.val, c.val)); } +#else +#define OPENCV_HAL_IMPL_AVX_MULADD(_Tpvec, suffix) \ + inline _Tpvec v_fma(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ + { return _Tpvec(_mm256_add_##suffix(_mm256_mul_##suffix(a.val, b.val), c.val)); } \ + inline _Tpvec v_muladd(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ + { return _Tpvec(_mm256_add_##suffix(_mm256_mul_##suffix(a.val, b.val), c.val)); } +#endif + +#define OPENCV_HAL_IMPL_AVX_MISC(_Tpvec, suffix) \ inline _Tpvec v_sqrt(const _Tpvec& x) \ { return _Tpvec(_mm256_sqrt_##suffix(x.val)); } \ inline _Tpvec v_sqr_magnitude(const _Tpvec& a, const _Tpvec& b) \ @@ -1404,6 +1414,8 @@ OPENCV_HAL_IMPL_AVX_CHECK_SHORT(v_int16x16) OPENCV_HAL_IMPL_AVX_MULADD(v_float32x8, ps) OPENCV_HAL_IMPL_AVX_MULADD(v_float64x4, pd) +OPENCV_HAL_IMPL_AVX_MISC(v_float32x8, ps) +OPENCV_HAL_IMPL_AVX_MISC(v_float64x4, pd) inline v_int32x8 v_fma(const v_int32x8& a, const v_int32x8& b, const v_int32x8& c) { diff --git a/modules/core/include/opencv2/core/hal/intrin_avx512.hpp b/modules/core/include/opencv2/core/hal/intrin_avx512.hpp index 75a3bd4b85..d20d6dd1ff 100644 --- a/modules/core/include/opencv2/core/hal/intrin_avx512.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_avx512.hpp @@ -1385,11 +1385,21 @@ inline v_uint64x8 v_popcount(const v_uint64x8& a) { return v_popcount(v_reinte ////////// Other math ///////// /** Some frequent operations **/ +#if CV_FMA3 #define OPENCV_HAL_IMPL_AVX512_MULADD(_Tpvec, suffix) \ inline _Tpvec v_fma(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ { return _Tpvec(_mm512_fmadd_##suffix(a.val, b.val, c.val)); } \ inline _Tpvec v_muladd(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ - { return _Tpvec(_mm512_fmadd_##suffix(a.val, b.val, c.val)); } \ + { return _Tpvec(_mm512_fmadd_##suffix(a.val, b.val, c.val)); } +#else +#define OPENCV_HAL_IMPL_AVX512_MULADD(_Tpvec, suffix) \ + inline _Tpvec v_fma(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ + { return _Tpvec(_mm512_add_##suffix(_mm512_mul_##suffix(a.val, b.val), c.val)); } \ + inline _Tpvec v_muladd(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ + { return _Tpvec(_mm512_add_##suffix(_mm512_mul_##suffix(a.val, b.val), c.val)); } +#endif + +#define OPENCV_HAL_IMPL_AVX512_MISC(_Tpvec, suffix) \ inline _Tpvec v_sqrt(const _Tpvec& x) \ { return _Tpvec(_mm512_sqrt_##suffix(x.val)); } \ inline _Tpvec v_sqr_magnitude(const _Tpvec& a, const _Tpvec& b) \ @@ -1399,6 +1409,8 @@ inline v_uint64x8 v_popcount(const v_uint64x8& a) { return v_popcount(v_reinte OPENCV_HAL_IMPL_AVX512_MULADD(v_float32x16, ps) OPENCV_HAL_IMPL_AVX512_MULADD(v_float64x8, pd) +OPENCV_HAL_IMPL_AVX512_MISC(v_float32x16, ps) +OPENCV_HAL_IMPL_AVX512_MISC(v_float64x8, pd) inline v_int32x16 v_fma(const v_int32x16& a, const v_int32x16& b, const v_int32x16& c) { return a * b + c; } From 968d94d4174c75dd8debfa62eff050dc0b9b42a8 Mon Sep 17 00:00:00 2001 From: Nikolaos Pappas Date: Wed, 3 Nov 2021 22:13:30 +0000 Subject: [PATCH 054/226] Fix trackbar in falsecolor cpp sample --- samples/cpp/falsecolor.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/samples/cpp/falsecolor.cpp b/samples/cpp/falsecolor.cpp index f73ffad4ce..bfe43a72ca 100644 --- a/samples/cpp/falsecolor.cpp +++ b/samples/cpp/falsecolor.cpp @@ -16,14 +16,14 @@ struct ParamColorMap { String winName="False color"; static const String ColorMaps[] = { "Autumn", "Bone", "Jet", "Winter", "Rainbow", "Ocean", "Summer", "Spring", "Cool", "HSV", "Pink", "Hot", "Parula", "Magma", "Inferno", "Plasma", "Viridis", - "Cividis", "Twilight", "Twilight Shifted", "Turbo", "User defined (random)" }; + "Cividis", "Twilight", "Twilight Shifted", "Turbo", "Deep Green", "User defined (random)" }; static void TrackColorMap(int x, void *r) { ParamColorMap *p = (ParamColorMap*)r; Mat dst; p->iColormap= x; - if (x == COLORMAP_TURBO + 1) + if (x == COLORMAP_DEEPGREEN + 1) { Mat lutRND(256, 1, CV_8UC3); randu(lutRND, Scalar(0, 0, 0), Scalar(255, 255, 255)); @@ -97,10 +97,10 @@ int main(int argc, char** argv) imshow("Gray image",img); namedWindow(winName); - createTrackbar("colormap", winName,&p.iColormap,1,TrackColorMap,(void*)&p); + createTrackbar("colormap", winName, NULL, COLORMAP_DEEPGREEN + 1, TrackColorMap, (void*)&p); setTrackbarMin("colormap", winName, COLORMAP_AUTUMN); - setTrackbarMax("colormap", winName, COLORMAP_TURBO+1); - setTrackbarPos("colormap", winName, -1); + setTrackbarMax("colormap", winName, COLORMAP_DEEPGREEN + 1); + setTrackbarPos("colormap", winName, COLORMAP_AUTUMN); TrackColorMap(0, (void*)&p); From a2716712ab9efcefe7dc0f3c83f5f3d5f1cd402d Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 6 Nov 2021 18:16:42 +0300 Subject: [PATCH 055/226] highgui(win32): fix trackbar setRange --- modules/highgui/src/window_w32.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/highgui/src/window_w32.cpp b/modules/highgui/src/window_w32.cpp index 43a19baf89..76f320f19f 100644 --- a/modules/highgui/src/window_w32.cpp +++ b/modules/highgui/src/window_w32.cpp @@ -2984,7 +2984,7 @@ public: CvTrackbar& trackbar = *trackbar_ptr; CV_CheckLE(range.start, range.end, "Invalid trackbar range"); trackbar.minval = range.start; - trackbar.maxval = range.start; + trackbar.maxval = range.end; SendMessage(trackbar.hwnd, TBM_SETRANGEMIN, (WPARAM)TRUE, (LPARAM)trackbar.minval); SendMessage(trackbar.hwnd, TBM_SETRANGEMAX, (WPARAM)TRUE, (LPARAM)trackbar.maxval); } From fa5c7a9e75ed5b87dbcce17a2a5164acec65f5c7 Mon Sep 17 00:00:00 2001 From: Lukas-Alexander Weber <32765578+lukasalexanderweber@users.noreply.github.com> Date: Mon, 8 Nov 2021 12:54:06 +0100 Subject: [PATCH 056/226] Merge pull request #21020 from lukasalexanderweber:squash Created Stitching Tool based on stitching_detailed.py --- apps/opencv_stitching_tool/README.md | 3 + .../opencv_stitching/.gitignore | 4 + .../opencv_stitching/__init__.py | 0 .../opencv_stitching/blender.py | 48 +++ .../opencv_stitching/camera_adjuster.py | 49 +++ .../opencv_stitching/camera_estimator.py | 27 ++ .../opencv_stitching/camera_wave_corrector.py | 28 ++ .../exposure_error_compensator.py | 40 ++ .../opencv_stitching/feature_detector.py | 44 ++ .../opencv_stitching/feature_matcher.py | 98 +++++ .../opencv_stitching/image_handler.py | 94 ++++ .../opencv_stitching/megapix_downscaler.py | 12 + .../opencv_stitching/megapix_scaler.py | 27 ++ .../opencv_stitching/panorama_estimation.py | 27 ++ .../opencv_stitching/seam_finder.py | 127 ++++++ .../opencv_stitching/stitcher.py | 207 +++++++++ .../opencv_stitching/stitching_error.py | 2 + .../opencv_stitching/subsetter.py | 95 ++++ .../opencv_stitching/test/.gitignore | 13 + .../test/SAMPLE_IMAGES_TO_DOWNLOAD.txt | 5 + .../test/stitching_detailed.py | 406 ++++++++++++++++++ .../opencv_stitching/test/test_composition.py | 67 +++ .../opencv_stitching/test/test_matcher.py | 47 ++ .../test/test_megapix_scaler.py | 59 +++ .../opencv_stitching/test/test_performance.py | 65 +++ .../test/test_registration.py | 100 +++++ .../opencv_stitching/test/test_stitcher.py | 108 +++++ .../opencv_stitching/timelapser.py | 50 +++ .../opencv_stitching/warper.py | 71 +++ .../opencv_stitching_tool.py | 232 ++++++++++ 30 files changed, 2155 insertions(+) create mode 100644 apps/opencv_stitching_tool/README.md create mode 100644 apps/opencv_stitching_tool/opencv_stitching/.gitignore create mode 100644 apps/opencv_stitching_tool/opencv_stitching/__init__.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/blender.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/camera_adjuster.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/camera_estimator.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/camera_wave_corrector.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/exposure_error_compensator.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/feature_detector.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/feature_matcher.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/image_handler.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/megapix_downscaler.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/megapix_scaler.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/panorama_estimation.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/seam_finder.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/stitcher.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/stitching_error.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/subsetter.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/test/.gitignore create mode 100644 apps/opencv_stitching_tool/opencv_stitching/test/SAMPLE_IMAGES_TO_DOWNLOAD.txt create mode 100644 apps/opencv_stitching_tool/opencv_stitching/test/stitching_detailed.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/test/test_composition.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/test/test_matcher.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/test/test_megapix_scaler.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/test/test_performance.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/test/test_registration.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/test/test_stitcher.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/timelapser.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/warper.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching_tool.py diff --git a/apps/opencv_stitching_tool/README.md b/apps/opencv_stitching_tool/README.md new file mode 100644 index 0000000000..1cf3f019d0 --- /dev/null +++ b/apps/opencv_stitching_tool/README.md @@ -0,0 +1,3 @@ +## In-Depth Stitching Tool for experiments and research + +Visit [opencv_stitching_tutorial](https://github.com/lukasalexanderweber/opencv_stitching_tutorial) for a detailed Tutorial diff --git a/apps/opencv_stitching_tool/opencv_stitching/.gitignore b/apps/opencv_stitching_tool/opencv_stitching/.gitignore new file mode 100644 index 0000000000..1f4d07f716 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/.gitignore @@ -0,0 +1,4 @@ +# python binary files +*.pyc +__pycache__ +.pylint* diff --git a/apps/opencv_stitching_tool/opencv_stitching/__init__.py b/apps/opencv_stitching_tool/opencv_stitching/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/apps/opencv_stitching_tool/opencv_stitching/blender.py b/apps/opencv_stitching_tool/opencv_stitching/blender.py new file mode 100644 index 0000000000..04e6efe528 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/blender.py @@ -0,0 +1,48 @@ +import cv2 as cv +import numpy as np + + +class Blender: + + BLENDER_CHOICES = ('multiband', 'feather', 'no',) + DEFAULT_BLENDER = 'multiband' + DEFAULT_BLEND_STRENGTH = 5 + + def __init__(self, blender_type=DEFAULT_BLENDER, + blend_strength=DEFAULT_BLEND_STRENGTH): + self.blender_type = blender_type + self.blend_strength = blend_strength + self.blender = None + + def prepare(self, corners, sizes): + dst_sz = cv.detail.resultRoi(corners=corners, sizes=sizes) + blend_width = (np.sqrt(dst_sz[2] * dst_sz[3]) * + self.blend_strength / 100) + + if self.blender_type == 'no' or blend_width < 1: + self.blender = cv.detail.Blender_createDefault( + cv.detail.Blender_NO + ) + + elif self.blender_type == "multiband": + self.blender = cv.detail_MultiBandBlender() + self.blender.setNumBands((np.log(blend_width) / + np.log(2.) - 1.).astype(np.int)) + + elif self.blender_type == "feather": + self.blender = cv.detail_FeatherBlender() + self.blender.setSharpness(1. / blend_width) + + self.blender.prepare(dst_sz) + + def feed(self, img, mask, corner): + """https://docs.opencv.org/master/d6/d4a/classcv_1_1detail_1_1Blender.html#a64837308bcf4e414a6219beff6cbe37a""" # noqa + self.blender.feed(cv.UMat(img.astype(np.int16)), mask, corner) + + def blend(self): + """https://docs.opencv.org/master/d6/d4a/classcv_1_1detail_1_1Blender.html#aa0a91ce0d6046d3a63e0123cbb1b5c00""" # noqa + result = None + result_mask = None + result, result_mask = self.blender.blend(result, result_mask) + result = cv.convertScaleAbs(result) + return result diff --git a/apps/opencv_stitching_tool/opencv_stitching/camera_adjuster.py b/apps/opencv_stitching_tool/opencv_stitching/camera_adjuster.py new file mode 100644 index 0000000000..03aa834d31 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/camera_adjuster.py @@ -0,0 +1,49 @@ +from collections import OrderedDict +import cv2 as cv +import numpy as np + +from .stitching_error import StitchingError + + +class CameraAdjuster: + """https://docs.opencv.org/master/d5/d56/classcv_1_1detail_1_1BundleAdjusterBase.html""" # noqa + + CAMERA_ADJUSTER_CHOICES = OrderedDict() + CAMERA_ADJUSTER_CHOICES['ray'] = cv.detail_BundleAdjusterRay + CAMERA_ADJUSTER_CHOICES['reproj'] = cv.detail_BundleAdjusterReproj + CAMERA_ADJUSTER_CHOICES['affine'] = cv.detail_BundleAdjusterAffinePartial + CAMERA_ADJUSTER_CHOICES['no'] = cv.detail_NoBundleAdjuster + + DEFAULT_CAMERA_ADJUSTER = list(CAMERA_ADJUSTER_CHOICES.keys())[0] + DEFAULT_REFINEMENT_MASK = "xxxxx" + + def __init__(self, + adjuster=DEFAULT_CAMERA_ADJUSTER, + refinement_mask=DEFAULT_REFINEMENT_MASK): + + self.adjuster = CameraAdjuster.CAMERA_ADJUSTER_CHOICES[adjuster]() + self.set_refinement_mask(refinement_mask) + self.adjuster.setConfThresh(1) + + def set_refinement_mask(self, refinement_mask): + mask_matrix = np.zeros((3, 3), np.uint8) + if refinement_mask[0] == 'x': + mask_matrix[0, 0] = 1 + if refinement_mask[1] == 'x': + mask_matrix[0, 1] = 1 + if refinement_mask[2] == 'x': + mask_matrix[0, 2] = 1 + if refinement_mask[3] == 'x': + mask_matrix[1, 1] = 1 + if refinement_mask[4] == 'x': + mask_matrix[1, 2] = 1 + self.adjuster.setRefinementMask(mask_matrix) + + def adjust(self, features, pairwise_matches, estimated_cameras): + b, cameras = self.adjuster.apply(features, + pairwise_matches, + estimated_cameras) + if not b: + raise StitchingError("Camera parameters adjusting failed.") + + return cameras diff --git a/apps/opencv_stitching_tool/opencv_stitching/camera_estimator.py b/apps/opencv_stitching_tool/opencv_stitching/camera_estimator.py new file mode 100644 index 0000000000..8520eb0ddf --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/camera_estimator.py @@ -0,0 +1,27 @@ +from collections import OrderedDict +import cv2 as cv +import numpy as np + +from .stitching_error import StitchingError + + +class CameraEstimator: + + CAMERA_ESTIMATOR_CHOICES = OrderedDict() + CAMERA_ESTIMATOR_CHOICES['homography'] = cv.detail_HomographyBasedEstimator + CAMERA_ESTIMATOR_CHOICES['affine'] = cv.detail_AffineBasedEstimator + + DEFAULT_CAMERA_ESTIMATOR = list(CAMERA_ESTIMATOR_CHOICES.keys())[0] + + def __init__(self, estimator=DEFAULT_CAMERA_ESTIMATOR, **kwargs): + self.estimator = CameraEstimator.CAMERA_ESTIMATOR_CHOICES[estimator]( + **kwargs + ) + + def estimate(self, features, pairwise_matches): + b, cameras = self.estimator.apply(features, pairwise_matches, None) + if not b: + raise StitchingError("Homography estimation failed.") + for cam in cameras: + cam.R = cam.R.astype(np.float32) + return cameras diff --git a/apps/opencv_stitching_tool/opencv_stitching/camera_wave_corrector.py b/apps/opencv_stitching_tool/opencv_stitching/camera_wave_corrector.py new file mode 100644 index 0000000000..6a9142d7f5 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/camera_wave_corrector.py @@ -0,0 +1,28 @@ +from collections import OrderedDict +import cv2 as cv +import numpy as np + + +class WaveCorrector: + """https://docs.opencv.org/master/d7/d74/group__stitching__rotation.html#ga83b24d4c3e93584986a56d9e43b9cf7f""" # noqa + WAVE_CORRECT_CHOICES = OrderedDict() + WAVE_CORRECT_CHOICES['horiz'] = cv.detail.WAVE_CORRECT_HORIZ + WAVE_CORRECT_CHOICES['vert'] = cv.detail.WAVE_CORRECT_VERT + WAVE_CORRECT_CHOICES['auto'] = cv.detail.WAVE_CORRECT_AUTO + WAVE_CORRECT_CHOICES['no'] = None + + DEFAULT_WAVE_CORRECTION = list(WAVE_CORRECT_CHOICES.keys())[0] + + def __init__(self, wave_correct_kind=DEFAULT_WAVE_CORRECTION): + self.wave_correct_kind = WaveCorrector.WAVE_CORRECT_CHOICES[ + wave_correct_kind + ] + + def correct(self, cameras): + if self.wave_correct_kind is not None: + rmats = [np.copy(cam.R) for cam in cameras] + rmats = cv.detail.waveCorrect(rmats, self.wave_correct_kind) + for idx, cam in enumerate(cameras): + cam.R = rmats[idx] + return cameras + return cameras diff --git a/apps/opencv_stitching_tool/opencv_stitching/exposure_error_compensator.py b/apps/opencv_stitching_tool/opencv_stitching/exposure_error_compensator.py new file mode 100644 index 0000000000..36e0292091 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/exposure_error_compensator.py @@ -0,0 +1,40 @@ +from collections import OrderedDict +import cv2 as cv + + +class ExposureErrorCompensator: + + COMPENSATOR_CHOICES = OrderedDict() + COMPENSATOR_CHOICES['gain_blocks'] = cv.detail.ExposureCompensator_GAIN_BLOCKS # noqa + COMPENSATOR_CHOICES['gain'] = cv.detail.ExposureCompensator_GAIN + COMPENSATOR_CHOICES['channel'] = cv.detail.ExposureCompensator_CHANNELS + COMPENSATOR_CHOICES['channel_blocks'] = cv.detail.ExposureCompensator_CHANNELS_BLOCKS # noqa + COMPENSATOR_CHOICES['no'] = cv.detail.ExposureCompensator_NO + + DEFAULT_COMPENSATOR = list(COMPENSATOR_CHOICES.keys())[0] + DEFAULT_NR_FEEDS = 1 + DEFAULT_BLOCK_SIZE = 32 + + def __init__(self, + compensator=DEFAULT_COMPENSATOR, + nr_feeds=DEFAULT_NR_FEEDS, + block_size=DEFAULT_BLOCK_SIZE): + + if compensator == 'channel': + self.compensator = cv.detail_ChannelsCompensator(nr_feeds) + elif compensator == 'channel_blocks': + self.compensator = cv.detail_BlocksChannelsCompensator( + block_size, block_size, nr_feeds + ) + else: + self.compensator = cv.detail.ExposureCompensator_createDefault( + ExposureErrorCompensator.COMPENSATOR_CHOICES[compensator] + ) + + def feed(self, *args): + """https://docs.opencv.org/master/d2/d37/classcv_1_1detail_1_1ExposureCompensator.html#ae6b0cc69a7bc53818ddea53eddb6bdba""" # noqa + self.compensator.feed(*args) + + def apply(self, *args): + """https://docs.opencv.org/master/d2/d37/classcv_1_1detail_1_1ExposureCompensator.html#a473eaf1e585804c08d77c91e004f93aa""" # noqa + return self.compensator.apply(*args) diff --git a/apps/opencv_stitching_tool/opencv_stitching/feature_detector.py b/apps/opencv_stitching_tool/opencv_stitching/feature_detector.py new file mode 100644 index 0000000000..995517b01b --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/feature_detector.py @@ -0,0 +1,44 @@ +from collections import OrderedDict +import cv2 as cv + + +class FeatureDetector: + DETECTOR_CHOICES = OrderedDict() + try: + cv.xfeatures2d_SURF.create() # check if the function can be called + DETECTOR_CHOICES['surf'] = cv.xfeatures2d_SURF.create + except (AttributeError, cv.error): + print("SURF not available") + + # if SURF not available, ORB is default + DETECTOR_CHOICES['orb'] = cv.ORB.create + + try: + DETECTOR_CHOICES['sift'] = cv.SIFT_create + except AttributeError: + print("SIFT not available") + + try: + DETECTOR_CHOICES['brisk'] = cv.BRISK_create + except AttributeError: + print("BRISK not available") + + try: + DETECTOR_CHOICES['akaze'] = cv.AKAZE_create + except AttributeError: + print("AKAZE not available") + + DEFAULT_DETECTOR = list(DETECTOR_CHOICES.keys())[0] + + def __init__(self, detector=DEFAULT_DETECTOR, **kwargs): + self.detector = FeatureDetector.DETECTOR_CHOICES[detector](**kwargs) + + def detect_features(self, img, *args, **kwargs): + return cv.detail.computeImageFeatures2(self.detector, img, + *args, **kwargs) + + @staticmethod + def draw_keypoints(img, features, **kwargs): + kwargs.setdefault('color', (0, 255, 0)) + keypoints = features.getKeypoints() + return cv.drawKeypoints(img, keypoints, None, **kwargs) diff --git a/apps/opencv_stitching_tool/opencv_stitching/feature_matcher.py b/apps/opencv_stitching_tool/opencv_stitching/feature_matcher.py new file mode 100644 index 0000000000..8c1d384b7b --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/feature_matcher.py @@ -0,0 +1,98 @@ +import math +import cv2 as cv +import numpy as np + + +class FeatureMatcher: + + MATCHER_CHOICES = ('homography', 'affine') + DEFAULT_MATCHER = 'homography' + DEFAULT_RANGE_WIDTH = -1 + + def __init__(self, + matcher_type=DEFAULT_MATCHER, + range_width=DEFAULT_RANGE_WIDTH, + **kwargs): + + if matcher_type == "affine": + """https://docs.opencv.org/master/d3/dda/classcv_1_1detail_1_1AffineBestOf2NearestMatcher.html""" # noqa + self.matcher = cv.detail_AffineBestOf2NearestMatcher(**kwargs) + elif range_width == -1: + """https://docs.opencv.org/master/d4/d26/classcv_1_1detail_1_1BestOf2NearestMatcher.html""" # noqa + self.matcher = cv.detail.BestOf2NearestMatcher_create(**kwargs) + else: + """https://docs.opencv.org/master/d8/d72/classcv_1_1detail_1_1BestOf2NearestRangeMatcher.html""" # noqa + self.matcher = cv.detail.BestOf2NearestRangeMatcher_create( + range_width, **kwargs + ) + + def match_features(self, features, *args, **kwargs): + pairwise_matches = self.matcher.apply2(features, *args, **kwargs) + self.matcher.collectGarbage() + return pairwise_matches + + @staticmethod + def draw_matches_matrix(imgs, features, matches, conf_thresh=1, + inliers=False, **kwargs): + matches_matrix = FeatureMatcher.get_matches_matrix(matches) + for idx1, idx2 in FeatureMatcher.get_all_img_combinations(len(imgs)): + match = matches_matrix[idx1, idx2] + if match.confidence < conf_thresh: + continue + if inliers: + kwargs['matchesMask'] = match.getInliers() + yield idx1, idx2, FeatureMatcher.draw_matches( + imgs[idx1], features[idx1], + imgs[idx2], features[idx2], + match, + **kwargs + ) + + @staticmethod + def draw_matches(img1, features1, img2, features2, match1to2, **kwargs): + kwargs.setdefault('flags', cv.DrawMatchesFlags_NOT_DRAW_SINGLE_POINTS) + + keypoints1 = features1.getKeypoints() + keypoints2 = features2.getKeypoints() + matches = match1to2.getMatches() + + return cv.drawMatches( + img1, keypoints1, img2, keypoints2, matches, None, **kwargs + ) + + @staticmethod + def get_matches_matrix(pairwise_matches): + return FeatureMatcher.array_in_sqare_matrix(pairwise_matches) + + @staticmethod + def get_confidence_matrix(pairwise_matches): + matches_matrix = FeatureMatcher.get_matches_matrix(pairwise_matches) + match_confs = [[m.confidence for m in row] for row in matches_matrix] + match_conf_matrix = np.array(match_confs) + return match_conf_matrix + + @staticmethod + def array_in_sqare_matrix(array): + matrix_dimension = int(math.sqrt(len(array))) + rows = [] + for i in range(0, len(array), matrix_dimension): + rows.append(array[i:i+matrix_dimension]) + return np.array(rows) + + def get_all_img_combinations(number_imgs): + ii, jj = np.triu_indices(number_imgs, k=1) + for i, j in zip(ii, jj): + yield i, j + + @staticmethod + def get_match_conf(match_conf, feature_detector_type): + if match_conf is None: + match_conf = \ + FeatureMatcher.get_default_match_conf(feature_detector_type) + return match_conf + + @staticmethod + def get_default_match_conf(feature_detector_type): + if feature_detector_type == 'orb': + return 0.3 + return 0.65 diff --git a/apps/opencv_stitching_tool/opencv_stitching/image_handler.py b/apps/opencv_stitching_tool/opencv_stitching/image_handler.py new file mode 100644 index 0000000000..a3b76b288a --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/image_handler.py @@ -0,0 +1,94 @@ +import cv2 as cv + +from .megapix_downscaler import MegapixDownscaler +from .stitching_error import StitchingError + +class ImageHandler: + + DEFAULT_MEDIUM_MEGAPIX = 0.6 + DEFAULT_LOW_MEGAPIX = 0.1 + DEFAULT_FINAL_MEGAPIX = -1 + + def __init__(self, + medium_megapix=DEFAULT_MEDIUM_MEGAPIX, + low_megapix=DEFAULT_LOW_MEGAPIX, + final_megapix=DEFAULT_FINAL_MEGAPIX): + + if medium_megapix < low_megapix: + raise StitchingError("Medium resolution megapix need to be " + "greater or equal than low resolution " + "megapix") + + self.medium_scaler = MegapixDownscaler(medium_megapix) + self.low_scaler = MegapixDownscaler(low_megapix) + self.final_scaler = MegapixDownscaler(final_megapix) + + self.scales_set = False + self.img_names = [] + self.img_sizes = [] + + def set_img_names(self, img_names): + self.img_names = img_names + + def resize_to_medium_resolution(self): + return self.read_and_resize_imgs(self.medium_scaler) + + def resize_to_low_resolution(self, medium_imgs=None): + if medium_imgs and self.scales_set: + return self.resize_medium_to_low(medium_imgs) + return self.read_and_resize_imgs(self.low_scaler) + + def resize_to_final_resolution(self): + return self.read_and_resize_imgs(self.final_scaler) + + def read_and_resize_imgs(self, scaler): + for img, size in self.input_images(): + yield self.resize_img_by_scaler(scaler, size, img) + + def resize_medium_to_low(self, medium_imgs): + for img, size in zip(medium_imgs, self.img_sizes): + yield self.resize_img_by_scaler(self.low_scaler, size, img) + + @staticmethod + def resize_img_by_scaler(scaler, size, img): + desired_size = scaler.get_scaled_img_size(size) + return cv.resize(img, desired_size, + interpolation=cv.INTER_LINEAR_EXACT) + + def input_images(self): + self.img_sizes = [] + for name in self.img_names: + img = self.read_image(name) + size = self.get_image_size(img) + self.img_sizes.append(size) + self.set_scaler_scales() + yield img, size + + @staticmethod + def get_image_size(img): + """(width, height)""" + return (img.shape[1], img.shape[0]) + + @staticmethod + def read_image(img_name): + img = cv.imread(img_name) + if img is None: + raise StitchingError("Cannot read image " + img_name) + return img + + def set_scaler_scales(self): + if not self.scales_set: + first_img_size = self.img_sizes[0] + self.medium_scaler.set_scale_by_img_size(first_img_size) + self.low_scaler.set_scale_by_img_size(first_img_size) + self.final_scaler.set_scale_by_img_size(first_img_size) + self.scales_set = True + + def get_medium_to_final_ratio(self): + return self.final_scaler.scale / self.medium_scaler.scale + + def get_medium_to_low_ratio(self): + return self.low_scaler.scale / self.medium_scaler.scale + + def get_final_to_low_ratio(self): + return self.low_scaler.scale / self.final_scaler.scale diff --git a/apps/opencv_stitching_tool/opencv_stitching/megapix_downscaler.py b/apps/opencv_stitching_tool/opencv_stitching/megapix_downscaler.py new file mode 100644 index 0000000000..f7553acc2e --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/megapix_downscaler.py @@ -0,0 +1,12 @@ +from .megapix_scaler import MegapixScaler + + +class MegapixDownscaler(MegapixScaler): + + @staticmethod + def force_downscale(scale): + return min(1.0, scale) + + def set_scale(self, scale): + scale = self.force_downscale(scale) + super().set_scale(scale) diff --git a/apps/opencv_stitching_tool/opencv_stitching/megapix_scaler.py b/apps/opencv_stitching_tool/opencv_stitching/megapix_scaler.py new file mode 100644 index 0000000000..96d47536f9 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/megapix_scaler.py @@ -0,0 +1,27 @@ +import numpy as np + + +class MegapixScaler: + def __init__(self, megapix): + self.megapix = megapix + self.is_scale_set = False + self.scale = None + + def set_scale_by_img_size(self, img_size): + self.set_scale( + self.get_scale_by_resolution(img_size[0] * img_size[1]) + ) + + def set_scale(self, scale): + self.scale = scale + self.is_scale_set = True + + def get_scale_by_resolution(self, resolution): + if self.megapix > 0: + return np.sqrt(self.megapix * 1e6 / resolution) + return 1.0 + + def get_scaled_img_size(self, img_size): + width = int(round(img_size[0] * self.scale)) + height = int(round(img_size[1] * self.scale)) + return (width, height) diff --git a/apps/opencv_stitching_tool/opencv_stitching/panorama_estimation.py b/apps/opencv_stitching_tool/opencv_stitching/panorama_estimation.py new file mode 100644 index 0000000000..e3a45773ea --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/panorama_estimation.py @@ -0,0 +1,27 @@ +import statistics + + +def estimate_final_panorama_dimensions(cameras, warper, img_handler): + medium_to_final_ratio = img_handler.get_medium_to_final_ratio() + + panorama_scale_determined_on_medium_img = \ + estimate_panorama_scale(cameras) + + panorama_scale = (panorama_scale_determined_on_medium_img * + medium_to_final_ratio) + panorama_corners = [] + panorama_sizes = [] + + for size, camera in zip(img_handler.img_sizes, cameras): + width, height = img_handler.final_scaler.get_scaled_img_size(size) + roi = warper.warp_roi(width, height, camera, panorama_scale, medium_to_final_ratio) + panorama_corners.append(roi[0:2]) + panorama_sizes.append(roi[2:4]) + + return panorama_scale, panorama_corners, panorama_sizes + + +def estimate_panorama_scale(cameras): + focals = [cam.focal for cam in cameras] + panorama_scale = statistics.median(focals) + return panorama_scale diff --git a/apps/opencv_stitching_tool/opencv_stitching/seam_finder.py b/apps/opencv_stitching_tool/opencv_stitching/seam_finder.py new file mode 100644 index 0000000000..675f266d02 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/seam_finder.py @@ -0,0 +1,127 @@ +from collections import OrderedDict +import cv2 as cv +import numpy as np + +from .blender import Blender + + +class SeamFinder: + """https://docs.opencv.org/master/d7/d09/classcv_1_1detail_1_1SeamFinder.html""" # noqa + SEAM_FINDER_CHOICES = OrderedDict() + SEAM_FINDER_CHOICES['dp_color'] = cv.detail_DpSeamFinder('COLOR') + SEAM_FINDER_CHOICES['dp_colorgrad'] = cv.detail_DpSeamFinder('COLOR_GRAD') + SEAM_FINDER_CHOICES['voronoi'] = cv.detail.SeamFinder_createDefault(cv.detail.SeamFinder_VORONOI_SEAM) # noqa + SEAM_FINDER_CHOICES['no'] = cv.detail.SeamFinder_createDefault(cv.detail.SeamFinder_NO) # noqa + + DEFAULT_SEAM_FINDER = list(SEAM_FINDER_CHOICES.keys())[0] + + def __init__(self, finder=DEFAULT_SEAM_FINDER): + self.finder = SeamFinder.SEAM_FINDER_CHOICES[finder] + + def find(self, imgs, corners, masks): + """https://docs.opencv.org/master/d0/dd5/classcv_1_1detail_1_1DpSeamFinder.html#a7914624907986f7a94dd424209a8a609""" # noqa + imgs_float = [img.astype(np.float32) for img in imgs] + return self.finder.find(imgs_float, corners, masks) + + @staticmethod + def resize(seam_mask, mask): + dilated_mask = cv.dilate(seam_mask, None) + resized_seam_mask = cv.resize(dilated_mask, (mask.shape[1], + mask.shape[0]), + 0, 0, cv.INTER_LINEAR_EXACT) + return cv.bitwise_and(resized_seam_mask, mask) + + @staticmethod + def draw_seam_mask(img, seam_mask, color=(0, 0, 0)): + seam_mask = cv.UMat.get(seam_mask) + overlayed_img = np.copy(img) + overlayed_img[seam_mask == 0] = color + return overlayed_img + + @staticmethod + def draw_seam_polygons(panorama, blended_seam_masks, alpha=0.5): + return add_weighted_image(panorama, blended_seam_masks, alpha) + + @staticmethod + def draw_seam_lines(panorama, blended_seam_masks, + linesize=1, color=(0, 0, 255)): + seam_lines = \ + SeamFinder.exctract_seam_lines(blended_seam_masks, linesize) + panorama_with_seam_lines = panorama.copy() + panorama_with_seam_lines[seam_lines == 255] = color + return panorama_with_seam_lines + + @staticmethod + def exctract_seam_lines(blended_seam_masks, linesize=1): + seam_lines = cv.Canny(np.uint8(blended_seam_masks), 100, 200) + seam_indices = (seam_lines == 255).nonzero() + seam_lines = remove_invalid_line_pixels( + seam_indices, seam_lines, blended_seam_masks + ) + kernelsize = linesize + linesize - 1 + kernel = np.ones((kernelsize, kernelsize), np.uint8) + return cv.dilate(seam_lines, kernel) + + @staticmethod + def blend_seam_masks(seam_masks, corners, sizes, colors=[ + (255, 000, 000), # Blue + (000, 000, 255), # Red + (000, 255, 000), # Green + (000, 255, 255), # Yellow + (255, 000, 255), # Magenta + (128, 128, 255), # Pink + (128, 128, 128), # Gray + (000, 000, 128), # Brown + (000, 128, 255)] # Orange + ): + + blender = Blender("no") + blender.prepare(corners, sizes) + + for idx, (seam_mask, size, corner) in enumerate( + zip(seam_masks, sizes, corners)): + if idx+1 > len(colors): + raise ValueError("Not enough default colors! Pass additional " + "colors to \"colors\" parameter") + one_color_img = create_img_by_size(size, colors[idx]) + blender.feed(one_color_img, seam_mask, corner) + + return blender.blend() + + +def create_img_by_size(size, color=(0, 0, 0)): + width, height = size + img = np.zeros((height, width, 3), np.uint8) + img[:] = color + return img + + +def add_weighted_image(img1, img2, alpha): + return cv.addWeighted( + img1, alpha, img2, (1.0 - alpha), 0.0 + ) + + +def remove_invalid_line_pixels(indices, lines, mask): + for x, y in zip(*indices): + if check_if_pixel_or_neighbor_is_black(mask, x, y): + lines[x, y] = 0 + return lines + + +def check_if_pixel_or_neighbor_is_black(img, x, y): + check = [is_pixel_black(img, x, y), + is_pixel_black(img, x+1, y), is_pixel_black(img, x-1, y), + is_pixel_black(img, x, y+1), is_pixel_black(img, x, y-1)] + return any(check) + + +def is_pixel_black(img, x, y): + return np.all(get_pixel_value(img, x, y) == 0) + + +def get_pixel_value(img, x, y): + try: + return img[x, y] + except IndexError: + pass diff --git a/apps/opencv_stitching_tool/opencv_stitching/stitcher.py b/apps/opencv_stitching_tool/opencv_stitching/stitcher.py new file mode 100644 index 0000000000..c08112664f --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/stitcher.py @@ -0,0 +1,207 @@ +from types import SimpleNamespace + +from .image_handler import ImageHandler +from .feature_detector import FeatureDetector +from .feature_matcher import FeatureMatcher +from .subsetter import Subsetter +from .camera_estimator import CameraEstimator +from .camera_adjuster import CameraAdjuster +from .camera_wave_corrector import WaveCorrector +from .warper import Warper +from .panorama_estimation import estimate_final_panorama_dimensions +from .exposure_error_compensator import ExposureErrorCompensator +from .seam_finder import SeamFinder +from .blender import Blender +from .timelapser import Timelapser +from .stitching_error import StitchingError + + +class Stitcher: + DEFAULT_SETTINGS = { + "medium_megapix": ImageHandler.DEFAULT_MEDIUM_MEGAPIX, + "detector": FeatureDetector.DEFAULT_DETECTOR, + "nfeatures": 500, + "matcher_type": FeatureMatcher.DEFAULT_MATCHER, + "range_width": FeatureMatcher.DEFAULT_RANGE_WIDTH, + "try_use_gpu": False, + "match_conf": None, + "confidence_threshold": Subsetter.DEFAULT_CONFIDENCE_THRESHOLD, + "matches_graph_dot_file": Subsetter.DEFAULT_MATCHES_GRAPH_DOT_FILE, + "estimator": CameraEstimator.DEFAULT_CAMERA_ESTIMATOR, + "adjuster": CameraAdjuster.DEFAULT_CAMERA_ADJUSTER, + "refinement_mask": CameraAdjuster.DEFAULT_REFINEMENT_MASK, + "wave_correct_kind": WaveCorrector.DEFAULT_WAVE_CORRECTION, + "warper_type": Warper.DEFAULT_WARP_TYPE, + "low_megapix": ImageHandler.DEFAULT_LOW_MEGAPIX, + "compensator": ExposureErrorCompensator.DEFAULT_COMPENSATOR, + "nr_feeds": ExposureErrorCompensator.DEFAULT_NR_FEEDS, + "block_size": ExposureErrorCompensator.DEFAULT_BLOCK_SIZE, + "finder": SeamFinder.DEFAULT_SEAM_FINDER, + "final_megapix": ImageHandler.DEFAULT_FINAL_MEGAPIX, + "blender_type": Blender.DEFAULT_BLENDER, + "blend_strength": Blender.DEFAULT_BLEND_STRENGTH, + "timelapse": Timelapser.DEFAULT_TIMELAPSE} + + def __init__(self, **kwargs): + self.initialize_stitcher(**kwargs) + + def initialize_stitcher(self, **kwargs): + self.settings = Stitcher.DEFAULT_SETTINGS.copy() + self.validate_kwargs(kwargs) + self.settings.update(kwargs) + + args = SimpleNamespace(**self.settings) + self.img_handler = ImageHandler(args.medium_megapix, + args.low_megapix, + args.final_megapix) + self.detector = \ + FeatureDetector(args.detector, nfeatures=args.nfeatures) + match_conf = \ + FeatureMatcher.get_match_conf(args.match_conf, args.detector) + self.matcher = FeatureMatcher(args.matcher_type, args.range_width, + try_use_gpu=args.try_use_gpu, + match_conf=match_conf) + self.subsetter = \ + Subsetter(args.confidence_threshold, args.matches_graph_dot_file) + self.camera_estimator = CameraEstimator(args.estimator) + self.camera_adjuster = \ + CameraAdjuster(args.adjuster, args.refinement_mask) + self.wave_corrector = WaveCorrector(args.wave_correct_kind) + self.warper = Warper(args.warper_type) + self.compensator = \ + ExposureErrorCompensator(args.compensator, args.nr_feeds, + args.block_size) + self.seam_finder = SeamFinder(args.finder) + self.blender = Blender(args.blender_type, args.blend_strength) + self.timelapser = Timelapser(args.timelapse) + + def stitch(self, img_names): + self.initialize_registration(img_names) + + imgs = self.resize_medium_resolution() + features = self.find_features(imgs) + matches = self.match_features(features) + imgs, features, matches = self.subset(imgs, features, matches) + cameras = self.estimate_camera_parameters(features, matches) + cameras = self.refine_camera_parameters(features, matches, cameras) + cameras = self.perform_wave_correction(cameras) + panorama_scale, panorama_corners, panorama_sizes = \ + self.estimate_final_panorama_dimensions(cameras) + + self.initialize_composition(panorama_corners, panorama_sizes) + + imgs = self.resize_low_resolution(imgs) + imgs = self.warp_low_resolution_images(imgs, cameras, panorama_scale) + self.estimate_exposure_errors(imgs) + seam_masks = self.find_seam_masks(imgs) + + imgs = self.resize_final_resolution() + imgs = self.warp_final_resolution_images(imgs, cameras, panorama_scale) + imgs = self.compensate_exposure_errors(imgs) + seam_masks = self.resize_seam_masks(seam_masks) + self.blend_images(imgs, seam_masks) + + return self.create_final_panorama() + + def initialize_registration(self, img_names): + self.img_handler.set_img_names(img_names) + + def resize_medium_resolution(self): + return list(self.img_handler.resize_to_medium_resolution()) + + def find_features(self, imgs): + return [self.detector.detect_features(img) for img in imgs] + + def match_features(self, features): + return self.matcher.match_features(features) + + def subset(self, imgs, features, matches): + names, sizes, imgs, features, matches = \ + self.subsetter.subset(self.img_handler.img_names, + self.img_handler.img_sizes, + imgs, features, matches) + self.img_handler.img_names, self.img_handler.img_sizes = names, sizes + return imgs, features, matches + + def estimate_camera_parameters(self, features, matches): + return self.camera_estimator.estimate(features, matches) + + def refine_camera_parameters(self, features, matches, cameras): + return self.camera_adjuster.adjust(features, matches, cameras) + + def perform_wave_correction(self, cameras): + return self.wave_corrector.correct(cameras) + + def estimate_final_panorama_dimensions(self, cameras): + return estimate_final_panorama_dimensions(cameras, self.warper, + self.img_handler) + + def initialize_composition(self, corners, sizes): + if self.timelapser.do_timelapse: + self.timelapser.initialize(corners, sizes) + else: + self.blender.prepare(corners, sizes) + + def resize_low_resolution(self, imgs=None): + return list(self.img_handler.resize_to_low_resolution(imgs)) + + def warp_low_resolution_images(self, imgs, cameras, final_scale): + camera_aspect = self.img_handler.get_medium_to_low_ratio() + scale = final_scale * self.img_handler.get_final_to_low_ratio() + return list(self.warp_images(imgs, cameras, scale, camera_aspect)) + + def warp_final_resolution_images(self, imgs, cameras, scale): + camera_aspect = self.img_handler.get_medium_to_final_ratio() + return self.warp_images(imgs, cameras, scale, camera_aspect) + + def warp_images(self, imgs, cameras, scale, aspect=1): + self._masks = [] + self._corners = [] + for img_warped, mask_warped, corner in \ + self.warper.warp_images_and_image_masks( + imgs, cameras, scale, aspect + ): + self._masks.append(mask_warped) + self._corners.append(corner) + yield img_warped + + def estimate_exposure_errors(self, imgs): + self.compensator.feed(self._corners, imgs, self._masks) + + def find_seam_masks(self, imgs): + return self.seam_finder.find(imgs, self._corners, self._masks) + + def resize_final_resolution(self): + return self.img_handler.resize_to_final_resolution() + + def compensate_exposure_errors(self, imgs): + for idx, img in enumerate(imgs): + yield self.compensator.apply(idx, self._corners[idx], + img, self._masks[idx]) + + def resize_seam_masks(self, seam_masks): + for idx, seam_mask in enumerate(seam_masks): + yield SeamFinder.resize(seam_mask, self._masks[idx]) + + def blend_images(self, imgs, masks): + for idx, (img, mask) in enumerate(zip(imgs, masks)): + if self.timelapser.do_timelapse: + self.timelapser.process_and_save_frame( + self.img_handler.img_names[idx], img, self._corners[idx] + ) + else: + self.blender.feed(img, mask, self._corners[idx]) + + def create_final_panorama(self): + if not self.timelapser.do_timelapse: + return self.blender.blend() + + @staticmethod + def validate_kwargs(kwargs): + for arg in kwargs: + if arg not in Stitcher.DEFAULT_SETTINGS: + raise StitchingError("Invalid Argument: " + arg) + + def collect_garbage(self): + del self.img_handler.img_names, self.img_handler.img_sizes, + del self._corners, self._masks diff --git a/apps/opencv_stitching_tool/opencv_stitching/stitching_error.py b/apps/opencv_stitching_tool/opencv_stitching/stitching_error.py new file mode 100644 index 0000000000..6d42e95437 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/stitching_error.py @@ -0,0 +1,2 @@ +class StitchingError(Exception): + pass diff --git a/apps/opencv_stitching_tool/opencv_stitching/subsetter.py b/apps/opencv_stitching_tool/opencv_stitching/subsetter.py new file mode 100644 index 0000000000..4ea6acc60d --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/subsetter.py @@ -0,0 +1,95 @@ +from itertools import chain +import math +import cv2 as cv +import numpy as np + +from .feature_matcher import FeatureMatcher +from .stitching_error import StitchingError + + +class Subsetter: + + DEFAULT_CONFIDENCE_THRESHOLD = 1 + DEFAULT_MATCHES_GRAPH_DOT_FILE = None + + def __init__(self, + confidence_threshold=DEFAULT_CONFIDENCE_THRESHOLD, + matches_graph_dot_file=DEFAULT_MATCHES_GRAPH_DOT_FILE): + self.confidence_threshold = confidence_threshold + self.save_file = matches_graph_dot_file + + def subset(self, img_names, img_sizes, imgs, features, matches): + self.save_matches_graph_dot_file(img_names, matches) + indices = self.get_indices_to_keep(features, matches) + + img_names = Subsetter.subset_list(img_names, indices) + img_sizes = Subsetter.subset_list(img_sizes, indices) + imgs = Subsetter.subset_list(imgs, indices) + features = Subsetter.subset_list(features, indices) + matches = Subsetter.subset_matches(matches, indices) + return img_names, img_sizes, imgs, features, matches + + def save_matches_graph_dot_file(self, img_names, pairwise_matches): + if self.save_file: + with open(self.save_file, 'w') as filehandler: + filehandler.write(self.get_matches_graph(img_names, + pairwise_matches) + ) + + def get_matches_graph(self, img_names, pairwise_matches): + return cv.detail.matchesGraphAsString(img_names, pairwise_matches, + self.confidence_threshold) + + def get_indices_to_keep(self, features, pairwise_matches): + indices = cv.detail.leaveBiggestComponent(features, + pairwise_matches, + self.confidence_threshold) + indices_as_list = [int(idx) for idx in list(indices[:, 0])] + + if len(indices_as_list) < 2: + raise StitchingError("No match exceeds the " + "given confidence theshold.") + + return indices_as_list + + @staticmethod + def subset_list(list_to_subset, indices): + return [list_to_subset[i] for i in indices] + + @staticmethod + def subset_matches(pairwise_matches, indices): + indices_to_delete = Subsetter.get_indices_to_delete( + math.sqrt(len(pairwise_matches)), + indices + ) + + matches_matrix = FeatureMatcher.get_matches_matrix(pairwise_matches) + matches_matrix_subset = Subsetter.subset_matrix(matches_matrix, + indices_to_delete) + matches_subset = Subsetter.matrix_rows_to_list(matches_matrix_subset) + + return matches_subset + + @staticmethod + def get_indices_to_delete(nr_elements, indices_to_keep): + return list(set(range(int(nr_elements))) - set(indices_to_keep)) + + @staticmethod + def subset_matrix(matrix_to_subset, indices_to_delete): + for idx, idx_to_delete in enumerate(indices_to_delete): + matrix_to_subset = Subsetter.delete_index_from_matrix( + matrix_to_subset, + idx_to_delete-idx # matrix shape reduced by one at each step + ) + + return matrix_to_subset + + @staticmethod + def delete_index_from_matrix(matrix, idx): + mask = np.ones(matrix.shape[0], bool) + mask[idx] = 0 + return matrix[mask, :][:, mask] + + @staticmethod + def matrix_rows_to_list(matrix): + return list(chain.from_iterable(matrix.tolist())) diff --git a/apps/opencv_stitching_tool/opencv_stitching/test/.gitignore b/apps/opencv_stitching_tool/opencv_stitching/test/.gitignore new file mode 100644 index 0000000000..93426ff2b0 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/test/.gitignore @@ -0,0 +1,13 @@ +# Ignore everything +* + +# But not these files... +!.gitignore +!test_matcher.py +!test_stitcher.py +!test_megapix_scaler.py +!test_registration.py +!test_composition.py +!test_performance.py +!stitching_detailed.py +!SAMPLE_IMAGES_TO_DOWNLOAD.txt \ No newline at end of file diff --git a/apps/opencv_stitching_tool/opencv_stitching/test/SAMPLE_IMAGES_TO_DOWNLOAD.txt b/apps/opencv_stitching_tool/opencv_stitching/test/SAMPLE_IMAGES_TO_DOWNLOAD.txt new file mode 100644 index 0000000000..cecf3b8ba7 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/test/SAMPLE_IMAGES_TO_DOWNLOAD.txt @@ -0,0 +1,5 @@ +https://github.com/opencv/opencv_extra/tree/master/testdata/stitching + +s1.jpg s2.jpg +boat1.jpg boat2.jpg boat3.jpg boat4.jpg boat5.jpg boat6.jpg +budapest1.jpg budapest2.jpg budapest3.jpg budapest4.jpg budapest5.jpg budapest6.jpg \ No newline at end of file diff --git a/apps/opencv_stitching_tool/opencv_stitching/test/stitching_detailed.py b/apps/opencv_stitching_tool/opencv_stitching/test/stitching_detailed.py new file mode 100644 index 0000000000..b12210de66 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/test/stitching_detailed.py @@ -0,0 +1,406 @@ +""" +Stitching sample (advanced) +=========================== +Show how to use Stitcher API from python. +""" + +# Python 2/3 compatibility +from __future__ import print_function + +from types import SimpleNamespace +from collections import OrderedDict + +import cv2 as cv +import numpy as np + +EXPOS_COMP_CHOICES = OrderedDict() +EXPOS_COMP_CHOICES['gain_blocks'] = cv.detail.ExposureCompensator_GAIN_BLOCKS +EXPOS_COMP_CHOICES['gain'] = cv.detail.ExposureCompensator_GAIN +EXPOS_COMP_CHOICES['channel'] = cv.detail.ExposureCompensator_CHANNELS +EXPOS_COMP_CHOICES['channel_blocks'] = cv.detail.ExposureCompensator_CHANNELS_BLOCKS +EXPOS_COMP_CHOICES['no'] = cv.detail.ExposureCompensator_NO + +BA_COST_CHOICES = OrderedDict() +BA_COST_CHOICES['ray'] = cv.detail_BundleAdjusterRay +BA_COST_CHOICES['reproj'] = cv.detail_BundleAdjusterReproj +BA_COST_CHOICES['affine'] = cv.detail_BundleAdjusterAffinePartial +BA_COST_CHOICES['no'] = cv.detail_NoBundleAdjuster + +FEATURES_FIND_CHOICES = OrderedDict() +try: + cv.xfeatures2d_SURF.create() # check if the function can be called + FEATURES_FIND_CHOICES['surf'] = cv.xfeatures2d_SURF.create +except (AttributeError, cv.error) as e: + print("SURF not available") +# if SURF not available, ORB is default +FEATURES_FIND_CHOICES['orb'] = cv.ORB.create +try: + FEATURES_FIND_CHOICES['sift'] = cv.xfeatures2d_SIFT.create +except AttributeError: + print("SIFT not available") +try: + FEATURES_FIND_CHOICES['brisk'] = cv.BRISK_create +except AttributeError: + print("BRISK not available") +try: + FEATURES_FIND_CHOICES['akaze'] = cv.AKAZE_create +except AttributeError: + print("AKAZE not available") + +SEAM_FIND_CHOICES = OrderedDict() +SEAM_FIND_CHOICES['dp_color'] = cv.detail_DpSeamFinder('COLOR') +SEAM_FIND_CHOICES['dp_colorgrad'] = cv.detail_DpSeamFinder('COLOR_GRAD') +SEAM_FIND_CHOICES['voronoi'] = cv.detail.SeamFinder_createDefault(cv.detail.SeamFinder_VORONOI_SEAM) +SEAM_FIND_CHOICES['no'] = cv.detail.SeamFinder_createDefault(cv.detail.SeamFinder_NO) + +ESTIMATOR_CHOICES = OrderedDict() +ESTIMATOR_CHOICES['homography'] = cv.detail_HomographyBasedEstimator +ESTIMATOR_CHOICES['affine'] = cv.detail_AffineBasedEstimator + +WARP_CHOICES = ( + 'spherical', + 'plane', + 'affine', + 'cylindrical', + 'fisheye', + 'stereographic', + 'compressedPlaneA2B1', + 'compressedPlaneA1.5B1', + 'compressedPlanePortraitA2B1', + 'compressedPlanePortraitA1.5B1', + 'paniniA2B1', + 'paniniA1.5B1', + 'paniniPortraitA2B1', + 'paniniPortraitA1.5B1', + 'mercator', + 'transverseMercator', +) + +WAVE_CORRECT_CHOICES = OrderedDict() +WAVE_CORRECT_CHOICES['horiz'] = cv.detail.WAVE_CORRECT_HORIZ +WAVE_CORRECT_CHOICES['no'] = None +WAVE_CORRECT_CHOICES['vert'] = cv.detail.WAVE_CORRECT_VERT + +BLEND_CHOICES = ('multiband', 'feather', 'no',) + +def get_matcher(args): + try_cuda = args.try_cuda + matcher_type = args.matcher + if args.match_conf is None: + if args.features == 'orb': + match_conf = 0.3 + else: + match_conf = 0.65 + else: + match_conf = args.match_conf + range_width = args.rangewidth + if matcher_type == "affine": + matcher = cv.detail_AffineBestOf2NearestMatcher(False, try_cuda, match_conf) + elif range_width == -1: + matcher = cv.detail.BestOf2NearestMatcher_create(try_cuda, match_conf) + else: + matcher = cv.detail.BestOf2NearestRangeMatcher_create(range_width, try_cuda, match_conf) + return matcher + + +def get_compensator(args): + expos_comp_type = EXPOS_COMP_CHOICES[args.expos_comp] + expos_comp_nr_feeds = args.expos_comp_nr_feeds + expos_comp_block_size = args.expos_comp_block_size + # expos_comp_nr_filtering = args.expos_comp_nr_filtering + if expos_comp_type == cv.detail.ExposureCompensator_CHANNELS: + compensator = cv.detail_ChannelsCompensator(expos_comp_nr_feeds) + # compensator.setNrGainsFilteringIterations(expos_comp_nr_filtering) + elif expos_comp_type == cv.detail.ExposureCompensator_CHANNELS_BLOCKS: + compensator = cv.detail_BlocksChannelsCompensator( + expos_comp_block_size, expos_comp_block_size, + expos_comp_nr_feeds + ) + # compensator.setNrGainsFilteringIterations(expos_comp_nr_filtering) + else: + compensator = cv.detail.ExposureCompensator_createDefault(expos_comp_type) + return compensator + + +def main(): + + args = { + "img_names":["boat5.jpg", "boat2.jpg", + "boat3.jpg", "boat4.jpg", + "boat1.jpg", "boat6.jpg"], + "try_cuda": False, + "work_megapix": 0.6, + "features": "orb", + "matcher": "homography", + "estimator": "homography", + "match_conf": None, + "conf_thresh": 1.0, + "ba": "ray", + "ba_refine_mask": "xxxxx", + "wave_correct": "horiz", + "save_graph": None, + "warp": "spherical", + "seam_megapix": 0.1, + "seam": "dp_color", + "compose_megapix": 3, + "expos_comp": "gain_blocks", + "expos_comp_nr_feeds": 1, + "expos_comp_nr_filtering": 2, + "expos_comp_block_size": 32, + "blend": "multiband", + "blend_strength": 5, + "output": "time_test.jpg", + "timelapse": None, + "rangewidth": -1 + } + + args = SimpleNamespace(**args) + img_names = args.img_names + work_megapix = args.work_megapix + seam_megapix = args.seam_megapix + compose_megapix = args.compose_megapix + conf_thresh = args.conf_thresh + ba_refine_mask = args.ba_refine_mask + wave_correct = WAVE_CORRECT_CHOICES[args.wave_correct] + if args.save_graph is None: + save_graph = False + else: + save_graph = True + warp_type = args.warp + blend_type = args.blend + blend_strength = args.blend_strength + result_name = args.output + if args.timelapse is not None: + timelapse = True + if args.timelapse == "as_is": + timelapse_type = cv.detail.Timelapser_AS_IS + elif args.timelapse == "crop": + timelapse_type = cv.detail.Timelapser_CROP + else: + print("Bad timelapse method") + exit() + else: + timelapse = False + finder = FEATURES_FIND_CHOICES[args.features]() + seam_work_aspect = 1 + full_img_sizes = [] + features = [] + images = [] + is_work_scale_set = False + is_seam_scale_set = False + is_compose_scale_set = False + for name in img_names: + full_img = cv.imread(cv.samples.findFile(name)) + if full_img is None: + print("Cannot read image ", name) + exit() + full_img_sizes.append((full_img.shape[1], full_img.shape[0])) + if work_megapix < 0: + img = full_img + work_scale = 1 + is_work_scale_set = True + else: + if is_work_scale_set is False: + work_scale = min(1.0, np.sqrt(work_megapix * 1e6 / (full_img.shape[0] * full_img.shape[1]))) + is_work_scale_set = True + img = cv.resize(src=full_img, dsize=None, fx=work_scale, fy=work_scale, interpolation=cv.INTER_LINEAR_EXACT) + if is_seam_scale_set is False: + seam_scale = min(1.0, np.sqrt(seam_megapix * 1e6 / (full_img.shape[0] * full_img.shape[1]))) + seam_work_aspect = seam_scale / work_scale + is_seam_scale_set = True + img_feat = cv.detail.computeImageFeatures2(finder, img) + features.append(img_feat) + img = cv.resize(src=full_img, dsize=None, fx=seam_scale, fy=seam_scale, interpolation=cv.INTER_LINEAR_EXACT) + images.append(img) + + matcher = get_matcher(args) + p = matcher.apply2(features) + matcher.collectGarbage() + + if save_graph: + with open(args.save_graph, 'w') as fh: + fh.write(cv.detail.matchesGraphAsString(img_names, p, conf_thresh)) + + indices = cv.detail.leaveBiggestComponent(features, p, conf_thresh) + img_subset = [] + img_names_subset = [] + full_img_sizes_subset = [] + for i in range(len(indices)): + img_names_subset.append(img_names[indices[i, 0]]) + img_subset.append(images[indices[i, 0]]) + full_img_sizes_subset.append(full_img_sizes[indices[i, 0]]) + images = img_subset + img_names = img_names_subset + full_img_sizes = full_img_sizes_subset + num_images = len(img_names) + if num_images < 2: + print("Need more images") + exit() + + estimator = ESTIMATOR_CHOICES[args.estimator]() + b, cameras = estimator.apply(features, p, None) + if not b: + print("Homography estimation failed.") + exit() + for cam in cameras: + cam.R = cam.R.astype(np.float32) + + adjuster = BA_COST_CHOICES[args.ba]() + adjuster.setConfThresh(1) + refine_mask = np.zeros((3, 3), np.uint8) + if ba_refine_mask[0] == 'x': + refine_mask[0, 0] = 1 + if ba_refine_mask[1] == 'x': + refine_mask[0, 1] = 1 + if ba_refine_mask[2] == 'x': + refine_mask[0, 2] = 1 + if ba_refine_mask[3] == 'x': + refine_mask[1, 1] = 1 + if ba_refine_mask[4] == 'x': + refine_mask[1, 2] = 1 + adjuster.setRefinementMask(refine_mask) + b, cameras = adjuster.apply(features, p, cameras) + if not b: + print("Camera parameters adjusting failed.") + exit() + focals = [] + for cam in cameras: + focals.append(cam.focal) + focals.sort() + if len(focals) % 2 == 1: + warped_image_scale = focals[len(focals) // 2] + else: + warped_image_scale = (focals[len(focals) // 2] + focals[len(focals) // 2 - 1]) / 2 + if wave_correct is not None: + rmats = [] + for cam in cameras: + rmats.append(np.copy(cam.R)) + rmats = cv.detail.waveCorrect(rmats, wave_correct) + for idx, cam in enumerate(cameras): + cam.R = rmats[idx] + corners = [] + masks_warped = [] + images_warped = [] + sizes = [] + masks = [] + for i in range(0, num_images): + um = cv.UMat(255 * np.ones((images[i].shape[0], images[i].shape[1]), np.uint8)) + masks.append(um) + + warper = cv.PyRotationWarper(warp_type, warped_image_scale * seam_work_aspect) # warper could be nullptr? + for idx in range(0, num_images): + K = cameras[idx].K().astype(np.float32) + swa = seam_work_aspect + K[0, 0] *= swa + K[0, 2] *= swa + K[1, 1] *= swa + K[1, 2] *= swa + corner, image_wp = warper.warp(images[idx], K, cameras[idx].R, cv.INTER_LINEAR, cv.BORDER_REFLECT) + corners.append(corner) + sizes.append((image_wp.shape[1], image_wp.shape[0])) + images_warped.append(image_wp) + p, mask_wp = warper.warp(masks[idx], K, cameras[idx].R, cv.INTER_NEAREST, cv.BORDER_CONSTANT) + masks_warped.append(mask_wp.get()) + + images_warped_f = [] + for img in images_warped: + imgf = img.astype(np.float32) + images_warped_f.append(imgf) + + compensator = get_compensator(args) + compensator.feed(corners=corners, images=images_warped, masks=masks_warped) + + seam_finder = SEAM_FIND_CHOICES[args.seam] + masks_warped = seam_finder.find(images_warped_f, corners, masks_warped) + compose_scale = 1 + corners = [] + sizes = [] + blender = None + timelapser = None + # https://github.com/opencv/opencv/blob/master/samples/cpp/stitching_detailed.cpp#L725 ? + for idx, name in enumerate(img_names): + full_img = cv.imread(name) + if not is_compose_scale_set: + if compose_megapix > 0: + compose_scale = min(1.0, np.sqrt(compose_megapix * 1e6 / (full_img.shape[0] * full_img.shape[1]))) + is_compose_scale_set = True + compose_work_aspect = compose_scale / work_scale + warped_image_scale *= compose_work_aspect + warper = cv.PyRotationWarper(warp_type, warped_image_scale) + for i in range(0, len(img_names)): + cameras[i].focal *= compose_work_aspect + cameras[i].ppx *= compose_work_aspect + cameras[i].ppy *= compose_work_aspect + sz = (int(round(full_img_sizes[i][0] * compose_scale)), + int(round(full_img_sizes[i][1] * compose_scale))) + K = cameras[i].K().astype(np.float32) + roi = warper.warpRoi(sz, K, cameras[i].R) + corners.append(roi[0:2]) + sizes.append(roi[2:4]) + if abs(compose_scale - 1) > 1e-1: + img = cv.resize(src=full_img, dsize=None, fx=compose_scale, fy=compose_scale, + interpolation=cv.INTER_LINEAR_EXACT) + else: + img = full_img + _img_size = (img.shape[1], img.shape[0]) + K = cameras[idx].K().astype(np.float32) + corner, image_warped = warper.warp(img, K, cameras[idx].R, cv.INTER_LINEAR, cv.BORDER_REFLECT) + mask = 255 * np.ones((img.shape[0], img.shape[1]), np.uint8) + p, mask_warped = warper.warp(mask, K, cameras[idx].R, cv.INTER_NEAREST, cv.BORDER_CONSTANT) + compensator.apply(idx, corners[idx], image_warped, mask_warped) + image_warped_s = image_warped.astype(np.int16) + dilated_mask = cv.dilate(masks_warped[idx], None) + seam_mask = cv.resize(dilated_mask, (mask_warped.shape[1], mask_warped.shape[0]), 0, 0, cv.INTER_LINEAR_EXACT) + mask_warped = cv.bitwise_and(seam_mask, mask_warped) + if blender is None and not timelapse: + blender = cv.detail.Blender_createDefault(cv.detail.Blender_NO) + dst_sz = cv.detail.resultRoi(corners=corners, sizes=sizes) + blend_width = np.sqrt(dst_sz[2] * dst_sz[3]) * blend_strength / 100 + if blend_width < 1: + blender = cv.detail.Blender_createDefault(cv.detail.Blender_NO) + elif blend_type == "multiband": + blender = cv.detail_MultiBandBlender() + blender.setNumBands((np.log(blend_width) / np.log(2.) - 1.).astype(np.int64)) + elif blend_type == "feather": + blender = cv.detail_FeatherBlender() + blender.setSharpness(1. / blend_width) + blender.prepare(dst_sz) + elif timelapser is None and timelapse: + timelapser = cv.detail.Timelapser_createDefault(timelapse_type) + timelapser.initialize(corners, sizes) + if timelapse: + ma_tones = np.ones((image_warped_s.shape[0], image_warped_s.shape[1]), np.uint8) + timelapser.process(image_warped_s, ma_tones, corners[idx]) + pos_s = img_names[idx].rfind("/") + if pos_s == -1: + fixed_file_name = "fixed_" + img_names[idx] + else: + fixed_file_name = img_names[idx][:pos_s + 1] + "fixed_" + img_names[idx][pos_s + 1:] + cv.imwrite(fixed_file_name, timelapser.getDst()) + else: + blender.feed(cv.UMat(image_warped_s), mask_warped, corners[idx]) + if not timelapse: + result = None + result_mask = None + result, result_mask = blender.blend(result, result_mask) + # cv.imwrite(result_name, result) + return result + # zoom_x = 600.0 / result.shape[1] + # dst = cv.normalize(src=result, dst=None, alpha=255., norm_type=cv.NORM_MINMAX, dtype=cv.CV_8U) + # dst = cv.resize(dst, dsize=None, fx=zoom_x, fy=zoom_x) + # cv.imshow(result_name, dst) + # cv.waitKey() + + + +if __name__ == '__main__': + import tracemalloc + import time + tracemalloc.start() + start = time.time() + result = main() + current, peak = tracemalloc.get_traced_memory() + print(f"Current memory usage is {current / 10**6}MB; Peak was {peak / 10**6}MB") + tracemalloc.stop() + end = time.time() + print(end - start) diff --git a/apps/opencv_stitching_tool/opencv_stitching/test/test_composition.py b/apps/opencv_stitching_tool/opencv_stitching/test/test_composition.py new file mode 100644 index 0000000000..b0b4d76c87 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/test/test_composition.py @@ -0,0 +1,67 @@ +import unittest +import os +import sys + +import numpy as np +import cv2 as cv + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), + '..', '..'))) + +from opencv_stitching.stitcher import Stitcher + + +class TestImageComposition(unittest.TestCase): + + # visual test: look especially in the sky + def test_exposure_compensation(self): + img = cv.imread("s1.jpg") + img = increase_brightness(img, value=25) + cv.imwrite("s1_bright.jpg", img) + + stitcher = Stitcher(compensator="no", blender_type="no") + result = stitcher.stitch(["s1_bright.jpg", "s2.jpg"]) + + cv.imwrite("without_exposure_comp.jpg", result) + + stitcher = Stitcher(blender_type="no") + result = stitcher.stitch(["s1_bright.jpg", "s2.jpg"]) + + cv.imwrite("with_exposure_comp.jpg", result) + + def test_timelapse(self): + stitcher = Stitcher(timelapse='as_is') + _ = stitcher.stitch(["s1.jpg", "s2.jpg"]) + frame1 = cv.imread("fixed_s1.jpg") + + max_image_shape_derivation = 3 + np.testing.assert_allclose(frame1.shape[:2], + (700, 1811), + atol=max_image_shape_derivation) + + left = cv.cvtColor(frame1[:, :1300, ], cv.COLOR_BGR2GRAY) + right = cv.cvtColor(frame1[:, 1300:, ], cv.COLOR_BGR2GRAY) + + self.assertGreater(cv.countNonZero(left), 800000) + self.assertEqual(cv.countNonZero(right), 0) + + +def increase_brightness(img, value=30): + hsv = cv.cvtColor(img, cv.COLOR_BGR2HSV) + h, s, v = cv.split(hsv) + + lim = 255 - value + v[v > lim] = 255 + v[v <= lim] += value + + final_hsv = cv.merge((h, s, v)) + img = cv.cvtColor(final_hsv, cv.COLOR_HSV2BGR) + return img + + +def starttest(): + unittest.main() + + +if __name__ == "__main__": + starttest() diff --git a/apps/opencv_stitching_tool/opencv_stitching/test/test_matcher.py b/apps/opencv_stitching_tool/opencv_stitching/test/test_matcher.py new file mode 100644 index 0000000000..a2424ec9ce --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/test/test_matcher.py @@ -0,0 +1,47 @@ +import unittest +import os +import sys + +import numpy as np + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), + '..', '..'))) + +from opencv_stitching.feature_matcher import FeatureMatcher +# %% + + +class TestMatcher(unittest.TestCase): + + def test_array_in_sqare_matrix(self): + array = np.zeros(9) + + matrix = FeatureMatcher.array_in_sqare_matrix(array) + + np.testing.assert_array_equal(matrix, np.array([[0., 0., 0.], + [0., 0., 0.], + [0., 0., 0.]])) + + def test_get_all_img_combinations(self): + nimgs = 3 + + combinations = list(FeatureMatcher.get_all_img_combinations(nimgs)) + + self.assertEqual(combinations, [(0, 1), (0, 2), (1, 2)]) + + def test_get_match_conf(self): + explicit_match_conf = FeatureMatcher.get_match_conf(1, 'orb') + implicit_match_conf_orb = FeatureMatcher.get_match_conf(None, 'orb') + implicit_match_conf_other = FeatureMatcher.get_match_conf(None, 'surf') + + self.assertEqual(explicit_match_conf, 1) + self.assertEqual(implicit_match_conf_orb, 0.3) + self.assertEqual(implicit_match_conf_other, 0.65) + + +def starttest(): + unittest.main() + + +if __name__ == "__main__": + starttest() diff --git a/apps/opencv_stitching_tool/opencv_stitching/test/test_megapix_scaler.py b/apps/opencv_stitching_tool/opencv_stitching/test/test_megapix_scaler.py new file mode 100644 index 0000000000..0afdad2628 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/test/test_megapix_scaler.py @@ -0,0 +1,59 @@ +import unittest +import os +import sys + +import cv2 as cv + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), + '..', '..'))) + +from opencv_stitching.megapix_scaler import MegapixScaler +from opencv_stitching.megapix_downscaler import MegapixDownscaler +#%% + + +class TestScaler(unittest.TestCase): + + def setUp(self): + self.img = cv.imread("s1.jpg") + self.size = (self.img.shape[1], self.img.shape[0]) + + def test_get_scale_by_resolution(self): + scaler = MegapixScaler(0.6) + + scale = scaler.get_scale_by_resolution(1_200_000) + + self.assertEqual(scale, 0.7071067811865476) + + def test_get_scale_by_image(self): + scaler = MegapixScaler(0.6) + + scaler.set_scale_by_img_size(self.size) + + self.assertEqual(scaler.scale, 0.8294067854101966) + + def test_get_scaled_img_size(self): + scaler = MegapixScaler(0.6) + scaler.set_scale_by_img_size(self.size) + + size = scaler.get_scaled_img_size(self.size) + self.assertEqual(size, (1033, 581)) + # 581*1033 = 600173 px = ~0.6 MP + + def test_force_of_downscaling(self): + normal_scaler = MegapixScaler(2) + downscaler = MegapixDownscaler(2) + + normal_scaler.set_scale_by_img_size(self.size) + downscaler.set_scale_by_img_size(self.size) + + self.assertEqual(normal_scaler.scale, 1.5142826857233715) + self.assertEqual(downscaler.scale, 1.0) + + +def starttest(): + unittest.main() + + +if __name__ == "__main__": + starttest() diff --git a/apps/opencv_stitching_tool/opencv_stitching/test/test_performance.py b/apps/opencv_stitching_tool/opencv_stitching/test/test_performance.py new file mode 100644 index 0000000000..60b03a8bfe --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/test/test_performance.py @@ -0,0 +1,65 @@ +import unittest +import os +import sys +import time +import tracemalloc + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), + '..', '..'))) + +from opencv_stitching.stitcher import Stitcher +from stitching_detailed import main +# %% + + +class TestStitcher(unittest.TestCase): + + def test_performance(self): + + print("Run new Stitcher class:") + + start = time.time() + tracemalloc.start() + + stitcher = Stitcher(final_megapix=3) + stitcher.stitch(["boat5.jpg", "boat2.jpg", + "boat3.jpg", "boat4.jpg", + "boat1.jpg", "boat6.jpg"]) + stitcher.collect_garbage() + + _, peak_memory = tracemalloc.get_traced_memory() + tracemalloc.stop() + end = time.time() + time_needed = end - start + + print(f"Peak was {peak_memory / 10**6} MB") + print(f"Time was {time_needed} s") + + print("Run original stitching_detailed.py:") + + start = time.time() + tracemalloc.start() + + main() + + _, peak_memory_detailed = tracemalloc.get_traced_memory() + tracemalloc.stop() + end = time.time() + time_needed_detailed = end - start + + print(f"Peak was {peak_memory_detailed / 10**6} MB") + print(f"Time was {time_needed_detailed} s") + + self.assertLessEqual(peak_memory / 10**6, + peak_memory_detailed / 10**6) + uncertainty_based_on_run = 0.25 + self.assertLessEqual(time_needed - uncertainty_based_on_run, + time_needed_detailed) + + +def starttest(): + unittest.main() + + +if __name__ == "__main__": + starttest() diff --git a/apps/opencv_stitching_tool/opencv_stitching/test/test_registration.py b/apps/opencv_stitching_tool/opencv_stitching/test/test_registration.py new file mode 100644 index 0000000000..98e792fd01 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/test/test_registration.py @@ -0,0 +1,100 @@ +import unittest +import os +import sys + +import numpy as np +import cv2 as cv + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), + '..', '..'))) + +from opencv_stitching.feature_detector import FeatureDetector +from opencv_stitching.feature_matcher import FeatureMatcher +from opencv_stitching.subsetter import Subsetter + + +class TestImageRegistration(unittest.TestCase): + + def test_feature_detector(self): + img1 = cv.imread("s1.jpg") + + default_number_of_keypoints = 500 + detector = FeatureDetector("orb") + features = detector.detect_features(img1) + self.assertEqual(len(features.getKeypoints()), + default_number_of_keypoints) + + other_keypoints = 1000 + detector = FeatureDetector("orb", nfeatures=other_keypoints) + features = detector.detect_features(img1) + self.assertEqual(len(features.getKeypoints()), other_keypoints) + + def test_feature_matcher(self): + img1, img2 = cv.imread("s1.jpg"), cv.imread("s2.jpg") + + detector = FeatureDetector("orb") + features = [detector.detect_features(img1), + detector.detect_features(img2)] + + matcher = FeatureMatcher() + pairwise_matches = matcher.match_features(features) + self.assertEqual(len(pairwise_matches), len(features)**2) + self.assertGreater(pairwise_matches[1].confidence, 2) + + matches_matrix = FeatureMatcher.get_matches_matrix(pairwise_matches) + self.assertEqual(matches_matrix.shape, (2, 2)) + conf_matrix = FeatureMatcher.get_confidence_matrix(pairwise_matches) + self.assertTrue(np.array_equal( + conf_matrix > 2, + np.array([[False, True], [True, False]]) + )) + + def test_subsetting(self): + img1, img2 = cv.imread("s1.jpg"), cv.imread("s2.jpg") + img3, img4 = cv.imread("boat1.jpg"), cv.imread("boat2.jpg") + img5 = cv.imread("boat3.jpg") + img_names = ["s1.jpg", "s2.jpg", "boat1.jpg", "boat2.jpg", "boat3.jpg"] + + detector = FeatureDetector("orb") + features = [detector.detect_features(img1), + detector.detect_features(img2), + detector.detect_features(img3), + detector.detect_features(img4), + detector.detect_features(img5)] + matcher = FeatureMatcher() + pairwise_matches = matcher.match_features(features) + subsetter = Subsetter(confidence_threshold=1, + matches_graph_dot_file="dot_graph.txt") # view in https://dreampuf.github.io # noqa + + indices = subsetter.get_indices_to_keep(features, pairwise_matches) + indices_to_delete = subsetter.get_indices_to_delete(len(img_names), + indices) + + self.assertEqual(indices, [2, 3, 4]) + self.assertEqual(indices_to_delete, [0, 1]) + + subsetted_image_names = subsetter.subset_list(img_names, indices) + self.assertEqual(subsetted_image_names, + ['boat1.jpg', 'boat2.jpg', 'boat3.jpg']) + + matches_subset = subsetter.subset_matches(pairwise_matches, indices) + # FeatureMatcher.get_confidence_matrix(pairwise_matches) + # FeatureMatcher.get_confidence_matrix(subsetted_matches) + self.assertEqual(pairwise_matches[13].confidence, + matches_subset[1].confidence) + + graph = subsetter.get_matches_graph(img_names, pairwise_matches) + self.assertTrue(graph.startswith("graph matches_graph{")) + + subsetter.save_matches_graph_dot_file(img_names, pairwise_matches) + with open('dot_graph.txt', 'r') as file: + graph = file.read() + self.assertTrue(graph.startswith("graph matches_graph{")) + + +def starttest(): + unittest.main() + + +if __name__ == "__main__": + starttest() diff --git a/apps/opencv_stitching_tool/opencv_stitching/test/test_stitcher.py b/apps/opencv_stitching_tool/opencv_stitching/test/test_stitcher.py new file mode 100644 index 0000000000..5a24f752c0 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/test/test_stitcher.py @@ -0,0 +1,108 @@ +import unittest +import os +import sys + +import numpy as np +import cv2 as cv + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), + '..', '..'))) + +from opencv_stitching.stitcher import Stitcher +# %% + + +class TestStitcher(unittest.TestCase): + + def test_stitcher_aquaduct(self): + stitcher = Stitcher(n_features=250) + result = stitcher.stitch(["s1.jpg", "s2.jpg"]) + cv.imwrite("result.jpg", result) + + max_image_shape_derivation = 3 + np.testing.assert_allclose(result.shape[:2], + (700, 1811), + atol=max_image_shape_derivation) + + @unittest.skip("skip boat test (high resuolution ran >30s)") + def test_stitcher_boat1(self): + settings = {"warper_type": "fisheye", + "wave_correct_kind": "no", + "finder": "dp_colorgrad", + "compensator": "no", + "conf_thresh": 0.3} + + stitcher = Stitcher(**settings) + result = stitcher.stitch(["boat5.jpg", "boat2.jpg", + "boat3.jpg", "boat4.jpg", + "boat1.jpg", "boat6.jpg"]) + + cv.imwrite("boat_fisheye.jpg", result) + + max_image_shape_derivation = 600 + np.testing.assert_allclose(result.shape[:2], + (14488, 7556), + atol=max_image_shape_derivation) + + @unittest.skip("skip boat test (high resuolution ran >30s)") + def test_stitcher_boat2(self): + settings = {"warper_type": "compressedPlaneA2B1", + "finder": "dp_colorgrad", + "compensator": "channel_blocks", + "conf_thresh": 0.3} + + stitcher = Stitcher(**settings) + result = stitcher.stitch(["boat5.jpg", "boat2.jpg", + "boat3.jpg", "boat4.jpg", + "boat1.jpg", "boat6.jpg"]) + + cv.imwrite("boat_plane.jpg", result) + + max_image_shape_derivation = 600 + np.testing.assert_allclose(result.shape[:2], + (7400, 12340), + atol=max_image_shape_derivation) + + def test_stitcher_boat_aquaduct_subset(self): + settings = {"final_megapix": 1} + + stitcher = Stitcher(**settings) + result = stitcher.stitch(["boat5.jpg", + "s1.jpg", "s2.jpg", + "boat2.jpg", + "boat3.jpg", "boat4.jpg", + "boat1.jpg", "boat6.jpg"]) + cv.imwrite("subset_low_res.jpg", result) + + max_image_shape_derivation = 100 + np.testing.assert_allclose(result.shape[:2], + (839, 3384), + atol=max_image_shape_derivation) + + def test_stitcher_budapest(self): + settings = {"matcher_type": "affine", + "estimator": "affine", + "adjuster": "affine", + "warper_type": "affine", + "wave_correct_kind": "no", + "confidence_threshold": 0.3} + + stitcher = Stitcher(**settings) + result = stitcher.stitch(["budapest1.jpg", "budapest2.jpg", + "budapest3.jpg", "budapest4.jpg", + "budapest5.jpg", "budapest6.jpg"]) + + cv.imwrite("budapest.jpg", result) + + max_image_shape_derivation = 50 + np.testing.assert_allclose(result.shape[:2], + (1155, 2310), + atol=max_image_shape_derivation) + + +def starttest(): + unittest.main() + + +if __name__ == "__main__": + starttest() diff --git a/apps/opencv_stitching_tool/opencv_stitching/timelapser.py b/apps/opencv_stitching_tool/opencv_stitching/timelapser.py new file mode 100644 index 0000000000..4085f473fa --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/timelapser.py @@ -0,0 +1,50 @@ +import os +import cv2 as cv +import numpy as np + + +class Timelapser: + + TIMELAPSE_CHOICES = ('no', 'as_is', 'crop',) + DEFAULT_TIMELAPSE = 'no' + + def __init__(self, timelapse=DEFAULT_TIMELAPSE): + self.do_timelapse = True + self.timelapse_type = None + self.timelapser = None + + if timelapse == "as_is": + self.timelapse_type = cv.detail.Timelapser_AS_IS + elif timelapse == "crop": + self.timelapse_type = cv.detail.Timelapser_CROP + else: + self.do_timelapse = False + + if self.do_timelapse: + self.timelapser = cv.detail.Timelapser_createDefault( + self.timelapse_type + ) + + def initialize(self, *args): + """https://docs.opencv.org/master/dd/dac/classcv_1_1detail_1_1Timelapser.html#aaf0f7c4128009f02473332a0c41f6345""" # noqa + self.timelapser.initialize(*args) + + def process_and_save_frame(self, img_name, img, corner): + self.process_frame(img, corner) + cv.imwrite(self.get_fixed_filename(img_name), self.get_frame()) + + def process_frame(self, img, corner): + mask = np.ones((img.shape[0], img.shape[1]), np.uint8) + img = img.astype(np.int16) + self.timelapser.process(img, mask, corner) + + def get_frame(self): + frame = self.timelapser.getDst() + frame = np.float32(cv.UMat.get(frame)) + frame = cv.convertScaleAbs(frame) + return frame + + @staticmethod + def get_fixed_filename(img_name): + dirname, filename = os.path.split(img_name) + return os.path.join(dirname, "fixed_" + filename) diff --git a/apps/opencv_stitching_tool/opencv_stitching/warper.py b/apps/opencv_stitching_tool/opencv_stitching/warper.py new file mode 100644 index 0000000000..c31a8648c0 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/warper.py @@ -0,0 +1,71 @@ +import cv2 as cv +import numpy as np + + +class Warper: + + WARP_TYPE_CHOICES = ('spherical', 'plane', 'affine', 'cylindrical', + 'fisheye', 'stereographic', 'compressedPlaneA2B1', + 'compressedPlaneA1.5B1', + 'compressedPlanePortraitA2B1', + 'compressedPlanePortraitA1.5B1', + 'paniniA2B1', 'paniniA1.5B1', 'paniniPortraitA2B1', + 'paniniPortraitA1.5B1', 'mercator', + 'transverseMercator') + + DEFAULT_WARP_TYPE = 'spherical' + + def __init__(self, warper_type=DEFAULT_WARP_TYPE, scale=1): + self.warper_type = warper_type + self.warper = cv.PyRotationWarper(warper_type, scale) + self.scale = scale + + def warp_images_and_image_masks(self, imgs, cameras, scale=None, aspect=1): + self.update_scale(scale) + for img, camera in zip(imgs, cameras): + yield self.warp_image_and_image_mask(img, camera, scale, aspect) + + def warp_image_and_image_mask(self, img, camera, scale=None, aspect=1): + self.update_scale(scale) + corner, img_warped = self.warp_image(img, camera, aspect) + mask = 255 * np.ones((img.shape[0], img.shape[1]), np.uint8) + _, mask_warped = self.warp_image(mask, camera, aspect, mask=True) + return img_warped, mask_warped, corner + + def warp_image(self, image, camera, aspect=1, mask=False): + if mask: + interp_mode = cv.INTER_NEAREST + border_mode = cv.BORDER_CONSTANT + else: + interp_mode = cv.INTER_LINEAR + border_mode = cv.BORDER_REFLECT + + corner, warped_image = self.warper.warp(image, + Warper.get_K(camera, aspect), + camera.R, + interp_mode, + border_mode) + return corner, warped_image + + def warp_roi(self, width, height, camera, scale=None, aspect=1): + self.update_scale(scale) + roi = (width, height) + K = Warper.get_K(camera, aspect) + return self.warper.warpRoi(roi, K, camera.R) + + def update_scale(self, scale): + if scale is not None and scale != self.scale: + self.warper = cv.PyRotationWarper(self.warper_type, scale) # setScale not working: https://docs.opencv.org/master/d5/d76/classcv_1_1PyRotationWarper.html#a90b000bb75f95294f9b0b6ec9859eb55 + self.scale = scale + + @staticmethod + def get_K(camera, aspect=1): + K = camera.K().astype(np.float32) + """ Modification of intrinsic parameters needed if cameras were + obtained on different scale than the scale of the Images which should + be warped """ + K[0, 0] *= aspect + K[0, 2] *= aspect + K[1, 1] *= aspect + K[1, 2] *= aspect + return K diff --git a/apps/opencv_stitching_tool/opencv_stitching_tool.py b/apps/opencv_stitching_tool/opencv_stitching_tool.py new file mode 100644 index 0000000000..1ee96aa8cb --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching_tool.py @@ -0,0 +1,232 @@ +""" +Stitching sample (advanced) +=========================== + +Show how to use Stitcher API from python. +""" + +# Python 2/3 compatibility +from __future__ import print_function + +import argparse + +import cv2 as cv +import numpy as np + +from opencv_stitching.stitcher import Stitcher + +from opencv_stitching.image_handler import ImageHandler +from opencv_stitching.feature_detector import FeatureDetector +from opencv_stitching.feature_matcher import FeatureMatcher +from opencv_stitching.subsetter import Subsetter +from opencv_stitching.camera_estimator import CameraEstimator +from opencv_stitching.camera_adjuster import CameraAdjuster +from opencv_stitching.camera_wave_corrector import WaveCorrector +from opencv_stitching.warper import Warper +from opencv_stitching.exposure_error_compensator import ExposureErrorCompensator # noqa +from opencv_stitching.seam_finder import SeamFinder +from opencv_stitching.blender import Blender +from opencv_stitching.timelapser import Timelapser + +parser = argparse.ArgumentParser( + prog="opencv_stitching_tool.py", + description="Rotation model images stitcher" +) +parser.add_argument( + 'img_names', nargs='+', + help="Files to stitch", type=str +) +parser.add_argument( + '--medium_megapix', action='store', + default=ImageHandler.DEFAULT_MEDIUM_MEGAPIX, + help="Resolution for image registration step. " + "The default is %s Mpx" % ImageHandler.DEFAULT_MEDIUM_MEGAPIX, + type=float, dest='medium_megapix' +) +parser.add_argument( + '--detector', action='store', + default=FeatureDetector.DEFAULT_DETECTOR, + help="Type of features used for images matching. " + "The default is '%s'." % FeatureDetector.DEFAULT_DETECTOR, + choices=FeatureDetector.DETECTOR_CHOICES.keys(), + type=str, dest='detector' +) +parser.add_argument( + '--nfeatures', action='store', + default=500, + help="Type of features used for images matching. " + "The default is 500.", + type=int, dest='nfeatures' +) +parser.add_argument( + '--matcher_type', action='store', default=FeatureMatcher.DEFAULT_MATCHER, + help="Matcher used for pairwise image matching. " + "The default is '%s'." % FeatureMatcher.DEFAULT_MATCHER, + choices=FeatureMatcher.MATCHER_CHOICES, + type=str, dest='matcher_type' +) +parser.add_argument( + '--range_width', action='store', + default=FeatureMatcher.DEFAULT_RANGE_WIDTH, + help="uses range_width to limit number of images to match with.", + type=int, dest='range_width' +) +parser.add_argument( + '--try_use_gpu', + action='store', + default=False, + help="Try to use CUDA. The default value is no. " + "All default values are for CPU mode.", + type=bool, dest='try_use_gpu' +) +parser.add_argument( + '--match_conf', action='store', + help="Confidence for feature matching step. " + "The default is 0.3 for ORB and 0.65 for other feature types.", + type=float, dest='match_conf' +) +parser.add_argument( + '--confidence_threshold', action='store', + default=Subsetter.DEFAULT_CONFIDENCE_THRESHOLD, + help="Threshold for two images are from the same panorama confidence. " + "The default is '%s'." % Subsetter.DEFAULT_CONFIDENCE_THRESHOLD, + type=float, dest='confidence_threshold' +) +parser.add_argument( + '--matches_graph_dot_file', action='store', + default=Subsetter.DEFAULT_MATCHES_GRAPH_DOT_FILE, + help="Save matches graph represented in DOT language to file.", + type=str, dest='matches_graph_dot_file' +) +parser.add_argument( + '--estimator', action='store', + default=CameraEstimator.DEFAULT_CAMERA_ESTIMATOR, + help="Type of estimator used for transformation estimation. " + "The default is '%s'." % CameraEstimator.DEFAULT_CAMERA_ESTIMATOR, + choices=CameraEstimator.CAMERA_ESTIMATOR_CHOICES.keys(), + type=str, dest='estimator' +) +parser.add_argument( + '--adjuster', action='store', + default=CameraAdjuster.DEFAULT_CAMERA_ADJUSTER, + help="Bundle adjustment cost function. " + "The default is '%s'." % CameraAdjuster.DEFAULT_CAMERA_ADJUSTER, + choices=CameraAdjuster.CAMERA_ADJUSTER_CHOICES.keys(), + type=str, dest='adjuster' +) +parser.add_argument( + '--refinement_mask', action='store', + default=CameraAdjuster.DEFAULT_REFINEMENT_MASK, + help="Set refinement mask for bundle adjustment. It looks like 'x_xxx', " + "where 'x' means refine respective parameter and '_' means don't " + "refine, and has the following format:. " + "The default mask is '%s'. " + "If bundle adjustment doesn't support estimation of selected " + "parameter then the respective flag is ignored." + "" % CameraAdjuster.DEFAULT_REFINEMENT_MASK, + type=str, dest='refinement_mask' +) +parser.add_argument( + '--wave_correct_kind', action='store', + default=WaveCorrector.DEFAULT_WAVE_CORRECTION, + help="Perform wave effect correction. " + "The default is '%s'" % WaveCorrector.DEFAULT_WAVE_CORRECTION, + choices=WaveCorrector.WAVE_CORRECT_CHOICES.keys(), + type=str, dest='wave_correct_kind' +) +parser.add_argument( + '--warper_type', action='store', default=Warper.DEFAULT_WARP_TYPE, + help="Warp surface type. The default is '%s'." % Warper.DEFAULT_WARP_TYPE, + choices=Warper.WARP_TYPE_CHOICES, + type=str, dest='warper_type' +) +parser.add_argument( + '--low_megapix', action='store', default=ImageHandler.DEFAULT_LOW_MEGAPIX, + help="Resolution for seam estimation and exposure estimation step. " + "The default is %s Mpx." % ImageHandler.DEFAULT_LOW_MEGAPIX, + type=float, dest='low_megapix' +) +parser.add_argument( + '--compensator', action='store', + default=ExposureErrorCompensator.DEFAULT_COMPENSATOR, + help="Exposure compensation method. " + "The default is '%s'." % ExposureErrorCompensator.DEFAULT_COMPENSATOR, + choices=ExposureErrorCompensator.COMPENSATOR_CHOICES.keys(), + type=str, dest='compensator' +) +parser.add_argument( + '--nr_feeds', action='store', + default=ExposureErrorCompensator.DEFAULT_NR_FEEDS, + help="Number of exposure compensation feed.", + type=np.int32, dest='nr_feeds' +) +parser.add_argument( + '--block_size', action='store', + default=ExposureErrorCompensator.DEFAULT_BLOCK_SIZE, + help="BLock size in pixels used by the exposure compensator. " + "The default is '%s'." % ExposureErrorCompensator.DEFAULT_BLOCK_SIZE, + type=np.int32, dest='block_size' +) +parser.add_argument( + '--finder', action='store', default=SeamFinder.DEFAULT_SEAM_FINDER, + help="Seam estimation method. " + "The default is '%s'." % SeamFinder.DEFAULT_SEAM_FINDER, + choices=SeamFinder.SEAM_FINDER_CHOICES.keys(), + type=str, dest='finder' +) +parser.add_argument( + '--final_megapix', action='store', + default=ImageHandler.DEFAULT_FINAL_MEGAPIX, + help="Resolution for compositing step. Use -1 for original resolution. " + "The default is %s" % ImageHandler.DEFAULT_FINAL_MEGAPIX, + type=float, dest='final_megapix' +) +parser.add_argument( + '--blender_type', action='store', default=Blender.DEFAULT_BLENDER, + help="Blending method. The default is '%s'." % Blender.DEFAULT_BLENDER, + choices=Blender.BLENDER_CHOICES, + type=str, dest='blender_type' +) +parser.add_argument( + '--blend_strength', action='store', default=Blender.DEFAULT_BLEND_STRENGTH, + help="Blending strength from [0,100] range. " + "The default is '%s'." % Blender.DEFAULT_BLEND_STRENGTH, + type=np.int32, dest='blend_strength' +) +parser.add_argument( + '--timelapse', action='store', default=Timelapser.DEFAULT_TIMELAPSE, + help="Output warped images separately as frames of a time lapse movie, " + "with 'fixed_' prepended to input file names. " + "The default is '%s'." % Timelapser.DEFAULT_TIMELAPSE, + choices=Timelapser.TIMELAPSE_CHOICES, + type=str, dest='timelapse' +) +parser.add_argument( + '--output', action='store', default='result.jpg', + help="The default is 'result.jpg'", + type=str, dest='output' +) + +__doc__ += '\n' + parser.format_help() + +if __name__ == '__main__': + print(__doc__) + args = parser.parse_args() + args_dict = vars(args) + + # Extract In- and Output + img_names = args_dict.pop("img_names") + img_names = [cv.samples.findFile(img_name) for img_name in img_names] + output = args_dict.pop("output") + + stitcher = Stitcher(**args_dict) + panorama = stitcher.stitch(img_names) + + cv.imwrite(output, panorama) + + zoom_x = 600.0 / panorama.shape[1] + preview = cv.resize(panorama, dsize=None, fx=zoom_x, fy=zoom_x) + + cv.imshow(output, preview) + cv.waitKey() + cv.destroyAllWindows() From d2c1f1131bdc4fadd1a4ed3e9511b4ae4e752cf1 Mon Sep 17 00:00:00 2001 From: jcong Date: Tue, 9 Nov 2021 17:23:34 +0800 Subject: [PATCH 057/226] videoio: drop unnecessary offset for accessing video output buffer Fix: #21021 NDK API AMediaCodec_getOutputBuffer() returns MediaCodecBuffer::data() which is actually ABuffer::data(). The returned buffer address is already adjusted by offset. More info: ABuffer::base() returns base address without offset ABuffer::data() returns base + offset Change-Id: I2936339ce4fa9acf657a5a7d92adc1275d7b28a1 --- modules/videoio/src/cap_android_mediandk.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/videoio/src/cap_android_mediandk.cpp b/modules/videoio/src/cap_android_mediandk.cpp index 822024d53f..4fb4a82c2f 100644 --- a/modules/videoio/src/cap_android_mediandk.cpp +++ b/modules/videoio/src/cap_android_mediandk.cpp @@ -92,7 +92,7 @@ public: AMediaFormat_getInt32(mediaFormat.get(), AMEDIAFORMAT_KEY_HEIGHT, &frameHeight); AMediaFormat_getInt32(mediaFormat.get(), AMEDIAFORMAT_KEY_COLOR_FORMAT, &colorFormat); uint8_t* codecBuffer = AMediaCodec_getOutputBuffer(mediaCodec.get(), bufferIndex, &bufferSize); - buffer = std::vector(codecBuffer + info.offset, codecBuffer + bufferSize); + buffer = std::vector(codecBuffer, codecBuffer + bufferSize); LOGV("colorFormat: %d", colorFormat); LOGV("buffer size: %zu", bufferSize); LOGV("width (frame): %d", frameWidth); From 98b6ce353cc9342bee472e594eaf4b23136b0430 Mon Sep 17 00:00:00 2001 From: ZaKiiiiiiiii <56301098+Crayon-new@users.noreply.github.com> Date: Wed, 10 Nov 2021 00:24:04 +0800 Subject: [PATCH 058/226] Merge pull request #20904 from Crayon-new:fix_bug_in_maxLayer fix bug: wrong output dimension when "keep_dims" is false in pooling layer. * fix bug in max layer * code align * delete permute layer and add test case * add name assert * check other cases * remove c++11 features * style:add "const" remove assert * style:sanitize file names --- modules/dnn/src/tensorflow/tf_importer.cpp | 77 +++++++++++++++------- modules/dnn/test/test_tf_importer.cpp | 13 ++++ 2 files changed, 66 insertions(+), 24 deletions(-) diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index 1a01ac87d6..8cbe1c4b23 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -2142,6 +2142,7 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& const std::string& type = layer.op(); const int num_inputs = layer.input_size(); std::string pool_type = cv::toLowerCase(type); + DataLayout layout = getDataLayout(name, data_layouts); if (pool_type == "mean") { @@ -2205,6 +2206,16 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& if (!keepDims) { + if (layout == DATA_LAYOUT_NHWC) + { + LayerParams permLP; + int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. + std::string permName = name + "/nhwc"; + Pin inpId = Pin(layerShapeName); + addPermuteLayer(order, permName, inpId); + layerShapeName = permName; + } + LayerParams squeezeLp; std::string squeezeName = name + "/squeeze"; CV_Assert(layer_id.find(squeezeName) == layer_id.end()); @@ -2227,22 +2238,30 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& layerParams.set("pool", pool_type); layerParams.set(axis == 2 ? "kernel_w" : "kernel_h", 1); layerParams.set(axis == 2 ? "global_pooling_h" : "global_pooling_w", true); - int id = dstNet.addLayer(name, "Pooling", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - if (!keepDims) + if (keepDims) + { + int id = dstNet.addLayer(name, "Pooling", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + } + else { // To keep correct order after squeeze dims we first need to change layout from NCHW to NHWC + std::string poolingName = name + "/Pooling"; + CV_Assert(layer_id.find(poolingName) == layer_id.end()); + int id = dstNet.addLayer(poolingName, "Pooling", layerParams); + layer_id[poolingName] = id; + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + LayerParams permLP; int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. - std::string permName = name + "/nchw"; - Pin inpId = Pin(name); + std::string permName = name + "/nhwc"; + Pin inpId = Pin(poolingName); addPermuteLayer(order, permName, inpId); LayerParams squeezeLp; - std::string squeezeName = name + "/squeeze"; - CV_Assert(layer_id.find(squeezeName) == layer_id.end()); + const std::string& squeezeName = name; squeezeLp.set("axis", indices.at(0)); squeezeLp.set("end_axis", indices.at(0) + 1); int squeezeId = dstNet.addLayer(squeezeName, "Flatten", squeezeLp); @@ -2254,32 +2273,34 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& { int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. Pin inpId = parsePin(layer.input(0)); - addPermuteLayer(order, name + "/nhwc", inpId); + std::string permName = name + "/nhwc"; + addPermuteLayer(order, permName, inpId); layerParams.set("pool", pool_type); layerParams.set("kernel_h", 1); layerParams.set("global_pooling_w", true); - int id = dstNet.addLayer(name, "Pooling", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, inpId, id, 0); + std::string poolingName = name + "/Pooling"; + CV_Assert(layer_id.find(poolingName) == layer_id.end()); + int id = dstNet.addLayer(poolingName, "Pooling", layerParams); + layer_id[poolingName] = id; + connect(layer_id, dstNet, Pin(permName), id, 0); if (!keepDims) { LayerParams squeezeLp; - std::string squeezeName = name + "/squeeze"; - CV_Assert(layer_id.find(squeezeName) == layer_id.end()); + const std::string& squeezeName = name; int channel_id = 3; // TF NHWC layout squeezeLp.set("axis", channel_id - 1); squeezeLp.set("end_axis", channel_id); int squeezeId = dstNet.addLayer(squeezeName, "Flatten", squeezeLp); layer_id[squeezeName] = squeezeId; - connect(layer_id, dstNet, Pin(name), squeezeId, 0); + connect(layer_id, dstNet, Pin(poolingName), squeezeId, 0); } else { int order[] = {0, 3, 1, 2}; // From NHWC to OpenCV's NCHW. - Pin inpId = parsePin(name); - addPermuteLayer(order, name + "/nchw", inpId); + Pin inpId = parsePin(poolingName); + addPermuteLayer(order, name, inpId); } } } else { @@ -2288,18 +2309,26 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& layerParams.set("pool", pool_type); layerParams.set("global_pooling", true); - int id = dstNet.addLayer(name, "Pooling", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - if (!keepDims) + if (keepDims) { + int id = dstNet.addLayer(name, "Pooling", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + } + else + { + std::string poolingName = name + "/Pooling"; + CV_Assert(layer_id.find(poolingName) == layer_id.end()); + int id = dstNet.addLayer(poolingName, "Pooling", layerParams); + layer_id[poolingName] = id; + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); LayerParams flattenLp; - std::string flattenName = name + "/flatten"; - CV_Assert(layer_id.find(flattenName) == layer_id.end()); + const std::string& flattenName = name; int flattenId = dstNet.addLayer(flattenName, "Flatten", flattenLp); layer_id[flattenName] = flattenId; - connect(layer_id, dstNet, Pin(name), flattenId, 0); + connect(layer_id, dstNet, Pin(poolingName), flattenId, 0); + data_layouts[name] = DATA_LAYOUT_PLANAR; } } } diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index cdf3794bf1..b688c31383 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -413,6 +413,19 @@ TEST_P(Test_TensorFlow_layers, pooling_reduce_sum) runTensorFlowNet("reduce_sum"); // a SUM pooling over all spatial dimensions. } +TEST_P(Test_TensorFlow_layers, pooling_reduce_sum2) +{ + int axises[] = {0, 1, 2, 3}; + for (int keepdims = 0; keepdims <= 1; ++keepdims) + { + for (int i = 0; i < sizeof(axises)/sizeof(axises[0]); ++i) + { + runTensorFlowNet(cv::format("reduce_sum_%d_%s", axises[i], (keepdims ? "True" : "False"))); + } + runTensorFlowNet(cv::format("reduce_sum_1_2_%s", keepdims ? "True" : "False")); + } +} + TEST_P(Test_TensorFlow_layers, max_pool_grad) { if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) From 0c0a8392e5f97224676c312207d181a03e77c083 Mon Sep 17 00:00:00 2001 From: AleksandrPanov Date: Wed, 10 Nov 2021 11:14:06 +0300 Subject: [PATCH 059/226] fix markers parse in gen_pattern.py --- doc/pattern_tools/gen_pattern.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/doc/pattern_tools/gen_pattern.py b/doc/pattern_tools/gen_pattern.py index 83ed115d36..4618bc318b 100755 --- a/doc/pattern_tools/gen_pattern.py +++ b/doc/pattern_tools/gen_pattern.py @@ -171,7 +171,7 @@ def main(): parser.add_argument("-m", "--markers", help="list of cells with markers for the radon checkerboard. Marker " "coordinates as list of numbers: -m 1 2 3 4 means markers in cells " "[1, 2] and [3, 4]", - action="store", dest="markers", nargs="+", type=int) + default=argparse.SUPPRESS, action="store", dest="markers", nargs="+", type=int) args = parser.parse_args() show_help = args.show_help @@ -195,14 +195,16 @@ def main(): "A5": [148, 210]} page_width = page_sizes[page_size][0] page_height = page_sizes[page_size][1] - if len(args.markers) % 2 == 1: - raise ValueError("The length of the markers array={} must be even".format(len(args.markers))) - markers = set() - for x, y in zip(args.markers[::2], args.markers[1::2]): - if x in range(0, columns) and y in range(0, rows): - markers.add((x, y)) - else: - raise ValueError("The marker {},{} is outside the checkerboard".format(x, y)) + markers = None + if p_type == "radon_checkerboard" and "markers" in args: + if len(args.markers) % 2 == 1: + raise ValueError("The length of the markers array={} must be even".format(len(args.markers))) + markers = set() + for x, y in zip(args.markers[::2], args.markers[1::2]): + if x in range(0, columns) and y in range(0, rows): + markers.add((x, y)) + else: + raise ValueError("The marker {},{} is outside the checkerboard".format(x, y)) pm = PatternMaker(columns, rows, output, units, square_size, radius_rate, page_width, page_height, markers) # dict for easy lookup of pattern type From d934bb15b0c44b0ac50f2b2556393d9f4f28a620 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 10 Nov 2021 15:03:45 +0300 Subject: [PATCH 060/226] Merge pull request #20998 from alalek:update_protobuf_3.19.1 3rdparty(protobuf): upgrade 3.5.2 => 3.19.1 * 3rdparty(protobuf): upgrade 3.5.2 => 3.19.1 * dnn: update protobuf files (3.19.1) * 3rdparty(protobuf): re-apply OpenCV patch for custom fields (3.19.1) * protobuf: suppress new build warnings * protobuf: remove unused files --- 3rdparty/protobuf/CMakeLists.txt | 106 +- 3rdparty/protobuf/LICENSE | 12 +- 3rdparty/protobuf/README.md | 2 +- 3rdparty/protobuf/src/google/protobuf/any.cc | 87 +- 3rdparty/protobuf/src/google/protobuf/any.h | 83 +- .../protobuf/src/google/protobuf/any.pb.cc | 440 - .../protobuf/src/google/protobuf/any.pb.h | 323 - .../protobuf/src/google/protobuf/any_lite.cc | 99 + .../protobuf/src/google/protobuf/api.pb.cc | 1608 - .../protobuf/src/google/protobuf/api.pb.h | 1165 - .../protobuf/src/google/protobuf/arena.cc | 688 +- 3rdparty/protobuf/src/google/protobuf/arena.h | 1061 +- .../protobuf/src/google/protobuf/arena_impl.h | 594 +- .../src/google/protobuf/arenastring.cc | 249 +- .../src/google/protobuf/arenastring.h | 596 +- .../src/google/protobuf/descriptor.cc | 5144 +- .../protobuf/src/google/protobuf/descriptor.h | 1164 +- .../src/google/protobuf/descriptor.pb.cc | 16178 +++--- .../src/google/protobuf/descriptor.pb.h | 16815 +++--- .../google/protobuf/descriptor_database.cc | 792 +- .../src/google/protobuf/descriptor_database.h | 188 +- .../src/google/protobuf/duration.pb.cc | 431 - .../src/google/protobuf/duration.pb.h | 232 - .../src/google/protobuf/dynamic_message.cc | 781 +- .../src/google/protobuf/dynamic_message.h | 95 +- .../protobuf/src/google/protobuf/empty.pb.cc | 342 - .../protobuf/src/google/protobuf/empty.pb.h | 190 - .../time.h => explicitly_constructed.h} | 82 +- .../src/google/protobuf/extension_set.cc | 2134 +- .../src/google/protobuf/extension_set.h | 1347 +- .../google/protobuf/extension_set_heavy.cc | 656 +- .../src/google/protobuf/extension_set_inl.h | 276 + .../google/protobuf/field_access_listener.h | 172 + .../src/google/protobuf/field_mask.pb.cc | 373 - .../src/google/protobuf/field_mask.pb.h | 285 +- .../protobuf/generated_enum_reflection.h | 36 +- .../src/google/protobuf/generated_enum_util.h | 43 +- .../google/protobuf/generated_message_bases.h | 87 + .../protobuf/generated_message_reflection.cc | 2908 +- .../protobuf/generated_message_reflection.h | 710 +- .../generated_message_table_driven.cc | 103 - .../protobuf/generated_message_table_driven.h | 240 +- .../generated_message_table_driven_lite.cc | 109 - .../generated_message_table_driven_lite.h | 823 - .../protobuf/generated_message_tctable_decl.h | 139 + .../protobuf/generated_message_tctable_impl.h | 302 + .../generated_message_tctable_impl.inc | 92 + .../google/protobuf/generated_message_util.cc | 327 +- .../google/protobuf/generated_message_util.h | 329 +- .../protobuf/src/google/protobuf/has_bits.h | 29 +- .../statusor.cc => implicit_weak_message.cc} | 33 +- .../google/protobuf/implicit_weak_message.h | 186 + .../google/protobuf/inlined_string_field.h | 384 + .../src/google/protobuf/io/coded_stream.cc | 733 +- .../src/google/protobuf/io/coded_stream.h | 1397 +- .../src/google/protobuf/io/coded_stream_inl.h | 90 - .../src/google/protobuf/io/gzip_stream.cc | 330 - .../src/google/protobuf/io/gzip_stream.h | 209 - .../google/protobuf/{stubs => io}/io_win32.cc | 107 +- .../google/protobuf/{stubs => io}/io_win32.h | 78 +- .../src/google/protobuf/io/package_info.h | 1 - .../src/google/protobuf/io/printer.cc | 372 - .../protobuf/src/google/protobuf/io/printer.h | 363 - .../protobuf/src/google/protobuf/io/strtod.cc | 91 +- .../protobuf/src/google/protobuf/io/strtod.h | 2 +- .../src/google/protobuf/io/tokenizer.cc | 374 +- .../src/google/protobuf/io/tokenizer.h | 113 +- .../src/google/protobuf/io/zero_copy_stream.h | 17 +- .../protobuf/io/zero_copy_stream_impl.cc | 206 +- .../protobuf/io/zero_copy_stream_impl.h | 130 +- .../protobuf/io/zero_copy_stream_impl_lite.cc | 228 +- .../protobuf/io/zero_copy_stream_impl_lite.h | 150 +- .../internal/error_listener.cc => map.cc} | 9 +- 3rdparty/protobuf/src/google/protobuf/map.h | 1016 +- .../protobuf/src/google/protobuf/map_entry.h | 86 +- .../src/google/protobuf/map_entry_lite.h | 532 +- .../protobuf/src/google/protobuf/map_field.cc | 523 +- .../protobuf/src/google/protobuf/map_field.h | 1109 +- .../src/google/protobuf/map_field_inl.h | 234 +- .../src/google/protobuf/map_field_lite.h | 107 +- .../src/google/protobuf/map_type_handler.h | 752 +- .../protobuf/src/google/protobuf/message.cc | 429 +- .../protobuf/src/google/protobuf/message.h | 1273 +- .../src/google/protobuf/message_lite.cc | 562 +- .../src/google/protobuf/message_lite.h | 429 +- .../protobuf/src/google/protobuf/metadata.h | 44 +- .../src/google/protobuf/metadata_lite.h | 246 +- .../src/google/protobuf/package_info.h | 64 - .../src/google/protobuf/parse_context.cc | 559 + .../src/google/protobuf/parse_context.h | 938 + .../google/protobuf/{service.cc => port.h} | 20 +- .../protobuf/src/google/protobuf/port_def.inc | 824 + .../src/google/protobuf/port_undef.inc | 145 + .../protobuf/src/google/protobuf/reflection.h | 317 +- .../src/google/protobuf/reflection_internal.h | 182 +- .../src/google/protobuf/reflection_ops.cc | 333 +- .../src/google/protobuf/reflection_ops.h | 18 +- .../src/google/protobuf/repeated_field.cc | 78 +- .../src/google/protobuf/repeated_field.h | 2492 +- .../src/google/protobuf/repeated_ptr_field.cc | 157 + .../src/google/protobuf/repeated_ptr_field.h | 2014 + .../protobuf/src/google/protobuf/service.h | 292 - .../src/google/protobuf/source_context.pb.cc | 381 - .../src/google/protobuf/source_context.pb.h | 243 - .../protobuf/src/google/protobuf/struct.pb.cc | 1475 - .../protobuf/src/google/protobuf/struct.pb.h | 1030 - .../protobuf/stubs/atomic_sequence_num.h | 54 - .../src/google/protobuf/stubs/atomicops.h | 237 - .../stubs/atomicops_internals_arm64_gcc.h | 325 - .../stubs/atomicops_internals_arm_gcc.h | 151 - .../stubs/atomicops_internals_arm_qnx.h | 146 - .../atomicops_internals_generic_c11_atomic.h | 231 - .../stubs/atomicops_internals_generic_gcc.h | 163 - .../stubs/atomicops_internals_mips_gcc.h | 313 - .../stubs/atomicops_internals_power.h | 440 - .../stubs/atomicops_internals_ppc_gcc.h | 155 - .../stubs/atomicops_internals_solaris.h | 188 - .../protobuf/stubs/atomicops_internals_tsan.h | 219 - .../stubs/atomicops_internals_x86_gcc.cc | 137 - .../stubs/atomicops_internals_x86_gcc.h | 293 - .../stubs/atomicops_internals_x86_msvc.cc | 113 - .../stubs/atomicops_internals_x86_msvc.h | 150 - .../src/google/protobuf/stubs/bytestream.cc | 12 +- .../src/google/protobuf/stubs/bytestream.h | 53 +- .../src/google/protobuf/stubs/callback.h | 99 +- .../src/google/protobuf/stubs/casts.h | 23 +- .../src/google/protobuf/stubs/common.cc | 261 +- .../src/google/protobuf/stubs/common.h | 101 +- .../src/google/protobuf/stubs/fastmem.h | 153 - .../protobuf/src/google/protobuf/stubs/hash.h | 350 +- .../src/google/protobuf/stubs/int128.cc | 75 +- .../src/google/protobuf/stubs/int128.h | 16 +- .../src/google/protobuf/stubs/logging.h | 38 +- .../src/google/protobuf/stubs/macros.h | 87 +- .../src/google/protobuf/stubs/map_util.h | 47 +- .../src/google/protobuf/stubs/mathlimits.cc | 144 - .../src/google/protobuf/stubs/mathlimits.h | 303 - .../src/google/protobuf/stubs/mathutil.h | 141 - .../src/google/protobuf/stubs/mutex.h | 146 +- .../src/google/protobuf/stubs/once.cc | 99 - .../protobuf/src/google/protobuf/stubs/once.h | 136 +- .../google/protobuf/stubs/platform_macros.h | 12 +- .../protobuf/src/google/protobuf/stubs/port.h | 285 +- .../src/google/protobuf/stubs/scoped_ptr.h | 236 - .../src/google/protobuf/stubs/shared_ptr.h | 471 - .../src/google/protobuf/stubs/singleton.h | 68 - .../src/google/protobuf/stubs/status.cc | 194 +- .../src/google/protobuf/stubs/status.h | 170 +- .../src/google/protobuf/stubs/status_macros.h | 89 - .../src/google/protobuf/stubs/statusor.h | 259 - .../src/google/protobuf/stubs/stl_util.h | 70 +- .../src/google/protobuf/stubs/stringpiece.cc | 102 +- .../src/google/protobuf/stubs/stringpiece.h | 281 +- .../src/google/protobuf/stubs/stringprintf.cc | 27 +- .../src/google/protobuf/stubs/stringprintf.h | 21 +- .../protobuf/stubs/structurally_valid.cc | 45 +- .../src/google/protobuf/stubs/strutil.cc | 663 +- .../src/google/protobuf/stubs/strutil.h | 412 +- .../src/google/protobuf/stubs/substitute.cc | 38 +- .../src/google/protobuf/stubs/substitute.h | 64 +- .../src/google/protobuf/stubs/template_util.h | 138 - .../src/google/protobuf/stubs/time.cc | 365 - .../src/google/protobuf/stubs/type_traits.h | 364 - .../src/google/protobuf/text_format.cc | 1720 +- .../src/google/protobuf/text_format.h | 406 +- .../src/google/protobuf/timestamp.pb.cc | 431 - .../src/google/protobuf/timestamp.pb.h | 232 - .../protobuf/src/google/protobuf/type.pb.cc | 2844 - .../protobuf/src/google/protobuf/type.pb.h | 2383 - .../src/google/protobuf/unknown_field_set.cc | 238 +- .../src/google/protobuf/unknown_field_set.h | 216 +- .../protobuf/util/delimited_message_util.cc | 79 - .../protobuf/util/delimited_message_util.h | 66 - .../google/protobuf/util/field_comparator.cc | 208 - .../google/protobuf/util/field_comparator.h | 259 - .../google/protobuf/util/field_mask_util.cc | 690 - .../google/protobuf/util/field_mask_util.h | 236 - .../google/protobuf/util/internal/constants.h | 103 - .../protobuf/util/internal/datapiece.cc | 406 - .../google/protobuf/util/internal/datapiece.h | 219 - .../internal/default_value_objectwriter.cc | 647 - .../internal/default_value_objectwriter.h | 324 - .../protobuf/util/internal/error_listener.h | 103 - .../util/internal/expecting_objectwriter.h | 238 - .../util/internal/field_mask_utility.cc | 220 - .../util/internal/field_mask_utility.h | 72 - .../protobuf/util/internal/json_escaping.cc | 356 - .../protobuf/util/internal/json_escaping.h | 91 - .../util/internal/json_objectwriter.cc | 196 - .../util/internal/json_objectwriter.h | 233 - .../util/internal/json_stream_parser.cc | 864 - .../util/internal/json_stream_parser.h | 272 - .../protobuf/util/internal/location_tracker.h | 65 - .../util/internal/mock_error_listener.h | 63 - .../util/internal/object_location_tracker.h | 64 - .../protobuf/util/internal/object_source.h | 79 - .../protobuf/util/internal/object_writer.cc | 92 - .../protobuf/util/internal/object_writer.h | 146 - .../protobuf/util/internal/proto_writer.cc | 799 - .../protobuf/util/internal/proto_writer.h | 353 - .../util/internal/protostream_objectsource.cc | 1136 - .../util/internal/protostream_objectsource.h | 326 - .../util/internal/protostream_objectwriter.cc | 1274 - .../util/internal/protostream_objectwriter.h | 408 - .../util/internal/structured_objectwriter.h | 118 - .../protobuf/util/internal/type_info.cc | 179 - .../google/protobuf/util/internal/type_info.h | 92 - .../google/protobuf/util/internal/utility.cc | 409 - .../google/protobuf/util/internal/utility.h | 208 - .../src/google/protobuf/util/json_util.cc | 252 - .../src/google/protobuf/util/json_util.h | 200 - .../protobuf/util/message_differencer.cc | 1756 - .../protobuf/util/message_differencer.h | 879 - .../src/google/protobuf/util/package_info.h | 46 - .../src/google/protobuf/util/time_util.cc | 504 - .../src/google/protobuf/util/time_util.h | 296 - .../src/google/protobuf/util/type_resolver.h | 77 - .../protobuf/util/type_resolver_util.cc | 247 - .../google/protobuf/util/type_resolver_util.h | 54 - .../src/google/protobuf/wire_format.cc | 1289 +- .../src/google/protobuf/wire_format.h | 235 +- .../src/google/protobuf/wire_format_lite.cc | 594 +- .../src/google/protobuf/wire_format_lite.h | 1865 +- .../google/protobuf/wire_format_lite_inl.h | 1077 - .../src/google/protobuf/wrappers.pb.cc | 2796 - .../src/google/protobuf/wrappers.pb.h | 1487 - modules/dnn/CMakeLists.txt | 7 + modules/dnn/misc/caffe/opencv-caffe.pb.cc | 46302 +++++++--------- modules/dnn/misc/caffe/opencv-caffe.pb.h | 43632 +++++++++------ modules/dnn/misc/onnx/opencv-onnx.pb.cc | 8215 ++- modules/dnn/misc/onnx/opencv-onnx.pb.h | 7967 +-- modules/dnn/misc/tensorflow/attr_value.pb.cc | 2213 +- modules/dnn/misc/tensorflow/attr_value.pb.h | 1809 +- modules/dnn/misc/tensorflow/function.pb.cc | 2125 +- modules/dnn/misc/tensorflow/function.pb.h | 1559 +- modules/dnn/misc/tensorflow/graph.pb.cc | 1491 +- modules/dnn/misc/tensorflow/graph.pb.h | 1116 +- modules/dnn/misc/tensorflow/op_def.pb.cc | 3410 +- modules/dnn/misc/tensorflow/op_def.pb.h | 2868 +- modules/dnn/misc/tensorflow/tensor.pb.cc | 1248 +- modules/dnn/misc/tensorflow/tensor.pb.h | 984 +- .../dnn/misc/tensorflow/tensor_shape.pb.cc | 881 +- modules/dnn/misc/tensorflow/tensor_shape.pb.h | 560 +- modules/dnn/misc/tensorflow/types.pb.cc | 120 +- modules/dnn/misc/tensorflow/types.pb.h | 93 +- modules/dnn/misc/tensorflow/versions.pb.cc | 552 +- modules/dnn/misc/tensorflow/versions.pb.h | 311 +- 247 files changed, 113839 insertions(+), 142988 deletions(-) delete mode 100644 3rdparty/protobuf/src/google/protobuf/any.pb.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/any.pb.h create mode 100644 3rdparty/protobuf/src/google/protobuf/any_lite.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/api.pb.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/api.pb.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/duration.pb.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/duration.pb.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/empty.pb.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/empty.pb.h rename 3rdparty/protobuf/src/google/protobuf/{stubs/time.h => explicitly_constructed.h} (52%) create mode 100644 3rdparty/protobuf/src/google/protobuf/extension_set_inl.h create mode 100644 3rdparty/protobuf/src/google/protobuf/field_access_listener.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/field_mask.pb.cc create mode 100644 3rdparty/protobuf/src/google/protobuf/generated_message_bases.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/generated_message_table_driven.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/generated_message_table_driven_lite.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/generated_message_table_driven_lite.h create mode 100644 3rdparty/protobuf/src/google/protobuf/generated_message_tctable_decl.h create mode 100644 3rdparty/protobuf/src/google/protobuf/generated_message_tctable_impl.h create mode 100644 3rdparty/protobuf/src/google/protobuf/generated_message_tctable_impl.inc rename 3rdparty/protobuf/src/google/protobuf/{stubs/statusor.cc => implicit_weak_message.cc} (64%) create mode 100644 3rdparty/protobuf/src/google/protobuf/implicit_weak_message.h create mode 100644 3rdparty/protobuf/src/google/protobuf/inlined_string_field.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/io/coded_stream_inl.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/io/gzip_stream.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/io/gzip_stream.h rename 3rdparty/protobuf/src/google/protobuf/{stubs => io}/io_win32.cc (79%) rename 3rdparty/protobuf/src/google/protobuf/{stubs => io}/io_win32.h (58%) delete mode 100644 3rdparty/protobuf/src/google/protobuf/io/printer.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/io/printer.h rename 3rdparty/protobuf/src/google/protobuf/{util/internal/error_listener.cc => map.cc} (92%) delete mode 100644 3rdparty/protobuf/src/google/protobuf/package_info.h create mode 100644 3rdparty/protobuf/src/google/protobuf/parse_context.cc create mode 100644 3rdparty/protobuf/src/google/protobuf/parse_context.h rename 3rdparty/protobuf/src/google/protobuf/{service.cc => port.h} (82%) create mode 100644 3rdparty/protobuf/src/google/protobuf/port_def.inc create mode 100644 3rdparty/protobuf/src/google/protobuf/port_undef.inc create mode 100644 3rdparty/protobuf/src/google/protobuf/repeated_ptr_field.cc create mode 100644 3rdparty/protobuf/src/google/protobuf/repeated_ptr_field.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/service.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/source_context.pb.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/source_context.pb.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/struct.pb.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/struct.pb.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomic_sequence_num.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_arm64_gcc.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_arm_gcc.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_arm_qnx.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_generic_c11_atomic.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_generic_gcc.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_mips_gcc.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_power.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_ppc_gcc.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_solaris.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_tsan.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_x86_gcc.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_x86_gcc.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_x86_msvc.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_x86_msvc.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/fastmem.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/mathlimits.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/mathlimits.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/mathutil.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/once.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/scoped_ptr.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/shared_ptr.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/singleton.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/status_macros.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/statusor.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/template_util.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/time.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/type_traits.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/timestamp.pb.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/timestamp.pb.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/type.pb.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/type.pb.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/delimited_message_util.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/delimited_message_util.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/field_comparator.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/field_comparator.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/field_mask_util.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/field_mask_util.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/constants.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/datapiece.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/datapiece.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/default_value_objectwriter.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/default_value_objectwriter.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/error_listener.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/expecting_objectwriter.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/field_mask_utility.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/field_mask_utility.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/json_escaping.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/json_escaping.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/json_objectwriter.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/json_objectwriter.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/json_stream_parser.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/json_stream_parser.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/location_tracker.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/mock_error_listener.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/object_location_tracker.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/object_source.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/object_writer.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/object_writer.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/proto_writer.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/proto_writer.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/protostream_objectsource.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/protostream_objectsource.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/protostream_objectwriter.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/protostream_objectwriter.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/structured_objectwriter.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/type_info.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/type_info.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/utility.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/utility.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/json_util.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/json_util.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/message_differencer.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/message_differencer.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/package_info.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/time_util.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/time_util.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/type_resolver.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/type_resolver_util.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/type_resolver_util.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/wire_format_lite_inl.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/wrappers.pb.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/wrappers.pb.h diff --git a/3rdparty/protobuf/CMakeLists.txt b/3rdparty/protobuf/CMakeLists.txt index f249d2dcc3..6de8148e58 100644 --- a/3rdparty/protobuf/CMakeLists.txt +++ b/3rdparty/protobuf/CMakeLists.txt @@ -13,6 +13,9 @@ if(MSVC) /wd4701 /wd4703 # potentially uninitialized local/pointer variable 'value' used /wd4505 # unreferenced local function has been removed ) + if(MSVC_VERSION LESS 1910) # MSVS 2015 + ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4309) # 'static_cast': truncation of constant value + endif() else() #NOTE: -Wno-invalid-offsetof was used as solution for invalid offset warning on protobuf #3450 ocv_warnings_disable(CMAKE_CXX_FLAGS -Wno-deprecated -Wmissing-prototypes -Wmissing-declarations -Wshadow @@ -49,98 +52,101 @@ endfunction() set(PROTOBUF_ROOT "${CMAKE_CURRENT_LIST_DIR}") -if(MSVC) - set(ATOMICOPS_INTERNALS ${PROTOBUF_ROOT}/src/google/protobuf/stubs/atomicops_internals_x86_msvc.cc) -else() - set(ATOMICOPS_INTERNALS ${PROTOBUF_ROOT}/src/google/protobuf/stubs/atomicops_internals_x86_gcc.cc) -endif() - append_if_exist(Protobuf_SRCS # libprotobuf-lite + ${PROTOBUF_ROOT}/src/google/protobuf/any_lite.cc ${PROTOBUF_ROOT}/src/google/protobuf/arena.cc ${PROTOBUF_ROOT}/src/google/protobuf/arenastring.cc ${PROTOBUF_ROOT}/src/google/protobuf/extension_set.cc - ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_table_driven_lite.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/generated_enum_util.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_table_driven_lite.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_tctable_lite.cc ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_util.cc + ${PROTOBUF_ROOT}/src/google/protobuf/implicit_weak_message.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/inlined_string_field.cc ${PROTOBUF_ROOT}/src/google/protobuf/io/coded_stream.cc + ${PROTOBUF_ROOT}/src/google/protobuf/io/io_win32.cc + ${PROTOBUF_ROOT}/src/google/protobuf/io/strtod.cc ${PROTOBUF_ROOT}/src/google/protobuf/io/zero_copy_stream.cc + ${PROTOBUF_ROOT}/src/google/protobuf/io/zero_copy_stream_impl.cc ${PROTOBUF_ROOT}/src/google/protobuf/io/zero_copy_stream_impl_lite.cc + ${PROTOBUF_ROOT}/src/google/protobuf/map.cc ${PROTOBUF_ROOT}/src/google/protobuf/message_lite.cc + ${PROTOBUF_ROOT}/src/google/protobuf/parse_context.cc ${PROTOBUF_ROOT}/src/google/protobuf/repeated_field.cc - ${ATOMICOPS_INTERNALS} + ${PROTOBUF_ROOT}/src/google/protobuf/repeated_ptr_field.cc ${PROTOBUF_ROOT}/src/google/protobuf/stubs/bytestream.cc ${PROTOBUF_ROOT}/src/google/protobuf/stubs/common.cc ${PROTOBUF_ROOT}/src/google/protobuf/stubs/int128.cc - ${PROTOBUF_ROOT}/src/google/protobuf/stubs/io_win32.cc - ${PROTOBUF_ROOT}/src/google/protobuf/stubs/once.cc ${PROTOBUF_ROOT}/src/google/protobuf/stubs/status.cc - ${PROTOBUF_ROOT}/src/google/protobuf/stubs/statusor.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/stubs/statusor.cc ${PROTOBUF_ROOT}/src/google/protobuf/stubs/stringpiece.cc ${PROTOBUF_ROOT}/src/google/protobuf/stubs/stringprintf.cc ${PROTOBUF_ROOT}/src/google/protobuf/stubs/structurally_valid.cc ${PROTOBUF_ROOT}/src/google/protobuf/stubs/strutil.cc - ${PROTOBUF_ROOT}/src/google/protobuf/stubs/time.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/stubs/time.cc ${PROTOBUF_ROOT}/src/google/protobuf/wire_format_lite.cc + # libprotobuf ${PROTOBUF_ROOT}/src/google/protobuf/any.cc - ${PROTOBUF_ROOT}/src/google/protobuf/any.pb.cc - ${PROTOBUF_ROOT}/src/google/protobuf/api.pb.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/any.pb.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/api.pb.cc # ${PROTOBUF_ROOT}/src/google/protobuf/compiler/importer.cc # ${PROTOBUF_ROOT}/src/google/protobuf/compiler/parser.cc ${PROTOBUF_ROOT}/src/google/protobuf/descriptor.cc ${PROTOBUF_ROOT}/src/google/protobuf/descriptor.pb.cc ${PROTOBUF_ROOT}/src/google/protobuf/descriptor_database.cc - ${PROTOBUF_ROOT}/src/google/protobuf/duration.pb.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/duration.pb.cc ${PROTOBUF_ROOT}/src/google/protobuf/dynamic_message.cc - ${PROTOBUF_ROOT}/src/google/protobuf/empty.pb.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/empty.pb.cc ${PROTOBUF_ROOT}/src/google/protobuf/extension_set_heavy.cc - ${PROTOBUF_ROOT}/src/google/protobuf/field_mask.pb.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/field_mask.pb.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_bases.cc ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_reflection.cc - ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_table_driven.cc - ${PROTOBUF_ROOT}/src/google/protobuf/io/gzip_stream.cc - ${PROTOBUF_ROOT}/src/google/protobuf/io/printer.cc - ${PROTOBUF_ROOT}/src/google/protobuf/io/strtod.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_table_driven.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_tctable_full.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/io/gzip_stream.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/io/printer.cc ${PROTOBUF_ROOT}/src/google/protobuf/io/tokenizer.cc - ${PROTOBUF_ROOT}/src/google/protobuf/io/zero_copy_stream_impl.cc ${PROTOBUF_ROOT}/src/google/protobuf/map_field.cc ${PROTOBUF_ROOT}/src/google/protobuf/message.cc ${PROTOBUF_ROOT}/src/google/protobuf/reflection_ops.cc - ${PROTOBUF_ROOT}/src/google/protobuf/service.cc - ${PROTOBUF_ROOT}/src/google/protobuf/source_context.pb.cc - ${PROTOBUF_ROOT}/src/google/protobuf/struct.pb.cc - ${PROTOBUF_ROOT}/src/google/protobuf/stubs/mathlimits.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/service.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/source_context.pb.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/struct.pb.cc ${PROTOBUF_ROOT}/src/google/protobuf/stubs/substitute.cc ${PROTOBUF_ROOT}/src/google/protobuf/text_format.cc - ${PROTOBUF_ROOT}/src/google/protobuf/timestamp.pb.cc - ${PROTOBUF_ROOT}/src/google/protobuf/type.pb.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/timestamp.pb.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/type.pb.cc ${PROTOBUF_ROOT}/src/google/protobuf/unknown_field_set.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/delimited_message_util.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/field_comparator.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/field_mask_util.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/datapiece.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/default_value_objectwriter.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/delimited_message_util.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/field_comparator.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/field_mask_util.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/datapiece.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/default_value_objectwriter.cc # ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/error_listener.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/field_mask_utility.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/json_escaping.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/json_objectwriter.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/json_stream_parser.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/object_writer.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/proto_writer.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/protostream_objectsource.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/protostream_objectwriter.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/type_info.cc -# ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/type_info_test_helper.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/utility.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/json_util.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/message_differencer.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/time_util.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/type_resolver_util.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/field_mask_utility.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/json_escaping.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/json_objectwriter.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/json_stream_parser.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/object_writer.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/proto_writer.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/protostream_objectsource.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/protostream_objectwriter.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/type_info.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/utility.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/json_util.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/message_differencer.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/time_util.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/type_resolver_util.cc ${PROTOBUF_ROOT}/src/google/protobuf/wire_format.cc - ${PROTOBUF_ROOT}/src/google/protobuf/wrappers.pb.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/wrappers.pb.cc ) -include_directories(BEFORE "${PROTOBUF_ROOT}/src") # ensure using if own headers: https://github.com/opencv/opencv/issues/13328 + +include_directories(BEFORE "${PROTOBUF_ROOT}/src") # ensure using of own headers: https://github.com/opencv/opencv/issues/13328 + add_library(libprotobuf STATIC ${OPENCV_3RDPARTY_EXCLUDE_FROM_ALL} ${Protobuf_SRCS}) target_include_directories(libprotobuf SYSTEM PUBLIC $) set_target_properties(libprotobuf diff --git a/3rdparty/protobuf/LICENSE b/3rdparty/protobuf/LICENSE index f028c82324..19b305b000 100644 --- a/3rdparty/protobuf/LICENSE +++ b/3rdparty/protobuf/LICENSE @@ -1,14 +1,4 @@ -This license applies to all parts of Protocol Buffers except the following: - - - Atomicops support for generic gcc, located in - src/google/protobuf/stubs/atomicops_internals_generic_gcc.h. - This file is copyrighted by Red Hat Inc. - - - Atomicops support for AIX/POWER, located in - src/google/protobuf/stubs/atomicops_internals_power.h. - This file is copyrighted by Bloomberg Finance LP. - -Copyright 2014, Google Inc. All rights reserved. +Copyright 2008 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/3rdparty/protobuf/README.md b/3rdparty/protobuf/README.md index 30404036a8..32aa03b64c 100644 --- a/3rdparty/protobuf/README.md +++ b/3rdparty/protobuf/README.md @@ -1,3 +1,3 @@ Project: Protocol Buffers - Google's data interchange format Source code: https://github.com/protocolbuffers/protobuf -Version: 3.5.2 +Version: 3.19.1 diff --git a/3rdparty/protobuf/src/google/protobuf/any.cc b/3rdparty/protobuf/src/google/protobuf/any.cc index 83edba5788..73c002f604 100644 --- a/3rdparty/protobuf/src/google/protobuf/any.cc +++ b/3rdparty/protobuf/src/google/protobuf/any.cc @@ -30,85 +30,54 @@ #include +#include +#include #include +#include + +#include namespace google { namespace protobuf { namespace internal { -namespace { -string GetTypeUrl(const Descriptor* message, - const string& type_url_prefix) { - if (!type_url_prefix.empty() && - type_url_prefix[type_url_prefix.size() - 1] == '/') { - return type_url_prefix + message->full_name(); - } else { - return type_url_prefix + "/" + message->full_name(); - } -} -} // namespace - -const char kAnyFullTypeName[] = "google.protobuf.Any"; -const char kTypeGoogleApisComPrefix[] = "type.googleapis.com/"; -const char kTypeGoogleProdComPrefix[] = "type.googleprod.com/"; - -AnyMetadata::AnyMetadata(UrlType* type_url, ValueType* value) - : type_url_(type_url), value_(value) { +bool AnyMetadata::PackFrom(Arena* arena, const Message& message) { + return PackFrom(arena, message, kTypeGoogleApisComPrefix); } -void AnyMetadata::PackFrom(const Message& message) { - PackFrom(message, kTypeGoogleApisComPrefix); -} - -void AnyMetadata::PackFrom(const Message& message, - const string& type_url_prefix) { - type_url_->SetNoArena(&::google::protobuf::internal::GetEmptyString(), - GetTypeUrl(message.GetDescriptor(), type_url_prefix)); - message.SerializeToString(value_->MutableNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited())); +bool AnyMetadata::PackFrom(Arena* arena, const Message& message, + StringPiece type_url_prefix) { + type_url_->Set( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString(), + GetTypeUrl(message.GetDescriptor()->full_name(), type_url_prefix), arena); + return message.SerializeToString( + value_->Mutable(ArenaStringPtr::EmptyDefault{}, arena)); } bool AnyMetadata::UnpackTo(Message* message) const { - if (!InternalIs(message->GetDescriptor())) { + if (!InternalIs(message->GetDescriptor()->full_name())) { return false; } - return message->ParseFromString(value_->GetNoArena()); + return message->ParseFromString(value_->Get()); } -bool AnyMetadata::InternalIs(const Descriptor* descriptor) const { - const string type_url = type_url_->GetNoArena(); - string full_name; - if (!ParseAnyTypeUrl(type_url, &full_name)) { - return false; - } - return full_name == descriptor->full_name(); -} - -bool ParseAnyTypeUrl(const string& type_url, string* full_type_name) { - size_t pos = type_url.find_last_of("/"); - if (pos == string::npos || pos + 1 == type_url.size()) { - return false; - } - *full_type_name = type_url.substr(pos + 1); - return true; -} - - bool GetAnyFieldDescriptors(const Message& message, const FieldDescriptor** type_url_field, const FieldDescriptor** value_field) { - const Descriptor* descriptor = message.GetDescriptor(); - if (descriptor->full_name() != kAnyFullTypeName) { - return false; - } - *type_url_field = descriptor->FindFieldByNumber(1); - *value_field = descriptor->FindFieldByNumber(2); - return (*type_url_field != NULL && - (*type_url_field)->type() == FieldDescriptor::TYPE_STRING && - *value_field != NULL && - (*value_field)->type() == FieldDescriptor::TYPE_BYTES); + const Descriptor* descriptor = message.GetDescriptor(); + if (descriptor->full_name() != kAnyFullTypeName) { + return false; + } + *type_url_field = descriptor->FindFieldByNumber(1); + *value_field = descriptor->FindFieldByNumber(2); + return (*type_url_field != nullptr && + (*type_url_field)->type() == FieldDescriptor::TYPE_STRING && + *value_field != nullptr && + (*value_field)->type() == FieldDescriptor::TYPE_BYTES); } } // namespace internal } // namespace protobuf } // namespace google + +#include diff --git a/3rdparty/protobuf/src/google/protobuf/any.h b/3rdparty/protobuf/src/google/protobuf/any.h index c2c27ec333..e8336fa14a 100644 --- a/3rdparty/protobuf/src/google/protobuf/any.h +++ b/3rdparty/protobuf/src/google/protobuf/any.h @@ -34,49 +34,89 @@ #include #include -#include -#include #include +#include + +#include namespace google { namespace protobuf { + +class FieldDescriptor; +class Message; + namespace internal { +extern const char kAnyFullTypeName[]; // "google.protobuf.Any". +extern const char kTypeGoogleApisComPrefix[]; // "type.googleapis.com/". +extern const char kTypeGoogleProdComPrefix[]; // "type.googleprod.com/". + +std::string GetTypeUrl(StringPiece message_name, + StringPiece type_url_prefix); + // Helper class used to implement google::protobuf::Any. -class LIBPROTOBUF_EXPORT AnyMetadata { +class PROTOBUF_EXPORT AnyMetadata { typedef ArenaStringPtr UrlType; typedef ArenaStringPtr ValueType; public: // AnyMetadata does not take ownership of "type_url" and "value". - AnyMetadata(UrlType* type_url, ValueType* value); + constexpr AnyMetadata(UrlType* type_url, ValueType* value) + : type_url_(type_url), value_(value) {} // Packs a message using the default type URL prefix: "type.googleapis.com". // The resulted type URL will be "type.googleapis.com/". - void PackFrom(const Message& message); + // Returns false if serializing the message failed. + template + bool PackFrom(Arena* arena, const T& message) { + return InternalPackFrom(arena, message, kTypeGoogleApisComPrefix, + T::FullMessageName()); + } + + bool PackFrom(Arena* arena, const Message& message); + // Packs a message using the given type URL prefix. The type URL will be // constructed by concatenating the message type's full name to the prefix - // with an optional "/" separator if the prefix doesn't already end up "/". + // with an optional "/" separator if the prefix doesn't already end with "/". // For example, both PackFrom(message, "type.googleapis.com") and // PackFrom(message, "type.googleapis.com/") yield the same result type // URL: "type.googleapis.com/". - void PackFrom(const Message& message, const string& type_url_prefix); + // Returns false if serializing the message failed. + template + bool PackFrom(Arena* arena, const T& message, + StringPiece type_url_prefix) { + return InternalPackFrom(arena, message, type_url_prefix, + T::FullMessageName()); + } + + bool PackFrom(Arena* arena, const Message& message, + StringPiece type_url_prefix); // Unpacks the payload into the given message. Returns false if the message's // type doesn't match the type specified in the type URL (i.e., the full // name after the last "/" of the type URL doesn't match the message's actual // full name) or parsing the payload has failed. + template + bool UnpackTo(T* message) const { + return InternalUnpackTo(T::FullMessageName(), message); + } + bool UnpackTo(Message* message) const; // Checks whether the type specified in the type URL matches the given type. - // A type is consdiered matching if its full name matches the full name after + // A type is considered matching if its full name matches the full name after // the last "/" in the type URL. - template + template bool Is() const { - return InternalIs(T::default_instance().GetDescriptor()); + return InternalIs(T::FullMessageName()); } private: - bool InternalIs(const Descriptor* message) const; + bool InternalPackFrom(Arena* arena, const MessageLite& message, + StringPiece type_url_prefix, + StringPiece type_name); + bool InternalUnpackTo(StringPiece type_name, + MessageLite* message) const; + bool InternalIs(StringPiece type_name) const; UrlType* type_url_; ValueType* value_; @@ -84,15 +124,22 @@ class LIBPROTOBUF_EXPORT AnyMetadata { GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(AnyMetadata); }; -extern const char kAnyFullTypeName[]; // "google.protobuf.Any". -extern const char kTypeGoogleApisComPrefix[]; // "type.googleapis.com/". -extern const char kTypeGoogleProdComPrefix[]; // "type.googleprod.com/". - // Get the proto type name from Any::type_url value. For example, passing // "type.googleapis.com/rpc.QueryOrigin" will return "rpc.QueryOrigin" in // *full_type_name. Returns false if the type_url does not have a "/" // in the type url separating the full type name. -bool ParseAnyTypeUrl(const string& type_url, string* full_type_name); +// +// NOTE: this function is available publicly as: +// google::protobuf::Any() // static method on the generated message type. +bool ParseAnyTypeUrl(StringPiece type_url, std::string* full_type_name); + +// Get the proto type name and prefix from Any::type_url value. For example, +// passing "type.googleapis.com/rpc.QueryOrigin" will return +// "type.googleapis.com/" in *url_prefix and "rpc.QueryOrigin" in +// *full_type_name. Returns false if the type_url does not have a "/" in the +// type url separating the full type name. +bool ParseAnyTypeUrl(StringPiece type_url, std::string* url_prefix, + std::string* full_type_name); // See if message is of type google.protobuf.Any, if so, return the descriptors // for "type_url" and "value" fields. @@ -102,6 +149,8 @@ bool GetAnyFieldDescriptors(const Message& message, } // namespace internal } // namespace protobuf - } // namespace google + +#include + #endif // GOOGLE_PROTOBUF_ANY_H__ diff --git a/3rdparty/protobuf/src/google/protobuf/any.pb.cc b/3rdparty/protobuf/src/google/protobuf/any.pb.cc deleted file mode 100644 index 296874770e..0000000000 --- a/3rdparty/protobuf/src/google/protobuf/any.pb.cc +++ /dev/null @@ -1,440 +0,0 @@ -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: google/protobuf/any.proto - -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -// This is a temporary google only hack -#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS -#include "third_party/protobuf/version.h" -#endif -// @@protoc_insertion_point(includes) -namespace google { -namespace protobuf { -class AnyDefaultTypeInternal { - public: - ::google::protobuf::internal::ExplicitlyConstructed - _instance; -} _Any_default_instance_; -} // namespace protobuf -} // namespace google -namespace protobuf_google_2fprotobuf_2fany_2eproto { -void InitDefaultsAnyImpl() { - GOOGLE_PROTOBUF_VERIFY_VERSION; - -#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS - ::google::protobuf::internal::InitProtobufDefaultsForceUnique(); -#else - ::google::protobuf::internal::InitProtobufDefaults(); -#endif // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS - { - void* ptr = &::google::protobuf::_Any_default_instance_; - new (ptr) ::google::protobuf::Any(); - ::google::protobuf::internal::OnShutdownDestroyMessage(ptr); - } - ::google::protobuf::Any::InitAsDefaultInstance(); -} - -void InitDefaultsAny() { - static GOOGLE_PROTOBUF_DECLARE_ONCE(once); - ::google::protobuf::GoogleOnceInit(&once, &InitDefaultsAnyImpl); -} - -::google::protobuf::Metadata file_level_metadata[1]; - -const ::google::protobuf::uint32 TableStruct::offsets[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { - ~0u, // no _has_bits_ - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Any, _internal_metadata_), - ~0u, // no _extensions_ - ~0u, // no _oneof_case_ - ~0u, // no _weak_field_map_ - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Any, type_url_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Any, value_), -}; -static const ::google::protobuf::internal::MigrationSchema schemas[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { - { 0, -1, sizeof(::google::protobuf::Any)}, -}; - -static ::google::protobuf::Message const * const file_default_instances[] = { - reinterpret_cast(&::google::protobuf::_Any_default_instance_), -}; - -void protobuf_AssignDescriptors() { - AddDescriptors(); - ::google::protobuf::MessageFactory* factory = NULL; - AssignDescriptors( - "google/protobuf/any.proto", schemas, file_default_instances, TableStruct::offsets, factory, - file_level_metadata, NULL, NULL); -} - -void protobuf_AssignDescriptorsOnce() { - static GOOGLE_PROTOBUF_DECLARE_ONCE(once); - ::google::protobuf::GoogleOnceInit(&once, &protobuf_AssignDescriptors); -} - -void protobuf_RegisterTypes(const ::std::string&) GOOGLE_PROTOBUF_ATTRIBUTE_COLD; -void protobuf_RegisterTypes(const ::std::string&) { - protobuf_AssignDescriptorsOnce(); - ::google::protobuf::internal::RegisterAllTypes(file_level_metadata, 1); -} - -void AddDescriptorsImpl() { - InitDefaults(); - static const char descriptor[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { - "\n\031google/protobuf/any.proto\022\017google.prot" - "obuf\"&\n\003Any\022\020\n\010type_url\030\001 \001(\t\022\r\n\005value\030\002" - " \001(\014Bo\n\023com.google.protobufB\010AnyProtoP\001Z" - "%github.com/golang/protobuf/ptypes/any\242\002" - "\003GPB\252\002\036Google.Protobuf.WellKnownTypesb\006p" - "roto3" - }; - ::google::protobuf::DescriptorPool::InternalAddGeneratedFile( - descriptor, 205); - ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile( - "google/protobuf/any.proto", &protobuf_RegisterTypes); -} - -void AddDescriptors() { - static GOOGLE_PROTOBUF_DECLARE_ONCE(once); - ::google::protobuf::GoogleOnceInit(&once, &AddDescriptorsImpl); -} -// Force AddDescriptors() to be called at dynamic initialization time. -struct StaticDescriptorInitializer { - StaticDescriptorInitializer() { - AddDescriptors(); - } -} static_descriptor_initializer; -} // namespace protobuf_google_2fprotobuf_2fany_2eproto -namespace google { -namespace protobuf { - -// =================================================================== - -void Any::InitAsDefaultInstance() { -} -void Any::PackFrom(const ::google::protobuf::Message& message) { - _any_metadata_.PackFrom(message); -} - -void Any::PackFrom(const ::google::protobuf::Message& message, - const ::std::string& type_url_prefix) { - _any_metadata_.PackFrom(message, type_url_prefix); -} - -bool Any::UnpackTo(::google::protobuf::Message* message) const { - return _any_metadata_.UnpackTo(message); -} - -#if !defined(_MSC_VER) || _MSC_VER >= 1900 -const int Any::kTypeUrlFieldNumber; -const int Any::kValueFieldNumber; -#endif // !defined(_MSC_VER) || _MSC_VER >= 1900 - -Any::Any() - : ::google::protobuf::Message(), _internal_metadata_(NULL), _any_metadata_(&type_url_, &value_) { - if (GOOGLE_PREDICT_TRUE(this != internal_default_instance())) { - ::protobuf_google_2fprotobuf_2fany_2eproto::InitDefaultsAny(); - } - SharedCtor(); - // @@protoc_insertion_point(constructor:google.protobuf.Any) -} -Any::Any(const Any& from) - : ::google::protobuf::Message(), - _internal_metadata_(NULL), - _cached_size_(0), - _any_metadata_(&type_url_, &value_) { - _internal_metadata_.MergeFrom(from._internal_metadata_); - type_url_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.type_url().size() > 0) { - type_url_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.type_url_); - } - value_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.value().size() > 0) { - value_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.value_); - } - // @@protoc_insertion_point(copy_constructor:google.protobuf.Any) -} - -void Any::SharedCtor() { - type_url_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - value_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - _cached_size_ = 0; -} - -Any::~Any() { - // @@protoc_insertion_point(destructor:google.protobuf.Any) - SharedDtor(); -} - -void Any::SharedDtor() { - type_url_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - value_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} - -void Any::SetCachedSize(int size) const { - GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); - _cached_size_ = size; - GOOGLE_SAFE_CONCURRENT_WRITES_END(); -} -const ::google::protobuf::Descriptor* Any::descriptor() { - ::protobuf_google_2fprotobuf_2fany_2eproto::protobuf_AssignDescriptorsOnce(); - return ::protobuf_google_2fprotobuf_2fany_2eproto::file_level_metadata[kIndexInFileMessages].descriptor; -} - -const Any& Any::default_instance() { - ::protobuf_google_2fprotobuf_2fany_2eproto::InitDefaultsAny(); - return *internal_default_instance(); -} - -Any* Any::New(::google::protobuf::Arena* arena) const { - Any* n = new Any; - if (arena != NULL) { - arena->Own(n); - } - return n; -} - -void Any::Clear() { -// @@protoc_insertion_point(message_clear_start:google.protobuf.Any) - ::google::protobuf::uint32 cached_has_bits = 0; - // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; - - type_url_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - value_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - _internal_metadata_.Clear(); -} - -bool Any::MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) { -#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure - ::google::protobuf::uint32 tag; - // @@protoc_insertion_point(parse_start:google.protobuf.Any) - for (;;) { - ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); - tag = p.first; - if (!p.second) goto handle_unusual; - switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { - // string type_url = 1; - case 1: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(10u /* 10 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_type_url())); - DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->type_url().data(), static_cast(this->type_url().length()), - ::google::protobuf::internal::WireFormatLite::PARSE, - "google.protobuf.Any.type_url")); - } else { - goto handle_unusual; - } - break; - } - - // bytes value = 2; - case 2: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(18u /* 18 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadBytes( - input, this->mutable_value())); - } else { - goto handle_unusual; - } - break; - } - - default: { - handle_unusual: - if (tag == 0) { - goto success; - } - DO_(::google::protobuf::internal::WireFormat::SkipField( - input, tag, _internal_metadata_.mutable_unknown_fields())); - break; - } - } - } -success: - // @@protoc_insertion_point(parse_success:google.protobuf.Any) - return true; -failure: - // @@protoc_insertion_point(parse_failure:google.protobuf.Any) - return false; -#undef DO_ -} - -void Any::SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const { - // @@protoc_insertion_point(serialize_start:google.protobuf.Any) - ::google::protobuf::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - // string type_url = 1; - if (this->type_url().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->type_url().data(), static_cast(this->type_url().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Any.type_url"); - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 1, this->type_url(), output); - } - - // bytes value = 2; - if (this->value().size() > 0) { - ::google::protobuf::internal::WireFormatLite::WriteBytesMaybeAliased( - 2, this->value(), output); - } - - if ((_internal_metadata_.have_unknown_fields() && ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) { - ::google::protobuf::internal::WireFormat::SerializeUnknownFields( - (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance()), output); - } - // @@protoc_insertion_point(serialize_end:google.protobuf.Any) -} - -::google::protobuf::uint8* Any::InternalSerializeWithCachedSizesToArray( - bool deterministic, ::google::protobuf::uint8* target) const { - (void)deterministic; // Unused - // @@protoc_insertion_point(serialize_to_array_start:google.protobuf.Any) - ::google::protobuf::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - // string type_url = 1; - if (this->type_url().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->type_url().data(), static_cast(this->type_url().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Any.type_url"); - target = - ::google::protobuf::internal::WireFormatLite::WriteStringToArray( - 1, this->type_url(), target); - } - - // bytes value = 2; - if (this->value().size() > 0) { - target = - ::google::protobuf::internal::WireFormatLite::WriteBytesToArray( - 2, this->value(), target); - } - - if ((_internal_metadata_.have_unknown_fields() && ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) { - target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray( - (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance()), target); - } - // @@protoc_insertion_point(serialize_to_array_end:google.protobuf.Any) - return target; -} - -size_t Any::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.Any) - size_t total_size = 0; - - if ((_internal_metadata_.have_unknown_fields() && ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) { - total_size += - ::google::protobuf::internal::WireFormat::ComputeUnknownFieldsSize( - (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance())); - } - // string type_url = 1; - if (this->type_url().size() > 0) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->type_url()); - } - - // bytes value = 2; - if (this->value().size() > 0) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::BytesSize( - this->value()); - } - - int cached_size = ::google::protobuf::internal::ToCachedSize(total_size); - GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); - _cached_size_ = cached_size; - GOOGLE_SAFE_CONCURRENT_WRITES_END(); - return total_size; -} - -void Any::MergeFrom(const ::google::protobuf::Message& from) { -// @@protoc_insertion_point(generalized_merge_from_start:google.protobuf.Any) - GOOGLE_DCHECK_NE(&from, this); - const Any* source = - ::google::protobuf::internal::DynamicCastToGenerated( - &from); - if (source == NULL) { - // @@protoc_insertion_point(generalized_merge_from_cast_fail:google.protobuf.Any) - ::google::protobuf::internal::ReflectionOps::Merge(from, this); - } else { - // @@protoc_insertion_point(generalized_merge_from_cast_success:google.protobuf.Any) - MergeFrom(*source); - } -} - -void Any::MergeFrom(const Any& from) { -// @@protoc_insertion_point(class_specific_merge_from_start:google.protobuf.Any) - GOOGLE_DCHECK_NE(&from, this); - _internal_metadata_.MergeFrom(from._internal_metadata_); - ::google::protobuf::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - if (from.type_url().size() > 0) { - - type_url_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.type_url_); - } - if (from.value().size() > 0) { - - value_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.value_); - } -} - -void Any::CopyFrom(const ::google::protobuf::Message& from) { -// @@protoc_insertion_point(generalized_copy_from_start:google.protobuf.Any) - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -void Any::CopyFrom(const Any& from) { -// @@protoc_insertion_point(class_specific_copy_from_start:google.protobuf.Any) - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -bool Any::IsInitialized() const { - return true; -} - -void Any::Swap(Any* other) { - if (other == this) return; - InternalSwap(other); -} -void Any::InternalSwap(Any* other) { - using std::swap; - type_url_.Swap(&other->type_url_); - value_.Swap(&other->value_); - _internal_metadata_.Swap(&other->_internal_metadata_); - swap(_cached_size_, other->_cached_size_); -} - -::google::protobuf::Metadata Any::GetMetadata() const { - protobuf_google_2fprotobuf_2fany_2eproto::protobuf_AssignDescriptorsOnce(); - return ::protobuf_google_2fprotobuf_2fany_2eproto::file_level_metadata[kIndexInFileMessages]; -} - - -// @@protoc_insertion_point(namespace_scope) -} // namespace protobuf -} // namespace google - -// @@protoc_insertion_point(global_scope) diff --git a/3rdparty/protobuf/src/google/protobuf/any.pb.h b/3rdparty/protobuf/src/google/protobuf/any.pb.h deleted file mode 100644 index 32847239be..0000000000 --- a/3rdparty/protobuf/src/google/protobuf/any.pb.h +++ /dev/null @@ -1,323 +0,0 @@ -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: google/protobuf/any.proto - -#ifndef PROTOBUF_google_2fprotobuf_2fany_2eproto__INCLUDED -#define PROTOBUF_google_2fprotobuf_2fany_2eproto__INCLUDED - -#include - -#include - -#if GOOGLE_PROTOBUF_VERSION < 3005000 -#error This file was generated by a newer version of protoc which is -#error incompatible with your Protocol Buffer headers. Please update -#error your headers. -#endif -#if 3005001 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION -#error This file was generated by an older version of protoc which is -#error incompatible with your Protocol Buffer headers. Please -#error regenerate this file with a newer version of protoc. -#endif - -#include -#include -#include -#include -#include -#include -#include -#include // IWYU pragma: export -#include // IWYU pragma: export -#include -#include -// @@protoc_insertion_point(includes) - -namespace protobuf_google_2fprotobuf_2fany_2eproto { -// Internal implementation detail -- do not use these members. -struct LIBPROTOBUF_EXPORT TableStruct { - static const ::google::protobuf::internal::ParseTableField entries[]; - static const ::google::protobuf::internal::AuxillaryParseTableField aux[]; - static const ::google::protobuf::internal::ParseTable schema[1]; - static const ::google::protobuf::internal::FieldMetadata field_metadata[]; - static const ::google::protobuf::internal::SerializationTable serialization_table[]; - static const ::google::protobuf::uint32 offsets[]; -}; -void LIBPROTOBUF_EXPORT AddDescriptors(); -void LIBPROTOBUF_EXPORT InitDefaultsAnyImpl(); -void LIBPROTOBUF_EXPORT InitDefaultsAny(); -inline void LIBPROTOBUF_EXPORT InitDefaults() { - InitDefaultsAny(); -} -} // namespace protobuf_google_2fprotobuf_2fany_2eproto -namespace google { -namespace protobuf { -class Any; -class AnyDefaultTypeInternal; -LIBPROTOBUF_EXPORT extern AnyDefaultTypeInternal _Any_default_instance_; -} // namespace protobuf -} // namespace google -namespace google { -namespace protobuf { - -// =================================================================== - -class LIBPROTOBUF_EXPORT Any : public ::google::protobuf::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Any) */ { - public: - Any(); - virtual ~Any(); - - Any(const Any& from); - - inline Any& operator=(const Any& from) { - CopyFrom(from); - return *this; - } - #if LANG_CXX11 - Any(Any&& from) noexcept - : Any() { - *this = ::std::move(from); - } - - inline Any& operator=(Any&& from) noexcept { - if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { - if (this != &from) InternalSwap(&from); - } else { - CopyFrom(from); - } - return *this; - } - #endif - static const ::google::protobuf::Descriptor* descriptor(); - static const Any& default_instance(); - - static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY - static inline const Any* internal_default_instance() { - return reinterpret_cast( - &_Any_default_instance_); - } - static PROTOBUF_CONSTEXPR int const kIndexInFileMessages = - 0; - - // implements Any ----------------------------------------------- - - void PackFrom(const ::google::protobuf::Message& message); - void PackFrom(const ::google::protobuf::Message& message, - const ::std::string& type_url_prefix); - bool UnpackTo(::google::protobuf::Message* message) const; - template bool Is() const { - return _any_metadata_.Is(); - } - - void Swap(Any* other); - friend void swap(Any& a, Any& b) { - a.Swap(&b); - } - - // implements Message ---------------------------------------------- - - inline Any* New() const PROTOBUF_FINAL { return New(NULL); } - - Any* New(::google::protobuf::Arena* arena) const PROTOBUF_FINAL; - void CopyFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL; - void MergeFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL; - void CopyFrom(const Any& from); - void MergeFrom(const Any& from); - void Clear() PROTOBUF_FINAL; - bool IsInitialized() const PROTOBUF_FINAL; - - size_t ByteSizeLong() const PROTOBUF_FINAL; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) PROTOBUF_FINAL; - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const PROTOBUF_FINAL; - ::google::protobuf::uint8* InternalSerializeWithCachedSizesToArray( - bool deterministic, ::google::protobuf::uint8* target) const PROTOBUF_FINAL; - int GetCachedSize() const PROTOBUF_FINAL { return _cached_size_; } - private: - void SharedCtor(); - void SharedDtor(); - void SetCachedSize(int size) const PROTOBUF_FINAL; - void InternalSwap(Any* other); - private: - inline ::google::protobuf::Arena* GetArenaNoVirtual() const { - return NULL; - } - inline void* MaybeArenaPtr() const { - return NULL; - } - public: - - ::google::protobuf::Metadata GetMetadata() const PROTOBUF_FINAL; - - // nested types ---------------------------------------------------- - - // accessors ------------------------------------------------------- - - // string type_url = 1; - void clear_type_url(); - static const int kTypeUrlFieldNumber = 1; - const ::std::string& type_url() const; - void set_type_url(const ::std::string& value); - #if LANG_CXX11 - void set_type_url(::std::string&& value); - #endif - void set_type_url(const char* value); - void set_type_url(const char* value, size_t size); - ::std::string* mutable_type_url(); - ::std::string* release_type_url(); - void set_allocated_type_url(::std::string* type_url); - - // bytes value = 2; - void clear_value(); - static const int kValueFieldNumber = 2; - const ::std::string& value() const; - void set_value(const ::std::string& value); - #if LANG_CXX11 - void set_value(::std::string&& value); - #endif - void set_value(const char* value); - void set_value(const void* value, size_t size); - ::std::string* mutable_value(); - ::std::string* release_value(); - void set_allocated_value(::std::string* value); - - // @@protoc_insertion_point(class_scope:google.protobuf.Any) - private: - - ::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_; - ::google::protobuf::internal::ArenaStringPtr type_url_; - ::google::protobuf::internal::ArenaStringPtr value_; - mutable int _cached_size_; - ::google::protobuf::internal::AnyMetadata _any_metadata_; - friend struct ::protobuf_google_2fprotobuf_2fany_2eproto::TableStruct; - friend void ::protobuf_google_2fprotobuf_2fany_2eproto::InitDefaultsAnyImpl(); -}; -// =================================================================== - - -// =================================================================== - -#ifdef __GNUC__ - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wstrict-aliasing" -#endif // __GNUC__ -// Any - -// string type_url = 1; -inline void Any::clear_type_url() { - type_url_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline const ::std::string& Any::type_url() const { - // @@protoc_insertion_point(field_get:google.protobuf.Any.type_url) - return type_url_.GetNoArena(); -} -inline void Any::set_type_url(const ::std::string& value) { - - type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); - // @@protoc_insertion_point(field_set:google.protobuf.Any.type_url) -} -#if LANG_CXX11 -inline void Any::set_type_url(::std::string&& value) { - - type_url_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); - // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Any.type_url) -} -#endif -inline void Any::set_type_url(const char* value) { - GOOGLE_DCHECK(value != NULL); - - type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); - // @@protoc_insertion_point(field_set_char:google.protobuf.Any.type_url) -} -inline void Any::set_type_url(const char* value, size_t size) { - - type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); - // @@protoc_insertion_point(field_set_pointer:google.protobuf.Any.type_url) -} -inline ::std::string* Any::mutable_type_url() { - - // @@protoc_insertion_point(field_mutable:google.protobuf.Any.type_url) - return type_url_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline ::std::string* Any::release_type_url() { - // @@protoc_insertion_point(field_release:google.protobuf.Any.type_url) - - return type_url_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline void Any::set_allocated_type_url(::std::string* type_url) { - if (type_url != NULL) { - - } else { - - } - type_url_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), type_url); - // @@protoc_insertion_point(field_set_allocated:google.protobuf.Any.type_url) -} - -// bytes value = 2; -inline void Any::clear_value() { - value_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline const ::std::string& Any::value() const { - // @@protoc_insertion_point(field_get:google.protobuf.Any.value) - return value_.GetNoArena(); -} -inline void Any::set_value(const ::std::string& value) { - - value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); - // @@protoc_insertion_point(field_set:google.protobuf.Any.value) -} -#if LANG_CXX11 -inline void Any::set_value(::std::string&& value) { - - value_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); - // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Any.value) -} -#endif -inline void Any::set_value(const char* value) { - GOOGLE_DCHECK(value != NULL); - - value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); - // @@protoc_insertion_point(field_set_char:google.protobuf.Any.value) -} -inline void Any::set_value(const void* value, size_t size) { - - value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); - // @@protoc_insertion_point(field_set_pointer:google.protobuf.Any.value) -} -inline ::std::string* Any::mutable_value() { - - // @@protoc_insertion_point(field_mutable:google.protobuf.Any.value) - return value_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline ::std::string* Any::release_value() { - // @@protoc_insertion_point(field_release:google.protobuf.Any.value) - - return value_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline void Any::set_allocated_value(::std::string* value) { - if (value != NULL) { - - } else { - - } - value_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); - // @@protoc_insertion_point(field_set_allocated:google.protobuf.Any.value) -} - -#ifdef __GNUC__ - #pragma GCC diagnostic pop -#endif // __GNUC__ - -// @@protoc_insertion_point(namespace_scope) - -} // namespace protobuf -} // namespace google - -// @@protoc_insertion_point(global_scope) - -#endif // PROTOBUF_google_2fprotobuf_2fany_2eproto__INCLUDED diff --git a/3rdparty/protobuf/src/google/protobuf/any_lite.cc b/3rdparty/protobuf/src/google/protobuf/any_lite.cc new file mode 100644 index 0000000000..a98559da14 --- /dev/null +++ b/3rdparty/protobuf/src/google/protobuf/any_lite.cc @@ -0,0 +1,99 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include + +#include +#include +#include +#include + +namespace google { +namespace protobuf { +namespace internal { + +std::string GetTypeUrl(StringPiece message_name, + StringPiece type_url_prefix) { + if (!type_url_prefix.empty() && + type_url_prefix[type_url_prefix.size() - 1] == '/') { + return StrCat(type_url_prefix, message_name); + } else { + return StrCat(type_url_prefix, "/", message_name); + } +} + +const char kAnyFullTypeName[] = "google.protobuf.Any"; +const char kTypeGoogleApisComPrefix[] = "type.googleapis.com/"; +const char kTypeGoogleProdComPrefix[] = "type.googleprod.com/"; + +bool AnyMetadata::InternalPackFrom(Arena* arena, const MessageLite& message, + StringPiece type_url_prefix, + StringPiece type_name) { + type_url_->Set(&::google::protobuf::internal::GetEmptyString(), + GetTypeUrl(type_name, type_url_prefix), arena); + return message.SerializeToString( + value_->Mutable(ArenaStringPtr::EmptyDefault{}, arena)); +} + +bool AnyMetadata::InternalUnpackTo(StringPiece type_name, + MessageLite* message) const { + if (!InternalIs(type_name)) { + return false; + } + return message->ParseFromString(value_->Get()); +} + +bool AnyMetadata::InternalIs(StringPiece type_name) const { + StringPiece type_url = type_url_->Get(); + return type_url.size() >= type_name.size() + 1 && + type_url[type_url.size() - type_name.size() - 1] == '/' && + HasSuffixString(type_url, type_name); +} + +bool ParseAnyTypeUrl(StringPiece type_url, std::string* url_prefix, + std::string* full_type_name) { + size_t pos = type_url.find_last_of('/'); + if (pos == std::string::npos || pos + 1 == type_url.size()) { + return false; + } + if (url_prefix) { + *url_prefix = std::string(type_url.substr(0, pos + 1)); + } + *full_type_name = std::string(type_url.substr(pos + 1)); + return true; +} + +bool ParseAnyTypeUrl(StringPiece type_url, std::string* full_type_name) { + return ParseAnyTypeUrl(type_url, nullptr, full_type_name); +} + +} // namespace internal +} // namespace protobuf +} // namespace google diff --git a/3rdparty/protobuf/src/google/protobuf/api.pb.cc b/3rdparty/protobuf/src/google/protobuf/api.pb.cc deleted file mode 100644 index 439a47c62d..0000000000 --- a/3rdparty/protobuf/src/google/protobuf/api.pb.cc +++ /dev/null @@ -1,1608 +0,0 @@ -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: google/protobuf/api.proto - -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -// This is a temporary google only hack -#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS -#include "third_party/protobuf/version.h" -#endif -// @@protoc_insertion_point(includes) -namespace google { -namespace protobuf { -class ApiDefaultTypeInternal { - public: - ::google::protobuf::internal::ExplicitlyConstructed - _instance; -} _Api_default_instance_; -class MethodDefaultTypeInternal { - public: - ::google::protobuf::internal::ExplicitlyConstructed - _instance; -} _Method_default_instance_; -class MixinDefaultTypeInternal { - public: - ::google::protobuf::internal::ExplicitlyConstructed - _instance; -} _Mixin_default_instance_; -} // namespace protobuf -} // namespace google -namespace protobuf_google_2fprotobuf_2fapi_2eproto { -void InitDefaultsApiImpl() { - GOOGLE_PROTOBUF_VERIFY_VERSION; - -#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS - ::google::protobuf::internal::InitProtobufDefaultsForceUnique(); -#else - ::google::protobuf::internal::InitProtobufDefaults(); -#endif // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS - protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsMethod(); - protobuf_google_2fprotobuf_2ftype_2eproto::InitDefaultsOption(); - protobuf_google_2fprotobuf_2fsource_5fcontext_2eproto::InitDefaultsSourceContext(); - protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsMixin(); - { - void* ptr = &::google::protobuf::_Api_default_instance_; - new (ptr) ::google::protobuf::Api(); - ::google::protobuf::internal::OnShutdownDestroyMessage(ptr); - } - ::google::protobuf::Api::InitAsDefaultInstance(); -} - -void InitDefaultsApi() { - static GOOGLE_PROTOBUF_DECLARE_ONCE(once); - ::google::protobuf::GoogleOnceInit(&once, &InitDefaultsApiImpl); -} - -void InitDefaultsMethodImpl() { - GOOGLE_PROTOBUF_VERIFY_VERSION; - -#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS - ::google::protobuf::internal::InitProtobufDefaultsForceUnique(); -#else - ::google::protobuf::internal::InitProtobufDefaults(); -#endif // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS - protobuf_google_2fprotobuf_2ftype_2eproto::InitDefaultsOption(); - { - void* ptr = &::google::protobuf::_Method_default_instance_; - new (ptr) ::google::protobuf::Method(); - ::google::protobuf::internal::OnShutdownDestroyMessage(ptr); - } - ::google::protobuf::Method::InitAsDefaultInstance(); -} - -void InitDefaultsMethod() { - static GOOGLE_PROTOBUF_DECLARE_ONCE(once); - ::google::protobuf::GoogleOnceInit(&once, &InitDefaultsMethodImpl); -} - -void InitDefaultsMixinImpl() { - GOOGLE_PROTOBUF_VERIFY_VERSION; - -#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS - ::google::protobuf::internal::InitProtobufDefaultsForceUnique(); -#else - ::google::protobuf::internal::InitProtobufDefaults(); -#endif // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS - { - void* ptr = &::google::protobuf::_Mixin_default_instance_; - new (ptr) ::google::protobuf::Mixin(); - ::google::protobuf::internal::OnShutdownDestroyMessage(ptr); - } - ::google::protobuf::Mixin::InitAsDefaultInstance(); -} - -void InitDefaultsMixin() { - static GOOGLE_PROTOBUF_DECLARE_ONCE(once); - ::google::protobuf::GoogleOnceInit(&once, &InitDefaultsMixinImpl); -} - -::google::protobuf::Metadata file_level_metadata[3]; - -const ::google::protobuf::uint32 TableStruct::offsets[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { - ~0u, // no _has_bits_ - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Api, _internal_metadata_), - ~0u, // no _extensions_ - ~0u, // no _oneof_case_ - ~0u, // no _weak_field_map_ - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Api, name_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Api, methods_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Api, options_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Api, version_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Api, source_context_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Api, mixins_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Api, syntax_), - ~0u, // no _has_bits_ - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Method, _internal_metadata_), - ~0u, // no _extensions_ - ~0u, // no _oneof_case_ - ~0u, // no _weak_field_map_ - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Method, name_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Method, request_type_url_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Method, request_streaming_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Method, response_type_url_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Method, response_streaming_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Method, options_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Method, syntax_), - ~0u, // no _has_bits_ - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Mixin, _internal_metadata_), - ~0u, // no _extensions_ - ~0u, // no _oneof_case_ - ~0u, // no _weak_field_map_ - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Mixin, name_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Mixin, root_), -}; -static const ::google::protobuf::internal::MigrationSchema schemas[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { - { 0, -1, sizeof(::google::protobuf::Api)}, - { 12, -1, sizeof(::google::protobuf::Method)}, - { 24, -1, sizeof(::google::protobuf::Mixin)}, -}; - -static ::google::protobuf::Message const * const file_default_instances[] = { - reinterpret_cast(&::google::protobuf::_Api_default_instance_), - reinterpret_cast(&::google::protobuf::_Method_default_instance_), - reinterpret_cast(&::google::protobuf::_Mixin_default_instance_), -}; - -void protobuf_AssignDescriptors() { - AddDescriptors(); - ::google::protobuf::MessageFactory* factory = NULL; - AssignDescriptors( - "google/protobuf/api.proto", schemas, file_default_instances, TableStruct::offsets, factory, - file_level_metadata, NULL, NULL); -} - -void protobuf_AssignDescriptorsOnce() { - static GOOGLE_PROTOBUF_DECLARE_ONCE(once); - ::google::protobuf::GoogleOnceInit(&once, &protobuf_AssignDescriptors); -} - -void protobuf_RegisterTypes(const ::std::string&) GOOGLE_PROTOBUF_ATTRIBUTE_COLD; -void protobuf_RegisterTypes(const ::std::string&) { - protobuf_AssignDescriptorsOnce(); - ::google::protobuf::internal::RegisterAllTypes(file_level_metadata, 3); -} - -void AddDescriptorsImpl() { - InitDefaults(); - static const char descriptor[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { - "\n\031google/protobuf/api.proto\022\017google.prot" - "obuf\032$google/protobuf/source_context.pro" - "to\032\032google/protobuf/type.proto\"\201\002\n\003Api\022\014" - "\n\004name\030\001 \001(\t\022(\n\007methods\030\002 \003(\0132\027.google.p" - "rotobuf.Method\022(\n\007options\030\003 \003(\0132\027.google" - ".protobuf.Option\022\017\n\007version\030\004 \001(\t\0226\n\016sou" - "rce_context\030\005 \001(\0132\036.google.protobuf.Sour" - "ceContext\022&\n\006mixins\030\006 \003(\0132\026.google.proto" - "buf.Mixin\022\'\n\006syntax\030\007 \001(\0162\027.google.proto" - "buf.Syntax\"\325\001\n\006Method\022\014\n\004name\030\001 \001(\t\022\030\n\020r" - "equest_type_url\030\002 \001(\t\022\031\n\021request_streami" - "ng\030\003 \001(\010\022\031\n\021response_type_url\030\004 \001(\t\022\032\n\022r" - "esponse_streaming\030\005 \001(\010\022(\n\007options\030\006 \003(\013" - "2\027.google.protobuf.Option\022\'\n\006syntax\030\007 \001(" - "\0162\027.google.protobuf.Syntax\"#\n\005Mixin\022\014\n\004n" - "ame\030\001 \001(\t\022\014\n\004root\030\002 \001(\tBu\n\023com.google.pr" - "otobufB\010ApiProtoP\001Z+google.golang.org/ge" - "nproto/protobuf/api;api\242\002\003GPB\252\002\036Google.P" - "rotobuf.WellKnownTypesb\006proto3" - }; - ::google::protobuf::DescriptorPool::InternalAddGeneratedFile( - descriptor, 750); - ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile( - "google/protobuf/api.proto", &protobuf_RegisterTypes); - ::protobuf_google_2fprotobuf_2fsource_5fcontext_2eproto::AddDescriptors(); - ::protobuf_google_2fprotobuf_2ftype_2eproto::AddDescriptors(); -} - -void AddDescriptors() { - static GOOGLE_PROTOBUF_DECLARE_ONCE(once); - ::google::protobuf::GoogleOnceInit(&once, &AddDescriptorsImpl); -} -// Force AddDescriptors() to be called at dynamic initialization time. -struct StaticDescriptorInitializer { - StaticDescriptorInitializer() { - AddDescriptors(); - } -} static_descriptor_initializer; -} // namespace protobuf_google_2fprotobuf_2fapi_2eproto -namespace google { -namespace protobuf { - -// =================================================================== - -void Api::InitAsDefaultInstance() { - ::google::protobuf::_Api_default_instance_._instance.get_mutable()->source_context_ = const_cast< ::google::protobuf::SourceContext*>( - ::google::protobuf::SourceContext::internal_default_instance()); -} -void Api::clear_options() { - options_.Clear(); -} -void Api::clear_source_context() { - if (GetArenaNoVirtual() == NULL && source_context_ != NULL) { - delete source_context_; - } - source_context_ = NULL; -} -#if !defined(_MSC_VER) || _MSC_VER >= 1900 -const int Api::kNameFieldNumber; -const int Api::kMethodsFieldNumber; -const int Api::kOptionsFieldNumber; -const int Api::kVersionFieldNumber; -const int Api::kSourceContextFieldNumber; -const int Api::kMixinsFieldNumber; -const int Api::kSyntaxFieldNumber; -#endif // !defined(_MSC_VER) || _MSC_VER >= 1900 - -Api::Api() - : ::google::protobuf::Message(), _internal_metadata_(NULL) { - if (GOOGLE_PREDICT_TRUE(this != internal_default_instance())) { - ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsApi(); - } - SharedCtor(); - // @@protoc_insertion_point(constructor:google.protobuf.Api) -} -Api::Api(const Api& from) - : ::google::protobuf::Message(), - _internal_metadata_(NULL), - methods_(from.methods_), - options_(from.options_), - mixins_(from.mixins_), - _cached_size_(0) { - _internal_metadata_.MergeFrom(from._internal_metadata_); - name_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.name().size() > 0) { - name_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.name_); - } - version_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.version().size() > 0) { - version_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.version_); - } - if (from.has_source_context()) { - source_context_ = new ::google::protobuf::SourceContext(*from.source_context_); - } else { - source_context_ = NULL; - } - syntax_ = from.syntax_; - // @@protoc_insertion_point(copy_constructor:google.protobuf.Api) -} - -void Api::SharedCtor() { - name_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - version_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - ::memset(&source_context_, 0, static_cast( - reinterpret_cast(&syntax_) - - reinterpret_cast(&source_context_)) + sizeof(syntax_)); - _cached_size_ = 0; -} - -Api::~Api() { - // @@protoc_insertion_point(destructor:google.protobuf.Api) - SharedDtor(); -} - -void Api::SharedDtor() { - name_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - version_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (this != internal_default_instance()) delete source_context_; -} - -void Api::SetCachedSize(int size) const { - GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); - _cached_size_ = size; - GOOGLE_SAFE_CONCURRENT_WRITES_END(); -} -const ::google::protobuf::Descriptor* Api::descriptor() { - ::protobuf_google_2fprotobuf_2fapi_2eproto::protobuf_AssignDescriptorsOnce(); - return ::protobuf_google_2fprotobuf_2fapi_2eproto::file_level_metadata[kIndexInFileMessages].descriptor; -} - -const Api& Api::default_instance() { - ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsApi(); - return *internal_default_instance(); -} - -Api* Api::New(::google::protobuf::Arena* arena) const { - Api* n = new Api; - if (arena != NULL) { - arena->Own(n); - } - return n; -} - -void Api::Clear() { -// @@protoc_insertion_point(message_clear_start:google.protobuf.Api) - ::google::protobuf::uint32 cached_has_bits = 0; - // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; - - methods_.Clear(); - options_.Clear(); - mixins_.Clear(); - name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - version_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (GetArenaNoVirtual() == NULL && source_context_ != NULL) { - delete source_context_; - } - source_context_ = NULL; - syntax_ = 0; - _internal_metadata_.Clear(); -} - -bool Api::MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) { -#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure - ::google::protobuf::uint32 tag; - // @@protoc_insertion_point(parse_start:google.protobuf.Api) - for (;;) { - ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); - tag = p.first; - if (!p.second) goto handle_unusual; - switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { - // string name = 1; - case 1: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(10u /* 10 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_name())); - DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->name().data(), static_cast(this->name().length()), - ::google::protobuf::internal::WireFormatLite::PARSE, - "google.protobuf.Api.name")); - } else { - goto handle_unusual; - } - break; - } - - // repeated .google.protobuf.Method methods = 2; - case 2: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(18u /* 18 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(input, add_methods())); - } else { - goto handle_unusual; - } - break; - } - - // repeated .google.protobuf.Option options = 3; - case 3: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(26u /* 26 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(input, add_options())); - } else { - goto handle_unusual; - } - break; - } - - // string version = 4; - case 4: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(34u /* 34 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_version())); - DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->version().data(), static_cast(this->version().length()), - ::google::protobuf::internal::WireFormatLite::PARSE, - "google.protobuf.Api.version")); - } else { - goto handle_unusual; - } - break; - } - - // .google.protobuf.SourceContext source_context = 5; - case 5: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(42u /* 42 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadMessage( - input, mutable_source_context())); - } else { - goto handle_unusual; - } - break; - } - - // repeated .google.protobuf.Mixin mixins = 6; - case 6: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(50u /* 50 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(input, add_mixins())); - } else { - goto handle_unusual; - } - break; - } - - // .google.protobuf.Syntax syntax = 7; - case 7: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(56u /* 56 & 0xFF */)) { - int value; - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( - input, &value))); - set_syntax(static_cast< ::google::protobuf::Syntax >(value)); - } else { - goto handle_unusual; - } - break; - } - - default: { - handle_unusual: - if (tag == 0) { - goto success; - } - DO_(::google::protobuf::internal::WireFormat::SkipField( - input, tag, _internal_metadata_.mutable_unknown_fields())); - break; - } - } - } -success: - // @@protoc_insertion_point(parse_success:google.protobuf.Api) - return true; -failure: - // @@protoc_insertion_point(parse_failure:google.protobuf.Api) - return false; -#undef DO_ -} - -void Api::SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const { - // @@protoc_insertion_point(serialize_start:google.protobuf.Api) - ::google::protobuf::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - // string name = 1; - if (this->name().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->name().data(), static_cast(this->name().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Api.name"); - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 1, this->name(), output); - } - - // repeated .google.protobuf.Method methods = 2; - for (unsigned int i = 0, - n = static_cast(this->methods_size()); i < n; i++) { - ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray( - 2, this->methods(static_cast(i)), output); - } - - // repeated .google.protobuf.Option options = 3; - for (unsigned int i = 0, - n = static_cast(this->options_size()); i < n; i++) { - ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray( - 3, this->options(static_cast(i)), output); - } - - // string version = 4; - if (this->version().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->version().data(), static_cast(this->version().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Api.version"); - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 4, this->version(), output); - } - - // .google.protobuf.SourceContext source_context = 5; - if (this->has_source_context()) { - ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray( - 5, *this->source_context_, output); - } - - // repeated .google.protobuf.Mixin mixins = 6; - for (unsigned int i = 0, - n = static_cast(this->mixins_size()); i < n; i++) { - ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray( - 6, this->mixins(static_cast(i)), output); - } - - // .google.protobuf.Syntax syntax = 7; - if (this->syntax() != 0) { - ::google::protobuf::internal::WireFormatLite::WriteEnum( - 7, this->syntax(), output); - } - - if ((_internal_metadata_.have_unknown_fields() && ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) { - ::google::protobuf::internal::WireFormat::SerializeUnknownFields( - (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance()), output); - } - // @@protoc_insertion_point(serialize_end:google.protobuf.Api) -} - -::google::protobuf::uint8* Api::InternalSerializeWithCachedSizesToArray( - bool deterministic, ::google::protobuf::uint8* target) const { - (void)deterministic; // Unused - // @@protoc_insertion_point(serialize_to_array_start:google.protobuf.Api) - ::google::protobuf::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - // string name = 1; - if (this->name().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->name().data(), static_cast(this->name().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Api.name"); - target = - ::google::protobuf::internal::WireFormatLite::WriteStringToArray( - 1, this->name(), target); - } - - // repeated .google.protobuf.Method methods = 2; - for (unsigned int i = 0, - n = static_cast(this->methods_size()); i < n; i++) { - target = ::google::protobuf::internal::WireFormatLite:: - InternalWriteMessageToArray( - 2, this->methods(static_cast(i)), deterministic, target); - } - - // repeated .google.protobuf.Option options = 3; - for (unsigned int i = 0, - n = static_cast(this->options_size()); i < n; i++) { - target = ::google::protobuf::internal::WireFormatLite:: - InternalWriteMessageToArray( - 3, this->options(static_cast(i)), deterministic, target); - } - - // string version = 4; - if (this->version().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->version().data(), static_cast(this->version().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Api.version"); - target = - ::google::protobuf::internal::WireFormatLite::WriteStringToArray( - 4, this->version(), target); - } - - // .google.protobuf.SourceContext source_context = 5; - if (this->has_source_context()) { - target = ::google::protobuf::internal::WireFormatLite:: - InternalWriteMessageToArray( - 5, *this->source_context_, deterministic, target); - } - - // repeated .google.protobuf.Mixin mixins = 6; - for (unsigned int i = 0, - n = static_cast(this->mixins_size()); i < n; i++) { - target = ::google::protobuf::internal::WireFormatLite:: - InternalWriteMessageToArray( - 6, this->mixins(static_cast(i)), deterministic, target); - } - - // .google.protobuf.Syntax syntax = 7; - if (this->syntax() != 0) { - target = ::google::protobuf::internal::WireFormatLite::WriteEnumToArray( - 7, this->syntax(), target); - } - - if ((_internal_metadata_.have_unknown_fields() && ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) { - target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray( - (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance()), target); - } - // @@protoc_insertion_point(serialize_to_array_end:google.protobuf.Api) - return target; -} - -size_t Api::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.Api) - size_t total_size = 0; - - if ((_internal_metadata_.have_unknown_fields() && ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) { - total_size += - ::google::protobuf::internal::WireFormat::ComputeUnknownFieldsSize( - (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance())); - } - // repeated .google.protobuf.Method methods = 2; - { - unsigned int count = static_cast(this->methods_size()); - total_size += 1UL * count; - for (unsigned int i = 0; i < count; i++) { - total_size += - ::google::protobuf::internal::WireFormatLite::MessageSize( - this->methods(static_cast(i))); - } - } - - // repeated .google.protobuf.Option options = 3; - { - unsigned int count = static_cast(this->options_size()); - total_size += 1UL * count; - for (unsigned int i = 0; i < count; i++) { - total_size += - ::google::protobuf::internal::WireFormatLite::MessageSize( - this->options(static_cast(i))); - } - } - - // repeated .google.protobuf.Mixin mixins = 6; - { - unsigned int count = static_cast(this->mixins_size()); - total_size += 1UL * count; - for (unsigned int i = 0; i < count; i++) { - total_size += - ::google::protobuf::internal::WireFormatLite::MessageSize( - this->mixins(static_cast(i))); - } - } - - // string name = 1; - if (this->name().size() > 0) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->name()); - } - - // string version = 4; - if (this->version().size() > 0) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->version()); - } - - // .google.protobuf.SourceContext source_context = 5; - if (this->has_source_context()) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::MessageSize( - *this->source_context_); - } - - // .google.protobuf.Syntax syntax = 7; - if (this->syntax() != 0) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::EnumSize(this->syntax()); - } - - int cached_size = ::google::protobuf::internal::ToCachedSize(total_size); - GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); - _cached_size_ = cached_size; - GOOGLE_SAFE_CONCURRENT_WRITES_END(); - return total_size; -} - -void Api::MergeFrom(const ::google::protobuf::Message& from) { -// @@protoc_insertion_point(generalized_merge_from_start:google.protobuf.Api) - GOOGLE_DCHECK_NE(&from, this); - const Api* source = - ::google::protobuf::internal::DynamicCastToGenerated( - &from); - if (source == NULL) { - // @@protoc_insertion_point(generalized_merge_from_cast_fail:google.protobuf.Api) - ::google::protobuf::internal::ReflectionOps::Merge(from, this); - } else { - // @@protoc_insertion_point(generalized_merge_from_cast_success:google.protobuf.Api) - MergeFrom(*source); - } -} - -void Api::MergeFrom(const Api& from) { -// @@protoc_insertion_point(class_specific_merge_from_start:google.protobuf.Api) - GOOGLE_DCHECK_NE(&from, this); - _internal_metadata_.MergeFrom(from._internal_metadata_); - ::google::protobuf::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - methods_.MergeFrom(from.methods_); - options_.MergeFrom(from.options_); - mixins_.MergeFrom(from.mixins_); - if (from.name().size() > 0) { - - name_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.name_); - } - if (from.version().size() > 0) { - - version_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.version_); - } - if (from.has_source_context()) { - mutable_source_context()->::google::protobuf::SourceContext::MergeFrom(from.source_context()); - } - if (from.syntax() != 0) { - set_syntax(from.syntax()); - } -} - -void Api::CopyFrom(const ::google::protobuf::Message& from) { -// @@protoc_insertion_point(generalized_copy_from_start:google.protobuf.Api) - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -void Api::CopyFrom(const Api& from) { -// @@protoc_insertion_point(class_specific_copy_from_start:google.protobuf.Api) - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -bool Api::IsInitialized() const { - return true; -} - -void Api::Swap(Api* other) { - if (other == this) return; - InternalSwap(other); -} -void Api::InternalSwap(Api* other) { - using std::swap; - methods_.InternalSwap(&other->methods_); - options_.InternalSwap(&other->options_); - mixins_.InternalSwap(&other->mixins_); - name_.Swap(&other->name_); - version_.Swap(&other->version_); - swap(source_context_, other->source_context_); - swap(syntax_, other->syntax_); - _internal_metadata_.Swap(&other->_internal_metadata_); - swap(_cached_size_, other->_cached_size_); -} - -::google::protobuf::Metadata Api::GetMetadata() const { - protobuf_google_2fprotobuf_2fapi_2eproto::protobuf_AssignDescriptorsOnce(); - return ::protobuf_google_2fprotobuf_2fapi_2eproto::file_level_metadata[kIndexInFileMessages]; -} - - -// =================================================================== - -void Method::InitAsDefaultInstance() { -} -void Method::clear_options() { - options_.Clear(); -} -#if !defined(_MSC_VER) || _MSC_VER >= 1900 -const int Method::kNameFieldNumber; -const int Method::kRequestTypeUrlFieldNumber; -const int Method::kRequestStreamingFieldNumber; -const int Method::kResponseTypeUrlFieldNumber; -const int Method::kResponseStreamingFieldNumber; -const int Method::kOptionsFieldNumber; -const int Method::kSyntaxFieldNumber; -#endif // !defined(_MSC_VER) || _MSC_VER >= 1900 - -Method::Method() - : ::google::protobuf::Message(), _internal_metadata_(NULL) { - if (GOOGLE_PREDICT_TRUE(this != internal_default_instance())) { - ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsMethod(); - } - SharedCtor(); - // @@protoc_insertion_point(constructor:google.protobuf.Method) -} -Method::Method(const Method& from) - : ::google::protobuf::Message(), - _internal_metadata_(NULL), - options_(from.options_), - _cached_size_(0) { - _internal_metadata_.MergeFrom(from._internal_metadata_); - name_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.name().size() > 0) { - name_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.name_); - } - request_type_url_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.request_type_url().size() > 0) { - request_type_url_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.request_type_url_); - } - response_type_url_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.response_type_url().size() > 0) { - response_type_url_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.response_type_url_); - } - ::memcpy(&request_streaming_, &from.request_streaming_, - static_cast(reinterpret_cast(&syntax_) - - reinterpret_cast(&request_streaming_)) + sizeof(syntax_)); - // @@protoc_insertion_point(copy_constructor:google.protobuf.Method) -} - -void Method::SharedCtor() { - name_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - request_type_url_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - response_type_url_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - ::memset(&request_streaming_, 0, static_cast( - reinterpret_cast(&syntax_) - - reinterpret_cast(&request_streaming_)) + sizeof(syntax_)); - _cached_size_ = 0; -} - -Method::~Method() { - // @@protoc_insertion_point(destructor:google.protobuf.Method) - SharedDtor(); -} - -void Method::SharedDtor() { - name_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - request_type_url_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - response_type_url_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} - -void Method::SetCachedSize(int size) const { - GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); - _cached_size_ = size; - GOOGLE_SAFE_CONCURRENT_WRITES_END(); -} -const ::google::protobuf::Descriptor* Method::descriptor() { - ::protobuf_google_2fprotobuf_2fapi_2eproto::protobuf_AssignDescriptorsOnce(); - return ::protobuf_google_2fprotobuf_2fapi_2eproto::file_level_metadata[kIndexInFileMessages].descriptor; -} - -const Method& Method::default_instance() { - ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsMethod(); - return *internal_default_instance(); -} - -Method* Method::New(::google::protobuf::Arena* arena) const { - Method* n = new Method; - if (arena != NULL) { - arena->Own(n); - } - return n; -} - -void Method::Clear() { -// @@protoc_insertion_point(message_clear_start:google.protobuf.Method) - ::google::protobuf::uint32 cached_has_bits = 0; - // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; - - options_.Clear(); - name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - request_type_url_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - response_type_url_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - ::memset(&request_streaming_, 0, static_cast( - reinterpret_cast(&syntax_) - - reinterpret_cast(&request_streaming_)) + sizeof(syntax_)); - _internal_metadata_.Clear(); -} - -bool Method::MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) { -#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure - ::google::protobuf::uint32 tag; - // @@protoc_insertion_point(parse_start:google.protobuf.Method) - for (;;) { - ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); - tag = p.first; - if (!p.second) goto handle_unusual; - switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { - // string name = 1; - case 1: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(10u /* 10 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_name())); - DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->name().data(), static_cast(this->name().length()), - ::google::protobuf::internal::WireFormatLite::PARSE, - "google.protobuf.Method.name")); - } else { - goto handle_unusual; - } - break; - } - - // string request_type_url = 2; - case 2: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(18u /* 18 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_request_type_url())); - DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->request_type_url().data(), static_cast(this->request_type_url().length()), - ::google::protobuf::internal::WireFormatLite::PARSE, - "google.protobuf.Method.request_type_url")); - } else { - goto handle_unusual; - } - break; - } - - // bool request_streaming = 3; - case 3: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(24u /* 24 & 0xFF */)) { - - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - bool, ::google::protobuf::internal::WireFormatLite::TYPE_BOOL>( - input, &request_streaming_))); - } else { - goto handle_unusual; - } - break; - } - - // string response_type_url = 4; - case 4: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(34u /* 34 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_response_type_url())); - DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->response_type_url().data(), static_cast(this->response_type_url().length()), - ::google::protobuf::internal::WireFormatLite::PARSE, - "google.protobuf.Method.response_type_url")); - } else { - goto handle_unusual; - } - break; - } - - // bool response_streaming = 5; - case 5: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(40u /* 40 & 0xFF */)) { - - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - bool, ::google::protobuf::internal::WireFormatLite::TYPE_BOOL>( - input, &response_streaming_))); - } else { - goto handle_unusual; - } - break; - } - - // repeated .google.protobuf.Option options = 6; - case 6: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(50u /* 50 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(input, add_options())); - } else { - goto handle_unusual; - } - break; - } - - // .google.protobuf.Syntax syntax = 7; - case 7: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(56u /* 56 & 0xFF */)) { - int value; - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( - input, &value))); - set_syntax(static_cast< ::google::protobuf::Syntax >(value)); - } else { - goto handle_unusual; - } - break; - } - - default: { - handle_unusual: - if (tag == 0) { - goto success; - } - DO_(::google::protobuf::internal::WireFormat::SkipField( - input, tag, _internal_metadata_.mutable_unknown_fields())); - break; - } - } - } -success: - // @@protoc_insertion_point(parse_success:google.protobuf.Method) - return true; -failure: - // @@protoc_insertion_point(parse_failure:google.protobuf.Method) - return false; -#undef DO_ -} - -void Method::SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const { - // @@protoc_insertion_point(serialize_start:google.protobuf.Method) - ::google::protobuf::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - // string name = 1; - if (this->name().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->name().data(), static_cast(this->name().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Method.name"); - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 1, this->name(), output); - } - - // string request_type_url = 2; - if (this->request_type_url().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->request_type_url().data(), static_cast(this->request_type_url().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Method.request_type_url"); - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 2, this->request_type_url(), output); - } - - // bool request_streaming = 3; - if (this->request_streaming() != 0) { - ::google::protobuf::internal::WireFormatLite::WriteBool(3, this->request_streaming(), output); - } - - // string response_type_url = 4; - if (this->response_type_url().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->response_type_url().data(), static_cast(this->response_type_url().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Method.response_type_url"); - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 4, this->response_type_url(), output); - } - - // bool response_streaming = 5; - if (this->response_streaming() != 0) { - ::google::protobuf::internal::WireFormatLite::WriteBool(5, this->response_streaming(), output); - } - - // repeated .google.protobuf.Option options = 6; - for (unsigned int i = 0, - n = static_cast(this->options_size()); i < n; i++) { - ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray( - 6, this->options(static_cast(i)), output); - } - - // .google.protobuf.Syntax syntax = 7; - if (this->syntax() != 0) { - ::google::protobuf::internal::WireFormatLite::WriteEnum( - 7, this->syntax(), output); - } - - if ((_internal_metadata_.have_unknown_fields() && ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) { - ::google::protobuf::internal::WireFormat::SerializeUnknownFields( - (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance()), output); - } - // @@protoc_insertion_point(serialize_end:google.protobuf.Method) -} - -::google::protobuf::uint8* Method::InternalSerializeWithCachedSizesToArray( - bool deterministic, ::google::protobuf::uint8* target) const { - (void)deterministic; // Unused - // @@protoc_insertion_point(serialize_to_array_start:google.protobuf.Method) - ::google::protobuf::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - // string name = 1; - if (this->name().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->name().data(), static_cast(this->name().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Method.name"); - target = - ::google::protobuf::internal::WireFormatLite::WriteStringToArray( - 1, this->name(), target); - } - - // string request_type_url = 2; - if (this->request_type_url().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->request_type_url().data(), static_cast(this->request_type_url().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Method.request_type_url"); - target = - ::google::protobuf::internal::WireFormatLite::WriteStringToArray( - 2, this->request_type_url(), target); - } - - // bool request_streaming = 3; - if (this->request_streaming() != 0) { - target = ::google::protobuf::internal::WireFormatLite::WriteBoolToArray(3, this->request_streaming(), target); - } - - // string response_type_url = 4; - if (this->response_type_url().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->response_type_url().data(), static_cast(this->response_type_url().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Method.response_type_url"); - target = - ::google::protobuf::internal::WireFormatLite::WriteStringToArray( - 4, this->response_type_url(), target); - } - - // bool response_streaming = 5; - if (this->response_streaming() != 0) { - target = ::google::protobuf::internal::WireFormatLite::WriteBoolToArray(5, this->response_streaming(), target); - } - - // repeated .google.protobuf.Option options = 6; - for (unsigned int i = 0, - n = static_cast(this->options_size()); i < n; i++) { - target = ::google::protobuf::internal::WireFormatLite:: - InternalWriteMessageToArray( - 6, this->options(static_cast(i)), deterministic, target); - } - - // .google.protobuf.Syntax syntax = 7; - if (this->syntax() != 0) { - target = ::google::protobuf::internal::WireFormatLite::WriteEnumToArray( - 7, this->syntax(), target); - } - - if ((_internal_metadata_.have_unknown_fields() && ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) { - target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray( - (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance()), target); - } - // @@protoc_insertion_point(serialize_to_array_end:google.protobuf.Method) - return target; -} - -size_t Method::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.Method) - size_t total_size = 0; - - if ((_internal_metadata_.have_unknown_fields() && ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) { - total_size += - ::google::protobuf::internal::WireFormat::ComputeUnknownFieldsSize( - (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance())); - } - // repeated .google.protobuf.Option options = 6; - { - unsigned int count = static_cast(this->options_size()); - total_size += 1UL * count; - for (unsigned int i = 0; i < count; i++) { - total_size += - ::google::protobuf::internal::WireFormatLite::MessageSize( - this->options(static_cast(i))); - } - } - - // string name = 1; - if (this->name().size() > 0) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->name()); - } - - // string request_type_url = 2; - if (this->request_type_url().size() > 0) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->request_type_url()); - } - - // string response_type_url = 4; - if (this->response_type_url().size() > 0) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->response_type_url()); - } - - // bool request_streaming = 3; - if (this->request_streaming() != 0) { - total_size += 1 + 1; - } - - // bool response_streaming = 5; - if (this->response_streaming() != 0) { - total_size += 1 + 1; - } - - // .google.protobuf.Syntax syntax = 7; - if (this->syntax() != 0) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::EnumSize(this->syntax()); - } - - int cached_size = ::google::protobuf::internal::ToCachedSize(total_size); - GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); - _cached_size_ = cached_size; - GOOGLE_SAFE_CONCURRENT_WRITES_END(); - return total_size; -} - -void Method::MergeFrom(const ::google::protobuf::Message& from) { -// @@protoc_insertion_point(generalized_merge_from_start:google.protobuf.Method) - GOOGLE_DCHECK_NE(&from, this); - const Method* source = - ::google::protobuf::internal::DynamicCastToGenerated( - &from); - if (source == NULL) { - // @@protoc_insertion_point(generalized_merge_from_cast_fail:google.protobuf.Method) - ::google::protobuf::internal::ReflectionOps::Merge(from, this); - } else { - // @@protoc_insertion_point(generalized_merge_from_cast_success:google.protobuf.Method) - MergeFrom(*source); - } -} - -void Method::MergeFrom(const Method& from) { -// @@protoc_insertion_point(class_specific_merge_from_start:google.protobuf.Method) - GOOGLE_DCHECK_NE(&from, this); - _internal_metadata_.MergeFrom(from._internal_metadata_); - ::google::protobuf::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - options_.MergeFrom(from.options_); - if (from.name().size() > 0) { - - name_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.name_); - } - if (from.request_type_url().size() > 0) { - - request_type_url_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.request_type_url_); - } - if (from.response_type_url().size() > 0) { - - response_type_url_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.response_type_url_); - } - if (from.request_streaming() != 0) { - set_request_streaming(from.request_streaming()); - } - if (from.response_streaming() != 0) { - set_response_streaming(from.response_streaming()); - } - if (from.syntax() != 0) { - set_syntax(from.syntax()); - } -} - -void Method::CopyFrom(const ::google::protobuf::Message& from) { -// @@protoc_insertion_point(generalized_copy_from_start:google.protobuf.Method) - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -void Method::CopyFrom(const Method& from) { -// @@protoc_insertion_point(class_specific_copy_from_start:google.protobuf.Method) - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -bool Method::IsInitialized() const { - return true; -} - -void Method::Swap(Method* other) { - if (other == this) return; - InternalSwap(other); -} -void Method::InternalSwap(Method* other) { - using std::swap; - options_.InternalSwap(&other->options_); - name_.Swap(&other->name_); - request_type_url_.Swap(&other->request_type_url_); - response_type_url_.Swap(&other->response_type_url_); - swap(request_streaming_, other->request_streaming_); - swap(response_streaming_, other->response_streaming_); - swap(syntax_, other->syntax_); - _internal_metadata_.Swap(&other->_internal_metadata_); - swap(_cached_size_, other->_cached_size_); -} - -::google::protobuf::Metadata Method::GetMetadata() const { - protobuf_google_2fprotobuf_2fapi_2eproto::protobuf_AssignDescriptorsOnce(); - return ::protobuf_google_2fprotobuf_2fapi_2eproto::file_level_metadata[kIndexInFileMessages]; -} - - -// =================================================================== - -void Mixin::InitAsDefaultInstance() { -} -#if !defined(_MSC_VER) || _MSC_VER >= 1900 -const int Mixin::kNameFieldNumber; -const int Mixin::kRootFieldNumber; -#endif // !defined(_MSC_VER) || _MSC_VER >= 1900 - -Mixin::Mixin() - : ::google::protobuf::Message(), _internal_metadata_(NULL) { - if (GOOGLE_PREDICT_TRUE(this != internal_default_instance())) { - ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsMixin(); - } - SharedCtor(); - // @@protoc_insertion_point(constructor:google.protobuf.Mixin) -} -Mixin::Mixin(const Mixin& from) - : ::google::protobuf::Message(), - _internal_metadata_(NULL), - _cached_size_(0) { - _internal_metadata_.MergeFrom(from._internal_metadata_); - name_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.name().size() > 0) { - name_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.name_); - } - root_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.root().size() > 0) { - root_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.root_); - } - // @@protoc_insertion_point(copy_constructor:google.protobuf.Mixin) -} - -void Mixin::SharedCtor() { - name_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - root_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - _cached_size_ = 0; -} - -Mixin::~Mixin() { - // @@protoc_insertion_point(destructor:google.protobuf.Mixin) - SharedDtor(); -} - -void Mixin::SharedDtor() { - name_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - root_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} - -void Mixin::SetCachedSize(int size) const { - GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); - _cached_size_ = size; - GOOGLE_SAFE_CONCURRENT_WRITES_END(); -} -const ::google::protobuf::Descriptor* Mixin::descriptor() { - ::protobuf_google_2fprotobuf_2fapi_2eproto::protobuf_AssignDescriptorsOnce(); - return ::protobuf_google_2fprotobuf_2fapi_2eproto::file_level_metadata[kIndexInFileMessages].descriptor; -} - -const Mixin& Mixin::default_instance() { - ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsMixin(); - return *internal_default_instance(); -} - -Mixin* Mixin::New(::google::protobuf::Arena* arena) const { - Mixin* n = new Mixin; - if (arena != NULL) { - arena->Own(n); - } - return n; -} - -void Mixin::Clear() { -// @@protoc_insertion_point(message_clear_start:google.protobuf.Mixin) - ::google::protobuf::uint32 cached_has_bits = 0; - // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; - - name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - root_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - _internal_metadata_.Clear(); -} - -bool Mixin::MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) { -#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure - ::google::protobuf::uint32 tag; - // @@protoc_insertion_point(parse_start:google.protobuf.Mixin) - for (;;) { - ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); - tag = p.first; - if (!p.second) goto handle_unusual; - switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { - // string name = 1; - case 1: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(10u /* 10 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_name())); - DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->name().data(), static_cast(this->name().length()), - ::google::protobuf::internal::WireFormatLite::PARSE, - "google.protobuf.Mixin.name")); - } else { - goto handle_unusual; - } - break; - } - - // string root = 2; - case 2: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(18u /* 18 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_root())); - DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->root().data(), static_cast(this->root().length()), - ::google::protobuf::internal::WireFormatLite::PARSE, - "google.protobuf.Mixin.root")); - } else { - goto handle_unusual; - } - break; - } - - default: { - handle_unusual: - if (tag == 0) { - goto success; - } - DO_(::google::protobuf::internal::WireFormat::SkipField( - input, tag, _internal_metadata_.mutable_unknown_fields())); - break; - } - } - } -success: - // @@protoc_insertion_point(parse_success:google.protobuf.Mixin) - return true; -failure: - // @@protoc_insertion_point(parse_failure:google.protobuf.Mixin) - return false; -#undef DO_ -} - -void Mixin::SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const { - // @@protoc_insertion_point(serialize_start:google.protobuf.Mixin) - ::google::protobuf::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - // string name = 1; - if (this->name().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->name().data(), static_cast(this->name().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Mixin.name"); - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 1, this->name(), output); - } - - // string root = 2; - if (this->root().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->root().data(), static_cast(this->root().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Mixin.root"); - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 2, this->root(), output); - } - - if ((_internal_metadata_.have_unknown_fields() && ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) { - ::google::protobuf::internal::WireFormat::SerializeUnknownFields( - (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance()), output); - } - // @@protoc_insertion_point(serialize_end:google.protobuf.Mixin) -} - -::google::protobuf::uint8* Mixin::InternalSerializeWithCachedSizesToArray( - bool deterministic, ::google::protobuf::uint8* target) const { - (void)deterministic; // Unused - // @@protoc_insertion_point(serialize_to_array_start:google.protobuf.Mixin) - ::google::protobuf::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - // string name = 1; - if (this->name().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->name().data(), static_cast(this->name().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Mixin.name"); - target = - ::google::protobuf::internal::WireFormatLite::WriteStringToArray( - 1, this->name(), target); - } - - // string root = 2; - if (this->root().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->root().data(), static_cast(this->root().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Mixin.root"); - target = - ::google::protobuf::internal::WireFormatLite::WriteStringToArray( - 2, this->root(), target); - } - - if ((_internal_metadata_.have_unknown_fields() && ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) { - target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray( - (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance()), target); - } - // @@protoc_insertion_point(serialize_to_array_end:google.protobuf.Mixin) - return target; -} - -size_t Mixin::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.Mixin) - size_t total_size = 0; - - if ((_internal_metadata_.have_unknown_fields() && ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) { - total_size += - ::google::protobuf::internal::WireFormat::ComputeUnknownFieldsSize( - (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance())); - } - // string name = 1; - if (this->name().size() > 0) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->name()); - } - - // string root = 2; - if (this->root().size() > 0) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->root()); - } - - int cached_size = ::google::protobuf::internal::ToCachedSize(total_size); - GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); - _cached_size_ = cached_size; - GOOGLE_SAFE_CONCURRENT_WRITES_END(); - return total_size; -} - -void Mixin::MergeFrom(const ::google::protobuf::Message& from) { -// @@protoc_insertion_point(generalized_merge_from_start:google.protobuf.Mixin) - GOOGLE_DCHECK_NE(&from, this); - const Mixin* source = - ::google::protobuf::internal::DynamicCastToGenerated( - &from); - if (source == NULL) { - // @@protoc_insertion_point(generalized_merge_from_cast_fail:google.protobuf.Mixin) - ::google::protobuf::internal::ReflectionOps::Merge(from, this); - } else { - // @@protoc_insertion_point(generalized_merge_from_cast_success:google.protobuf.Mixin) - MergeFrom(*source); - } -} - -void Mixin::MergeFrom(const Mixin& from) { -// @@protoc_insertion_point(class_specific_merge_from_start:google.protobuf.Mixin) - GOOGLE_DCHECK_NE(&from, this); - _internal_metadata_.MergeFrom(from._internal_metadata_); - ::google::protobuf::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - if (from.name().size() > 0) { - - name_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.name_); - } - if (from.root().size() > 0) { - - root_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.root_); - } -} - -void Mixin::CopyFrom(const ::google::protobuf::Message& from) { -// @@protoc_insertion_point(generalized_copy_from_start:google.protobuf.Mixin) - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -void Mixin::CopyFrom(const Mixin& from) { -// @@protoc_insertion_point(class_specific_copy_from_start:google.protobuf.Mixin) - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -bool Mixin::IsInitialized() const { - return true; -} - -void Mixin::Swap(Mixin* other) { - if (other == this) return; - InternalSwap(other); -} -void Mixin::InternalSwap(Mixin* other) { - using std::swap; - name_.Swap(&other->name_); - root_.Swap(&other->root_); - _internal_metadata_.Swap(&other->_internal_metadata_); - swap(_cached_size_, other->_cached_size_); -} - -::google::protobuf::Metadata Mixin::GetMetadata() const { - protobuf_google_2fprotobuf_2fapi_2eproto::protobuf_AssignDescriptorsOnce(); - return ::protobuf_google_2fprotobuf_2fapi_2eproto::file_level_metadata[kIndexInFileMessages]; -} - - -// @@protoc_insertion_point(namespace_scope) -} // namespace protobuf -} // namespace google - -// @@protoc_insertion_point(global_scope) diff --git a/3rdparty/protobuf/src/google/protobuf/api.pb.h b/3rdparty/protobuf/src/google/protobuf/api.pb.h deleted file mode 100644 index 96ff3f159b..0000000000 --- a/3rdparty/protobuf/src/google/protobuf/api.pb.h +++ /dev/null @@ -1,1165 +0,0 @@ -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: google/protobuf/api.proto - -#ifndef PROTOBUF_google_2fprotobuf_2fapi_2eproto__INCLUDED -#define PROTOBUF_google_2fprotobuf_2fapi_2eproto__INCLUDED - -#include - -#include - -#if GOOGLE_PROTOBUF_VERSION < 3005000 -#error This file was generated by a newer version of protoc which is -#error incompatible with your Protocol Buffer headers. Please update -#error your headers. -#endif -#if 3005001 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION -#error This file was generated by an older version of protoc which is -#error incompatible with your Protocol Buffer headers. Please -#error regenerate this file with a newer version of protoc. -#endif - -#include -#include -#include -#include -#include -#include -#include -#include // IWYU pragma: export -#include // IWYU pragma: export -#include -#include -#include -// @@protoc_insertion_point(includes) - -namespace protobuf_google_2fprotobuf_2fapi_2eproto { -// Internal implementation detail -- do not use these members. -struct LIBPROTOBUF_EXPORT TableStruct { - static const ::google::protobuf::internal::ParseTableField entries[]; - static const ::google::protobuf::internal::AuxillaryParseTableField aux[]; - static const ::google::protobuf::internal::ParseTable schema[3]; - static const ::google::protobuf::internal::FieldMetadata field_metadata[]; - static const ::google::protobuf::internal::SerializationTable serialization_table[]; - static const ::google::protobuf::uint32 offsets[]; -}; -void LIBPROTOBUF_EXPORT AddDescriptors(); -void LIBPROTOBUF_EXPORT InitDefaultsApiImpl(); -void LIBPROTOBUF_EXPORT InitDefaultsApi(); -void LIBPROTOBUF_EXPORT InitDefaultsMethodImpl(); -void LIBPROTOBUF_EXPORT InitDefaultsMethod(); -void LIBPROTOBUF_EXPORT InitDefaultsMixinImpl(); -void LIBPROTOBUF_EXPORT InitDefaultsMixin(); -inline void LIBPROTOBUF_EXPORT InitDefaults() { - InitDefaultsApi(); - InitDefaultsMethod(); - InitDefaultsMixin(); -} -} // namespace protobuf_google_2fprotobuf_2fapi_2eproto -namespace google { -namespace protobuf { -class Api; -class ApiDefaultTypeInternal; -LIBPROTOBUF_EXPORT extern ApiDefaultTypeInternal _Api_default_instance_; -class Method; -class MethodDefaultTypeInternal; -LIBPROTOBUF_EXPORT extern MethodDefaultTypeInternal _Method_default_instance_; -class Mixin; -class MixinDefaultTypeInternal; -LIBPROTOBUF_EXPORT extern MixinDefaultTypeInternal _Mixin_default_instance_; -} // namespace protobuf -} // namespace google -namespace google { -namespace protobuf { - -// =================================================================== - -class LIBPROTOBUF_EXPORT Api : public ::google::protobuf::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Api) */ { - public: - Api(); - virtual ~Api(); - - Api(const Api& from); - - inline Api& operator=(const Api& from) { - CopyFrom(from); - return *this; - } - #if LANG_CXX11 - Api(Api&& from) noexcept - : Api() { - *this = ::std::move(from); - } - - inline Api& operator=(Api&& from) noexcept { - if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { - if (this != &from) InternalSwap(&from); - } else { - CopyFrom(from); - } - return *this; - } - #endif - static const ::google::protobuf::Descriptor* descriptor(); - static const Api& default_instance(); - - static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY - static inline const Api* internal_default_instance() { - return reinterpret_cast( - &_Api_default_instance_); - } - static PROTOBUF_CONSTEXPR int const kIndexInFileMessages = - 0; - - void Swap(Api* other); - friend void swap(Api& a, Api& b) { - a.Swap(&b); - } - - // implements Message ---------------------------------------------- - - inline Api* New() const PROTOBUF_FINAL { return New(NULL); } - - Api* New(::google::protobuf::Arena* arena) const PROTOBUF_FINAL; - void CopyFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL; - void MergeFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL; - void CopyFrom(const Api& from); - void MergeFrom(const Api& from); - void Clear() PROTOBUF_FINAL; - bool IsInitialized() const PROTOBUF_FINAL; - - size_t ByteSizeLong() const PROTOBUF_FINAL; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) PROTOBUF_FINAL; - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const PROTOBUF_FINAL; - ::google::protobuf::uint8* InternalSerializeWithCachedSizesToArray( - bool deterministic, ::google::protobuf::uint8* target) const PROTOBUF_FINAL; - int GetCachedSize() const PROTOBUF_FINAL { return _cached_size_; } - private: - void SharedCtor(); - void SharedDtor(); - void SetCachedSize(int size) const PROTOBUF_FINAL; - void InternalSwap(Api* other); - private: - inline ::google::protobuf::Arena* GetArenaNoVirtual() const { - return NULL; - } - inline void* MaybeArenaPtr() const { - return NULL; - } - public: - - ::google::protobuf::Metadata GetMetadata() const PROTOBUF_FINAL; - - // nested types ---------------------------------------------------- - - // accessors ------------------------------------------------------- - - // repeated .google.protobuf.Method methods = 2; - int methods_size() const; - void clear_methods(); - static const int kMethodsFieldNumber = 2; - const ::google::protobuf::Method& methods(int index) const; - ::google::protobuf::Method* mutable_methods(int index); - ::google::protobuf::Method* add_methods(); - ::google::protobuf::RepeatedPtrField< ::google::protobuf::Method >* - mutable_methods(); - const ::google::protobuf::RepeatedPtrField< ::google::protobuf::Method >& - methods() const; - - // repeated .google.protobuf.Option options = 3; - int options_size() const; - void clear_options(); - static const int kOptionsFieldNumber = 3; - const ::google::protobuf::Option& options(int index) const; - ::google::protobuf::Option* mutable_options(int index); - ::google::protobuf::Option* add_options(); - ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option >* - mutable_options(); - const ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option >& - options() const; - - // repeated .google.protobuf.Mixin mixins = 6; - int mixins_size() const; - void clear_mixins(); - static const int kMixinsFieldNumber = 6; - const ::google::protobuf::Mixin& mixins(int index) const; - ::google::protobuf::Mixin* mutable_mixins(int index); - ::google::protobuf::Mixin* add_mixins(); - ::google::protobuf::RepeatedPtrField< ::google::protobuf::Mixin >* - mutable_mixins(); - const ::google::protobuf::RepeatedPtrField< ::google::protobuf::Mixin >& - mixins() const; - - // string name = 1; - void clear_name(); - static const int kNameFieldNumber = 1; - const ::std::string& name() const; - void set_name(const ::std::string& value); - #if LANG_CXX11 - void set_name(::std::string&& value); - #endif - void set_name(const char* value); - void set_name(const char* value, size_t size); - ::std::string* mutable_name(); - ::std::string* release_name(); - void set_allocated_name(::std::string* name); - - // string version = 4; - void clear_version(); - static const int kVersionFieldNumber = 4; - const ::std::string& version() const; - void set_version(const ::std::string& value); - #if LANG_CXX11 - void set_version(::std::string&& value); - #endif - void set_version(const char* value); - void set_version(const char* value, size_t size); - ::std::string* mutable_version(); - ::std::string* release_version(); - void set_allocated_version(::std::string* version); - - // .google.protobuf.SourceContext source_context = 5; - bool has_source_context() const; - void clear_source_context(); - static const int kSourceContextFieldNumber = 5; - const ::google::protobuf::SourceContext& source_context() const; - ::google::protobuf::SourceContext* release_source_context(); - ::google::protobuf::SourceContext* mutable_source_context(); - void set_allocated_source_context(::google::protobuf::SourceContext* source_context); - - // .google.protobuf.Syntax syntax = 7; - void clear_syntax(); - static const int kSyntaxFieldNumber = 7; - ::google::protobuf::Syntax syntax() const; - void set_syntax(::google::protobuf::Syntax value); - - // @@protoc_insertion_point(class_scope:google.protobuf.Api) - private: - - ::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_; - ::google::protobuf::RepeatedPtrField< ::google::protobuf::Method > methods_; - ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option > options_; - ::google::protobuf::RepeatedPtrField< ::google::protobuf::Mixin > mixins_; - ::google::protobuf::internal::ArenaStringPtr name_; - ::google::protobuf::internal::ArenaStringPtr version_; - ::google::protobuf::SourceContext* source_context_; - int syntax_; - mutable int _cached_size_; - friend struct ::protobuf_google_2fprotobuf_2fapi_2eproto::TableStruct; - friend void ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsApiImpl(); -}; -// ------------------------------------------------------------------- - -class LIBPROTOBUF_EXPORT Method : public ::google::protobuf::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Method) */ { - public: - Method(); - virtual ~Method(); - - Method(const Method& from); - - inline Method& operator=(const Method& from) { - CopyFrom(from); - return *this; - } - #if LANG_CXX11 - Method(Method&& from) noexcept - : Method() { - *this = ::std::move(from); - } - - inline Method& operator=(Method&& from) noexcept { - if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { - if (this != &from) InternalSwap(&from); - } else { - CopyFrom(from); - } - return *this; - } - #endif - static const ::google::protobuf::Descriptor* descriptor(); - static const Method& default_instance(); - - static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY - static inline const Method* internal_default_instance() { - return reinterpret_cast( - &_Method_default_instance_); - } - static PROTOBUF_CONSTEXPR int const kIndexInFileMessages = - 1; - - void Swap(Method* other); - friend void swap(Method& a, Method& b) { - a.Swap(&b); - } - - // implements Message ---------------------------------------------- - - inline Method* New() const PROTOBUF_FINAL { return New(NULL); } - - Method* New(::google::protobuf::Arena* arena) const PROTOBUF_FINAL; - void CopyFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL; - void MergeFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL; - void CopyFrom(const Method& from); - void MergeFrom(const Method& from); - void Clear() PROTOBUF_FINAL; - bool IsInitialized() const PROTOBUF_FINAL; - - size_t ByteSizeLong() const PROTOBUF_FINAL; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) PROTOBUF_FINAL; - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const PROTOBUF_FINAL; - ::google::protobuf::uint8* InternalSerializeWithCachedSizesToArray( - bool deterministic, ::google::protobuf::uint8* target) const PROTOBUF_FINAL; - int GetCachedSize() const PROTOBUF_FINAL { return _cached_size_; } - private: - void SharedCtor(); - void SharedDtor(); - void SetCachedSize(int size) const PROTOBUF_FINAL; - void InternalSwap(Method* other); - private: - inline ::google::protobuf::Arena* GetArenaNoVirtual() const { - return NULL; - } - inline void* MaybeArenaPtr() const { - return NULL; - } - public: - - ::google::protobuf::Metadata GetMetadata() const PROTOBUF_FINAL; - - // nested types ---------------------------------------------------- - - // accessors ------------------------------------------------------- - - // repeated .google.protobuf.Option options = 6; - int options_size() const; - void clear_options(); - static const int kOptionsFieldNumber = 6; - const ::google::protobuf::Option& options(int index) const; - ::google::protobuf::Option* mutable_options(int index); - ::google::protobuf::Option* add_options(); - ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option >* - mutable_options(); - const ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option >& - options() const; - - // string name = 1; - void clear_name(); - static const int kNameFieldNumber = 1; - const ::std::string& name() const; - void set_name(const ::std::string& value); - #if LANG_CXX11 - void set_name(::std::string&& value); - #endif - void set_name(const char* value); - void set_name(const char* value, size_t size); - ::std::string* mutable_name(); - ::std::string* release_name(); - void set_allocated_name(::std::string* name); - - // string request_type_url = 2; - void clear_request_type_url(); - static const int kRequestTypeUrlFieldNumber = 2; - const ::std::string& request_type_url() const; - void set_request_type_url(const ::std::string& value); - #if LANG_CXX11 - void set_request_type_url(::std::string&& value); - #endif - void set_request_type_url(const char* value); - void set_request_type_url(const char* value, size_t size); - ::std::string* mutable_request_type_url(); - ::std::string* release_request_type_url(); - void set_allocated_request_type_url(::std::string* request_type_url); - - // string response_type_url = 4; - void clear_response_type_url(); - static const int kResponseTypeUrlFieldNumber = 4; - const ::std::string& response_type_url() const; - void set_response_type_url(const ::std::string& value); - #if LANG_CXX11 - void set_response_type_url(::std::string&& value); - #endif - void set_response_type_url(const char* value); - void set_response_type_url(const char* value, size_t size); - ::std::string* mutable_response_type_url(); - ::std::string* release_response_type_url(); - void set_allocated_response_type_url(::std::string* response_type_url); - - // bool request_streaming = 3; - void clear_request_streaming(); - static const int kRequestStreamingFieldNumber = 3; - bool request_streaming() const; - void set_request_streaming(bool value); - - // bool response_streaming = 5; - void clear_response_streaming(); - static const int kResponseStreamingFieldNumber = 5; - bool response_streaming() const; - void set_response_streaming(bool value); - - // .google.protobuf.Syntax syntax = 7; - void clear_syntax(); - static const int kSyntaxFieldNumber = 7; - ::google::protobuf::Syntax syntax() const; - void set_syntax(::google::protobuf::Syntax value); - - // @@protoc_insertion_point(class_scope:google.protobuf.Method) - private: - - ::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_; - ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option > options_; - ::google::protobuf::internal::ArenaStringPtr name_; - ::google::protobuf::internal::ArenaStringPtr request_type_url_; - ::google::protobuf::internal::ArenaStringPtr response_type_url_; - bool request_streaming_; - bool response_streaming_; - int syntax_; - mutable int _cached_size_; - friend struct ::protobuf_google_2fprotobuf_2fapi_2eproto::TableStruct; - friend void ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsMethodImpl(); -}; -// ------------------------------------------------------------------- - -class LIBPROTOBUF_EXPORT Mixin : public ::google::protobuf::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Mixin) */ { - public: - Mixin(); - virtual ~Mixin(); - - Mixin(const Mixin& from); - - inline Mixin& operator=(const Mixin& from) { - CopyFrom(from); - return *this; - } - #if LANG_CXX11 - Mixin(Mixin&& from) noexcept - : Mixin() { - *this = ::std::move(from); - } - - inline Mixin& operator=(Mixin&& from) noexcept { - if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { - if (this != &from) InternalSwap(&from); - } else { - CopyFrom(from); - } - return *this; - } - #endif - static const ::google::protobuf::Descriptor* descriptor(); - static const Mixin& default_instance(); - - static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY - static inline const Mixin* internal_default_instance() { - return reinterpret_cast( - &_Mixin_default_instance_); - } - static PROTOBUF_CONSTEXPR int const kIndexInFileMessages = - 2; - - void Swap(Mixin* other); - friend void swap(Mixin& a, Mixin& b) { - a.Swap(&b); - } - - // implements Message ---------------------------------------------- - - inline Mixin* New() const PROTOBUF_FINAL { return New(NULL); } - - Mixin* New(::google::protobuf::Arena* arena) const PROTOBUF_FINAL; - void CopyFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL; - void MergeFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL; - void CopyFrom(const Mixin& from); - void MergeFrom(const Mixin& from); - void Clear() PROTOBUF_FINAL; - bool IsInitialized() const PROTOBUF_FINAL; - - size_t ByteSizeLong() const PROTOBUF_FINAL; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) PROTOBUF_FINAL; - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const PROTOBUF_FINAL; - ::google::protobuf::uint8* InternalSerializeWithCachedSizesToArray( - bool deterministic, ::google::protobuf::uint8* target) const PROTOBUF_FINAL; - int GetCachedSize() const PROTOBUF_FINAL { return _cached_size_; } - private: - void SharedCtor(); - void SharedDtor(); - void SetCachedSize(int size) const PROTOBUF_FINAL; - void InternalSwap(Mixin* other); - private: - inline ::google::protobuf::Arena* GetArenaNoVirtual() const { - return NULL; - } - inline void* MaybeArenaPtr() const { - return NULL; - } - public: - - ::google::protobuf::Metadata GetMetadata() const PROTOBUF_FINAL; - - // nested types ---------------------------------------------------- - - // accessors ------------------------------------------------------- - - // string name = 1; - void clear_name(); - static const int kNameFieldNumber = 1; - const ::std::string& name() const; - void set_name(const ::std::string& value); - #if LANG_CXX11 - void set_name(::std::string&& value); - #endif - void set_name(const char* value); - void set_name(const char* value, size_t size); - ::std::string* mutable_name(); - ::std::string* release_name(); - void set_allocated_name(::std::string* name); - - // string root = 2; - void clear_root(); - static const int kRootFieldNumber = 2; - const ::std::string& root() const; - void set_root(const ::std::string& value); - #if LANG_CXX11 - void set_root(::std::string&& value); - #endif - void set_root(const char* value); - void set_root(const char* value, size_t size); - ::std::string* mutable_root(); - ::std::string* release_root(); - void set_allocated_root(::std::string* root); - - // @@protoc_insertion_point(class_scope:google.protobuf.Mixin) - private: - - ::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_; - ::google::protobuf::internal::ArenaStringPtr name_; - ::google::protobuf::internal::ArenaStringPtr root_; - mutable int _cached_size_; - friend struct ::protobuf_google_2fprotobuf_2fapi_2eproto::TableStruct; - friend void ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsMixinImpl(); -}; -// =================================================================== - - -// =================================================================== - -#ifdef __GNUC__ - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wstrict-aliasing" -#endif // __GNUC__ -// Api - -// string name = 1; -inline void Api::clear_name() { - name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline const ::std::string& Api::name() const { - // @@protoc_insertion_point(field_get:google.protobuf.Api.name) - return name_.GetNoArena(); -} -inline void Api::set_name(const ::std::string& value) { - - name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); - // @@protoc_insertion_point(field_set:google.protobuf.Api.name) -} -#if LANG_CXX11 -inline void Api::set_name(::std::string&& value) { - - name_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); - // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Api.name) -} -#endif -inline void Api::set_name(const char* value) { - GOOGLE_DCHECK(value != NULL); - - name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); - // @@protoc_insertion_point(field_set_char:google.protobuf.Api.name) -} -inline void Api::set_name(const char* value, size_t size) { - - name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); - // @@protoc_insertion_point(field_set_pointer:google.protobuf.Api.name) -} -inline ::std::string* Api::mutable_name() { - - // @@protoc_insertion_point(field_mutable:google.protobuf.Api.name) - return name_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline ::std::string* Api::release_name() { - // @@protoc_insertion_point(field_release:google.protobuf.Api.name) - - return name_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline void Api::set_allocated_name(::std::string* name) { - if (name != NULL) { - - } else { - - } - name_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), name); - // @@protoc_insertion_point(field_set_allocated:google.protobuf.Api.name) -} - -// repeated .google.protobuf.Method methods = 2; -inline int Api::methods_size() const { - return methods_.size(); -} -inline void Api::clear_methods() { - methods_.Clear(); -} -inline const ::google::protobuf::Method& Api::methods(int index) const { - // @@protoc_insertion_point(field_get:google.protobuf.Api.methods) - return methods_.Get(index); -} -inline ::google::protobuf::Method* Api::mutable_methods(int index) { - // @@protoc_insertion_point(field_mutable:google.protobuf.Api.methods) - return methods_.Mutable(index); -} -inline ::google::protobuf::Method* Api::add_methods() { - // @@protoc_insertion_point(field_add:google.protobuf.Api.methods) - return methods_.Add(); -} -inline ::google::protobuf::RepeatedPtrField< ::google::protobuf::Method >* -Api::mutable_methods() { - // @@protoc_insertion_point(field_mutable_list:google.protobuf.Api.methods) - return &methods_; -} -inline const ::google::protobuf::RepeatedPtrField< ::google::protobuf::Method >& -Api::methods() const { - // @@protoc_insertion_point(field_list:google.protobuf.Api.methods) - return methods_; -} - -// repeated .google.protobuf.Option options = 3; -inline int Api::options_size() const { - return options_.size(); -} -inline const ::google::protobuf::Option& Api::options(int index) const { - // @@protoc_insertion_point(field_get:google.protobuf.Api.options) - return options_.Get(index); -} -inline ::google::protobuf::Option* Api::mutable_options(int index) { - // @@protoc_insertion_point(field_mutable:google.protobuf.Api.options) - return options_.Mutable(index); -} -inline ::google::protobuf::Option* Api::add_options() { - // @@protoc_insertion_point(field_add:google.protobuf.Api.options) - return options_.Add(); -} -inline ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option >* -Api::mutable_options() { - // @@protoc_insertion_point(field_mutable_list:google.protobuf.Api.options) - return &options_; -} -inline const ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option >& -Api::options() const { - // @@protoc_insertion_point(field_list:google.protobuf.Api.options) - return options_; -} - -// string version = 4; -inline void Api::clear_version() { - version_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline const ::std::string& Api::version() const { - // @@protoc_insertion_point(field_get:google.protobuf.Api.version) - return version_.GetNoArena(); -} -inline void Api::set_version(const ::std::string& value) { - - version_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); - // @@protoc_insertion_point(field_set:google.protobuf.Api.version) -} -#if LANG_CXX11 -inline void Api::set_version(::std::string&& value) { - - version_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); - // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Api.version) -} -#endif -inline void Api::set_version(const char* value) { - GOOGLE_DCHECK(value != NULL); - - version_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); - // @@protoc_insertion_point(field_set_char:google.protobuf.Api.version) -} -inline void Api::set_version(const char* value, size_t size) { - - version_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); - // @@protoc_insertion_point(field_set_pointer:google.protobuf.Api.version) -} -inline ::std::string* Api::mutable_version() { - - // @@protoc_insertion_point(field_mutable:google.protobuf.Api.version) - return version_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline ::std::string* Api::release_version() { - // @@protoc_insertion_point(field_release:google.protobuf.Api.version) - - return version_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline void Api::set_allocated_version(::std::string* version) { - if (version != NULL) { - - } else { - - } - version_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), version); - // @@protoc_insertion_point(field_set_allocated:google.protobuf.Api.version) -} - -// .google.protobuf.SourceContext source_context = 5; -inline bool Api::has_source_context() const { - return this != internal_default_instance() && source_context_ != NULL; -} -inline const ::google::protobuf::SourceContext& Api::source_context() const { - const ::google::protobuf::SourceContext* p = source_context_; - // @@protoc_insertion_point(field_get:google.protobuf.Api.source_context) - return p != NULL ? *p : *reinterpret_cast( - &::google::protobuf::_SourceContext_default_instance_); -} -inline ::google::protobuf::SourceContext* Api::release_source_context() { - // @@protoc_insertion_point(field_release:google.protobuf.Api.source_context) - - ::google::protobuf::SourceContext* temp = source_context_; - source_context_ = NULL; - return temp; -} -inline ::google::protobuf::SourceContext* Api::mutable_source_context() { - - if (source_context_ == NULL) { - source_context_ = new ::google::protobuf::SourceContext; - } - // @@protoc_insertion_point(field_mutable:google.protobuf.Api.source_context) - return source_context_; -} -inline void Api::set_allocated_source_context(::google::protobuf::SourceContext* source_context) { - ::google::protobuf::Arena* message_arena = GetArenaNoVirtual(); - if (message_arena == NULL) { - delete reinterpret_cast< ::google::protobuf::MessageLite*>(source_context_); - } - if (source_context) { - ::google::protobuf::Arena* submessage_arena = NULL; - if (message_arena != submessage_arena) { - source_context = ::google::protobuf::internal::GetOwnedMessage( - message_arena, source_context, submessage_arena); - } - - } else { - - } - source_context_ = source_context; - // @@protoc_insertion_point(field_set_allocated:google.protobuf.Api.source_context) -} - -// repeated .google.protobuf.Mixin mixins = 6; -inline int Api::mixins_size() const { - return mixins_.size(); -} -inline void Api::clear_mixins() { - mixins_.Clear(); -} -inline const ::google::protobuf::Mixin& Api::mixins(int index) const { - // @@protoc_insertion_point(field_get:google.protobuf.Api.mixins) - return mixins_.Get(index); -} -inline ::google::protobuf::Mixin* Api::mutable_mixins(int index) { - // @@protoc_insertion_point(field_mutable:google.protobuf.Api.mixins) - return mixins_.Mutable(index); -} -inline ::google::protobuf::Mixin* Api::add_mixins() { - // @@protoc_insertion_point(field_add:google.protobuf.Api.mixins) - return mixins_.Add(); -} -inline ::google::protobuf::RepeatedPtrField< ::google::protobuf::Mixin >* -Api::mutable_mixins() { - // @@protoc_insertion_point(field_mutable_list:google.protobuf.Api.mixins) - return &mixins_; -} -inline const ::google::protobuf::RepeatedPtrField< ::google::protobuf::Mixin >& -Api::mixins() const { - // @@protoc_insertion_point(field_list:google.protobuf.Api.mixins) - return mixins_; -} - -// .google.protobuf.Syntax syntax = 7; -inline void Api::clear_syntax() { - syntax_ = 0; -} -inline ::google::protobuf::Syntax Api::syntax() const { - // @@protoc_insertion_point(field_get:google.protobuf.Api.syntax) - return static_cast< ::google::protobuf::Syntax >(syntax_); -} -inline void Api::set_syntax(::google::protobuf::Syntax value) { - - syntax_ = value; - // @@protoc_insertion_point(field_set:google.protobuf.Api.syntax) -} - -// ------------------------------------------------------------------- - -// Method - -// string name = 1; -inline void Method::clear_name() { - name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline const ::std::string& Method::name() const { - // @@protoc_insertion_point(field_get:google.protobuf.Method.name) - return name_.GetNoArena(); -} -inline void Method::set_name(const ::std::string& value) { - - name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); - // @@protoc_insertion_point(field_set:google.protobuf.Method.name) -} -#if LANG_CXX11 -inline void Method::set_name(::std::string&& value) { - - name_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); - // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Method.name) -} -#endif -inline void Method::set_name(const char* value) { - GOOGLE_DCHECK(value != NULL); - - name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); - // @@protoc_insertion_point(field_set_char:google.protobuf.Method.name) -} -inline void Method::set_name(const char* value, size_t size) { - - name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); - // @@protoc_insertion_point(field_set_pointer:google.protobuf.Method.name) -} -inline ::std::string* Method::mutable_name() { - - // @@protoc_insertion_point(field_mutable:google.protobuf.Method.name) - return name_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline ::std::string* Method::release_name() { - // @@protoc_insertion_point(field_release:google.protobuf.Method.name) - - return name_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline void Method::set_allocated_name(::std::string* name) { - if (name != NULL) { - - } else { - - } - name_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), name); - // @@protoc_insertion_point(field_set_allocated:google.protobuf.Method.name) -} - -// string request_type_url = 2; -inline void Method::clear_request_type_url() { - request_type_url_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline const ::std::string& Method::request_type_url() const { - // @@protoc_insertion_point(field_get:google.protobuf.Method.request_type_url) - return request_type_url_.GetNoArena(); -} -inline void Method::set_request_type_url(const ::std::string& value) { - - request_type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); - // @@protoc_insertion_point(field_set:google.protobuf.Method.request_type_url) -} -#if LANG_CXX11 -inline void Method::set_request_type_url(::std::string&& value) { - - request_type_url_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); - // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Method.request_type_url) -} -#endif -inline void Method::set_request_type_url(const char* value) { - GOOGLE_DCHECK(value != NULL); - - request_type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); - // @@protoc_insertion_point(field_set_char:google.protobuf.Method.request_type_url) -} -inline void Method::set_request_type_url(const char* value, size_t size) { - - request_type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); - // @@protoc_insertion_point(field_set_pointer:google.protobuf.Method.request_type_url) -} -inline ::std::string* Method::mutable_request_type_url() { - - // @@protoc_insertion_point(field_mutable:google.protobuf.Method.request_type_url) - return request_type_url_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline ::std::string* Method::release_request_type_url() { - // @@protoc_insertion_point(field_release:google.protobuf.Method.request_type_url) - - return request_type_url_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline void Method::set_allocated_request_type_url(::std::string* request_type_url) { - if (request_type_url != NULL) { - - } else { - - } - request_type_url_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), request_type_url); - // @@protoc_insertion_point(field_set_allocated:google.protobuf.Method.request_type_url) -} - -// bool request_streaming = 3; -inline void Method::clear_request_streaming() { - request_streaming_ = false; -} -inline bool Method::request_streaming() const { - // @@protoc_insertion_point(field_get:google.protobuf.Method.request_streaming) - return request_streaming_; -} -inline void Method::set_request_streaming(bool value) { - - request_streaming_ = value; - // @@protoc_insertion_point(field_set:google.protobuf.Method.request_streaming) -} - -// string response_type_url = 4; -inline void Method::clear_response_type_url() { - response_type_url_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline const ::std::string& Method::response_type_url() const { - // @@protoc_insertion_point(field_get:google.protobuf.Method.response_type_url) - return response_type_url_.GetNoArena(); -} -inline void Method::set_response_type_url(const ::std::string& value) { - - response_type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); - // @@protoc_insertion_point(field_set:google.protobuf.Method.response_type_url) -} -#if LANG_CXX11 -inline void Method::set_response_type_url(::std::string&& value) { - - response_type_url_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); - // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Method.response_type_url) -} -#endif -inline void Method::set_response_type_url(const char* value) { - GOOGLE_DCHECK(value != NULL); - - response_type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); - // @@protoc_insertion_point(field_set_char:google.protobuf.Method.response_type_url) -} -inline void Method::set_response_type_url(const char* value, size_t size) { - - response_type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); - // @@protoc_insertion_point(field_set_pointer:google.protobuf.Method.response_type_url) -} -inline ::std::string* Method::mutable_response_type_url() { - - // @@protoc_insertion_point(field_mutable:google.protobuf.Method.response_type_url) - return response_type_url_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline ::std::string* Method::release_response_type_url() { - // @@protoc_insertion_point(field_release:google.protobuf.Method.response_type_url) - - return response_type_url_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline void Method::set_allocated_response_type_url(::std::string* response_type_url) { - if (response_type_url != NULL) { - - } else { - - } - response_type_url_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), response_type_url); - // @@protoc_insertion_point(field_set_allocated:google.protobuf.Method.response_type_url) -} - -// bool response_streaming = 5; -inline void Method::clear_response_streaming() { - response_streaming_ = false; -} -inline bool Method::response_streaming() const { - // @@protoc_insertion_point(field_get:google.protobuf.Method.response_streaming) - return response_streaming_; -} -inline void Method::set_response_streaming(bool value) { - - response_streaming_ = value; - // @@protoc_insertion_point(field_set:google.protobuf.Method.response_streaming) -} - -// repeated .google.protobuf.Option options = 6; -inline int Method::options_size() const { - return options_.size(); -} -inline const ::google::protobuf::Option& Method::options(int index) const { - // @@protoc_insertion_point(field_get:google.protobuf.Method.options) - return options_.Get(index); -} -inline ::google::protobuf::Option* Method::mutable_options(int index) { - // @@protoc_insertion_point(field_mutable:google.protobuf.Method.options) - return options_.Mutable(index); -} -inline ::google::protobuf::Option* Method::add_options() { - // @@protoc_insertion_point(field_add:google.protobuf.Method.options) - return options_.Add(); -} -inline ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option >* -Method::mutable_options() { - // @@protoc_insertion_point(field_mutable_list:google.protobuf.Method.options) - return &options_; -} -inline const ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option >& -Method::options() const { - // @@protoc_insertion_point(field_list:google.protobuf.Method.options) - return options_; -} - -// .google.protobuf.Syntax syntax = 7; -inline void Method::clear_syntax() { - syntax_ = 0; -} -inline ::google::protobuf::Syntax Method::syntax() const { - // @@protoc_insertion_point(field_get:google.protobuf.Method.syntax) - return static_cast< ::google::protobuf::Syntax >(syntax_); -} -inline void Method::set_syntax(::google::protobuf::Syntax value) { - - syntax_ = value; - // @@protoc_insertion_point(field_set:google.protobuf.Method.syntax) -} - -// ------------------------------------------------------------------- - -// Mixin - -// string name = 1; -inline void Mixin::clear_name() { - name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline const ::std::string& Mixin::name() const { - // @@protoc_insertion_point(field_get:google.protobuf.Mixin.name) - return name_.GetNoArena(); -} -inline void Mixin::set_name(const ::std::string& value) { - - name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); - // @@protoc_insertion_point(field_set:google.protobuf.Mixin.name) -} -#if LANG_CXX11 -inline void Mixin::set_name(::std::string&& value) { - - name_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); - // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Mixin.name) -} -#endif -inline void Mixin::set_name(const char* value) { - GOOGLE_DCHECK(value != NULL); - - name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); - // @@protoc_insertion_point(field_set_char:google.protobuf.Mixin.name) -} -inline void Mixin::set_name(const char* value, size_t size) { - - name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); - // @@protoc_insertion_point(field_set_pointer:google.protobuf.Mixin.name) -} -inline ::std::string* Mixin::mutable_name() { - - // @@protoc_insertion_point(field_mutable:google.protobuf.Mixin.name) - return name_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline ::std::string* Mixin::release_name() { - // @@protoc_insertion_point(field_release:google.protobuf.Mixin.name) - - return name_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline void Mixin::set_allocated_name(::std::string* name) { - if (name != NULL) { - - } else { - - } - name_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), name); - // @@protoc_insertion_point(field_set_allocated:google.protobuf.Mixin.name) -} - -// string root = 2; -inline void Mixin::clear_root() { - root_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline const ::std::string& Mixin::root() const { - // @@protoc_insertion_point(field_get:google.protobuf.Mixin.root) - return root_.GetNoArena(); -} -inline void Mixin::set_root(const ::std::string& value) { - - root_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); - // @@protoc_insertion_point(field_set:google.protobuf.Mixin.root) -} -#if LANG_CXX11 -inline void Mixin::set_root(::std::string&& value) { - - root_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); - // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Mixin.root) -} -#endif -inline void Mixin::set_root(const char* value) { - GOOGLE_DCHECK(value != NULL); - - root_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); - // @@protoc_insertion_point(field_set_char:google.protobuf.Mixin.root) -} -inline void Mixin::set_root(const char* value, size_t size) { - - root_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); - // @@protoc_insertion_point(field_set_pointer:google.protobuf.Mixin.root) -} -inline ::std::string* Mixin::mutable_root() { - - // @@protoc_insertion_point(field_mutable:google.protobuf.Mixin.root) - return root_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline ::std::string* Mixin::release_root() { - // @@protoc_insertion_point(field_release:google.protobuf.Mixin.root) - - return root_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline void Mixin::set_allocated_root(::std::string* root) { - if (root != NULL) { - - } else { - - } - root_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), root); - // @@protoc_insertion_point(field_set_allocated:google.protobuf.Mixin.root) -} - -#ifdef __GNUC__ - #pragma GCC diagnostic pop -#endif // __GNUC__ -// ------------------------------------------------------------------- - -// ------------------------------------------------------------------- - - -// @@protoc_insertion_point(namespace_scope) - -} // namespace protobuf -} // namespace google - -// @@protoc_insertion_point(global_scope) - -#endif // PROTOBUF_google_2fprotobuf_2fapi_2eproto__INCLUDED diff --git a/3rdparty/protobuf/src/google/protobuf/arena.cc b/3rdparty/protobuf/src/google/protobuf/arena.cc index e92899888d..7624e0b2f3 100644 --- a/3rdparty/protobuf/src/google/protobuf/arena.cc +++ b/3rdparty/protobuf/src/google/protobuf/arena.cc @@ -31,355 +31,481 @@ #include #include +#include +#include +#include #include +#include +#include +#include #ifdef ADDRESS_SANITIZER #include #endif // ADDRESS_SANITIZER -#include +#include namespace google { -static const size_t kMinCleanupListElements = 8; -static const size_t kMaxCleanupListElements = 64; // 1kB on 64-bit. - namespace protobuf { namespace internal { +static SerialArena::Memory AllocateMemory(const AllocationPolicy* policy_ptr, + size_t last_size, size_t min_bytes) { + AllocationPolicy policy; // default policy + if (policy_ptr) policy = *policy_ptr; + size_t size; + if (last_size != 0) { + // Double the current block size, up to a limit. + auto max_size = policy.max_block_size; + size = std::min(2 * last_size, max_size); + } else { + size = policy.start_block_size; + } + // Verify that min_bytes + kBlockHeaderSize won't overflow. + GOOGLE_CHECK_LE(min_bytes, + std::numeric_limits::max() - SerialArena::kBlockHeaderSize); + size = std::max(size, SerialArena::kBlockHeaderSize + min_bytes); -google::protobuf::internal::SequenceNumber ArenaImpl::lifecycle_id_generator_; + void* mem; + if (policy.block_alloc == nullptr) { + mem = ::operator new(size); + } else { + mem = policy.block_alloc(size); + } + return {mem, size}; +} + +class GetDeallocator { + public: + GetDeallocator(const AllocationPolicy* policy, size_t* space_allocated) + : dealloc_(policy ? policy->block_dealloc : nullptr), + space_allocated_(space_allocated) {} + + void operator()(SerialArena::Memory mem) const { +#ifdef ADDRESS_SANITIZER + // This memory was provided by the underlying allocator as unpoisoned, + // so return it in an unpoisoned state. + ASAN_UNPOISON_MEMORY_REGION(mem.ptr, mem.size); +#endif // ADDRESS_SANITIZER + if (dealloc_) { + dealloc_(mem.ptr, mem.size); + } else { +#if defined(__GXX_DELETE_WITH_SIZE__) || defined(__cpp_sized_deallocation) + ::operator delete(mem.ptr, mem.size); +#else + ::operator delete(mem.ptr); +#endif + } + *space_allocated_ += mem.size; + } + + private: + void (*dealloc_)(void*, size_t); + size_t* space_allocated_; +}; + +SerialArena::SerialArena(Block* b, void* owner) : space_allocated_(b->size) { + owner_ = owner; + head_ = b; + ptr_ = b->Pointer(kBlockHeaderSize + ThreadSafeArena::kSerialArenaSize); + limit_ = b->Pointer(b->size & static_cast(-8)); +} + +SerialArena* SerialArena::New(Memory mem, void* owner) { + GOOGLE_DCHECK_LE(kBlockHeaderSize + ThreadSafeArena::kSerialArenaSize, mem.size); + + auto b = new (mem.ptr) Block{nullptr, mem.size}; + return new (b->Pointer(kBlockHeaderSize)) SerialArena(b, owner); +} + +template +SerialArena::Memory SerialArena::Free(Deallocator deallocator) { + Block* b = head_; + Memory mem = {b, b->size}; + while (b->next) { + b = b->next; // We must first advance before deleting this block + deallocator(mem); + mem = {b, b->size}; + } + return mem; +} + +PROTOBUF_NOINLINE +std::pair +SerialArena::AllocateAlignedWithCleanupFallback( + size_t n, const AllocationPolicy* policy) { + AllocateNewBlock(n + kCleanupSize, policy); + return AllocateFromExistingWithCleanupFallback(n); +} + +PROTOBUF_NOINLINE +void* SerialArena::AllocateAlignedFallback(size_t n, + const AllocationPolicy* policy) { + AllocateNewBlock(n, policy); + return AllocateFromExisting(n); +} + +void SerialArena::AllocateNewBlock(size_t n, const AllocationPolicy* policy) { + // Sync limit to block + head_->start = reinterpret_cast(limit_); + + // Record how much used in this block. + space_used_ += ptr_ - head_->Pointer(kBlockHeaderSize); + + auto mem = AllocateMemory(policy, head_->size, n); + // We don't want to emit an expensive RMW instruction that requires + // exclusive access to a cacheline. Hence we write it in terms of a + // regular add. + auto relaxed = std::memory_order_relaxed; + space_allocated_.store(space_allocated_.load(relaxed) + mem.size, relaxed); + head_ = new (mem.ptr) Block{head_, mem.size}; + ptr_ = head_->Pointer(kBlockHeaderSize); + limit_ = head_->Pointer(head_->size); + +#ifdef ADDRESS_SANITIZER + ASAN_POISON_MEMORY_REGION(ptr_, limit_ - ptr_); +#endif // ADDRESS_SANITIZER +} + +uint64_t SerialArena::SpaceUsed() const { + uint64_t space_used = ptr_ - head_->Pointer(kBlockHeaderSize); + space_used += space_used_; + // Remove the overhead of the SerialArena itself. + space_used -= ThreadSafeArena::kSerialArenaSize; + return space_used; +} + +void SerialArena::CleanupList() { + Block* b = head_; + b->start = reinterpret_cast(limit_); + do { + auto* limit = reinterpret_cast( + b->Pointer(b->size & static_cast(-8))); + auto it = b->start; + auto num = limit - it; + if (num > 0) { + for (; it < limit; it++) { + it->cleanup(it->elem); + } + } + b = b->next; + } while (b); +} + + +ThreadSafeArena::CacheAlignedLifecycleIdGenerator + ThreadSafeArena::lifecycle_id_generator_; #if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL) -ArenaImpl::ThreadCache& ArenaImpl::thread_cache() { +ThreadSafeArena::ThreadCache& ThreadSafeArena::thread_cache() { static internal::ThreadLocalStorage* thread_cache_ = new internal::ThreadLocalStorage(); return *thread_cache_->Get(); } #elif defined(PROTOBUF_USE_DLLS) -ArenaImpl::ThreadCache& ArenaImpl::thread_cache() { - static GOOGLE_THREAD_LOCAL ThreadCache thread_cache_ = { -1, NULL }; +ThreadSafeArena::ThreadCache& ThreadSafeArena::thread_cache() { + static PROTOBUF_THREAD_LOCAL ThreadCache thread_cache_ = { + 0, static_cast(-1), nullptr}; return thread_cache_; } #else -GOOGLE_THREAD_LOCAL ArenaImpl::ThreadCache ArenaImpl::thread_cache_ = {-1, NULL}; +PROTOBUF_THREAD_LOCAL ThreadSafeArena::ThreadCache + ThreadSafeArena::thread_cache_ = {0, static_cast(-1), + nullptr}; #endif -void ArenaImpl::Init() { - lifecycle_id_ = lifecycle_id_generator_.GetNext(); - google::protobuf::internal::NoBarrier_Store(&hint_, 0); - google::protobuf::internal::NoBarrier_Store(&threads_, 0); - - if (initial_block_) { - // Thread which calls Init() owns the first block. This allows the - // single-threaded case to allocate on the first block without having to - // perform atomic operations. - InitBlock(initial_block_, &thread_cache(), options_.initial_block_size); - ThreadInfo* info = NewThreadInfo(initial_block_); - info->next = NULL; - google::protobuf::internal::NoBarrier_Store(&threads_, - reinterpret_cast(info)); - google::protobuf::internal::NoBarrier_Store(&space_allocated_, - options_.initial_block_size); - CacheBlock(initial_block_); - } else { - google::protobuf::internal::NoBarrier_Store(&space_allocated_, 0); - } -} - -ArenaImpl::~ArenaImpl() { - // Have to do this in a first pass, because some of the destructors might - // refer to memory in other blocks. - CleanupList(); - FreeBlocks(); -} - -uint64 ArenaImpl::Reset() { - // Have to do this in a first pass, because some of the destructors might - // refer to memory in other blocks. - CleanupList(); - uint64 space_allocated = FreeBlocks(); +void ThreadSafeArena::InitializeFrom(void* mem, size_t size) { + GOOGLE_DCHECK_EQ(reinterpret_cast(mem) & 7, 0u); + GOOGLE_DCHECK(!AllocPolicy()); // Reset should call InitializeWithPolicy instead. Init(); - return space_allocated; + // Ignore initial block if it is too small. + if (mem != nullptr && size >= kBlockHeaderSize + kSerialArenaSize) { + alloc_policy_.set_is_user_owned_initial_block(true); + SetInitialBlock(mem, size); + } } -ArenaImpl::Block* ArenaImpl::NewBlock(void* me, Block* my_last_block, - size_t min_bytes) { - size_t size; - if (my_last_block != NULL) { - // Double the current block size, up to a limit. - size = std::min(2 * my_last_block->size, options_.max_block_size); +void ThreadSafeArena::InitializeWithPolicy(void* mem, size_t size, + AllocationPolicy policy) { +#ifndef NDEBUG + const uint64_t old_alloc_policy = alloc_policy_.get_raw(); + // If there was a policy (e.g., in Reset()), make sure flags were preserved. +#define GOOGLE_DCHECK_POLICY_FLAGS_() \ + if (old_alloc_policy > 3) \ + GOOGLE_CHECK_EQ(old_alloc_policy & 3, alloc_policy_.get_raw() & 3) +#else +#define GOOGLE_DCHECK_POLICY_FLAGS_() +#endif // NDEBUG + + if (policy.IsDefault()) { + // Legacy code doesn't use the API above, but provides the initial block + // through ArenaOptions. I suspect most do not touch the allocation + // policy parameters. + InitializeFrom(mem, size); + GOOGLE_DCHECK_POLICY_FLAGS_(); + return; + } + GOOGLE_DCHECK_EQ(reinterpret_cast(mem) & 7, 0u); + Init(); + + // Ignore initial block if it is too small. We include an optional + // AllocationPolicy in this check, so that this can be allocated on the + // first block. + constexpr size_t kAPSize = internal::AlignUpTo8(sizeof(AllocationPolicy)); + constexpr size_t kMinimumSize = kBlockHeaderSize + kSerialArenaSize + kAPSize; + + // The value for alloc_policy_ stores whether or not allocations should be + // recorded. + alloc_policy_.set_should_record_allocs( + policy.metrics_collector != nullptr && + policy.metrics_collector->RecordAllocs()); + // Make sure we have an initial block to store the AllocationPolicy. + if (mem != nullptr && size >= kMinimumSize) { + alloc_policy_.set_is_user_owned_initial_block(true); } else { - size = options_.start_block_size; + auto tmp = AllocateMemory(&policy, 0, kMinimumSize); + mem = tmp.ptr; + size = tmp.size; } - // Verify that min_bytes + kHeaderSize won't overflow. - GOOGLE_CHECK_LE(min_bytes, std::numeric_limits::max() - kHeaderSize); - size = std::max(size, kHeaderSize + min_bytes); + SetInitialBlock(mem, size); - Block* b = reinterpret_cast(options_.block_alloc(size)); - InitBlock(b, me, size); - google::protobuf::internal::NoBarrier_AtomicIncrement(&space_allocated_, size); - return b; -} - -void ArenaImpl::InitBlock(Block* b, void *me, size_t size) { - b->pos = kHeaderSize; - b->size = size; - b->owner = me; - b->next = NULL; -#ifdef ADDRESS_SANITIZER - // Poison the rest of the block for ASAN. It was unpoisoned by the underlying - // malloc but it's not yet usable until we return it as part of an allocation. - ASAN_POISON_MEMORY_REGION( - reinterpret_cast(b) + b->pos, b->size - b->pos); -#endif // ADDRESS_SANITIZER -} - -ArenaImpl::CleanupChunk* ArenaImpl::ExpandCleanupList(CleanupChunk* cleanup, - Block* b) { - size_t size = cleanup ? cleanup->size * 2 : kMinCleanupListElements; - size = std::min(size, kMaxCleanupListElements); - size_t bytes = internal::AlignUpTo8(CleanupChunk::SizeOf(size)); - if (b->avail() < bytes) { - b = GetBlock(bytes); + auto sa = threads_.load(std::memory_order_relaxed); + // We ensured enough space so this cannot fail. + void* p; + if (!sa || !sa->MaybeAllocateAligned(kAPSize, &p)) { + GOOGLE_LOG(FATAL) << "MaybeAllocateAligned cannot fail here."; + return; } - CleanupChunk* list = - reinterpret_cast(AllocFromBlock(b, bytes)); - list->next = b->thread_info->cleanup; - list->size = size; - list->len = 0; - b->thread_info->cleanup = list; - return list; + new (p) AllocationPolicy{policy}; + // Low bits store flags, so they mustn't be overwritten. + GOOGLE_DCHECK_EQ(0, reinterpret_cast(p) & 3); + alloc_policy_.set_policy(reinterpret_cast(p)); + GOOGLE_DCHECK_POLICY_FLAGS_(); + +#undef GOOGLE_DCHECK_POLICY_FLAGS_ } -inline GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE -void ArenaImpl::AddCleanupInBlock( - Block* b, void* elem, void (*func)(void*)) { - CleanupChunk* cleanup = b->thread_info->cleanup; - if (cleanup == NULL || cleanup->len == cleanup->size) { - cleanup = ExpandCleanupList(cleanup, b); +void ThreadSafeArena::Init() { +#ifndef NDEBUG + const bool was_message_owned = IsMessageOwned(); +#endif // NDEBUG + ThreadCache& tc = thread_cache(); + auto id = tc.next_lifecycle_id; + // We increment lifecycle_id's by multiples of two so we can use bit 0 as + // a tag. + constexpr uint64_t kDelta = 2; + constexpr uint64_t kInc = ThreadCache::kPerThreadIds * kDelta; + if (PROTOBUF_PREDICT_FALSE((id & (kInc - 1)) == 0)) { + constexpr auto relaxed = std::memory_order_relaxed; + // On platforms that don't support uint64_t atomics we can certainly not + // afford to increment by large intervals and expect uniqueness due to + // wrapping, hence we only add by 1. + id = lifecycle_id_generator_.id.fetch_add(1, relaxed) * kInc; + } + tc.next_lifecycle_id = id + kDelta; + // Message ownership is stored in tag_and_id_, and is set in the constructor. + // This flag bit must be preserved, even across calls to Reset(). + tag_and_id_ = id | (tag_and_id_ & kMessageOwnedArena); + hint_.store(nullptr, std::memory_order_relaxed); + threads_.store(nullptr, std::memory_order_relaxed); +#ifndef NDEBUG + GOOGLE_CHECK_EQ(was_message_owned, IsMessageOwned()); +#endif // NDEBUG +} + +void ThreadSafeArena::SetInitialBlock(void* mem, size_t size) { + SerialArena* serial = SerialArena::New({mem, size}, &thread_cache()); + serial->set_next(NULL); + threads_.store(serial, std::memory_order_relaxed); + CacheSerialArena(serial); +} + +ThreadSafeArena::~ThreadSafeArena() { + // Have to do this in a first pass, because some of the destructors might + // refer to memory in other blocks. + CleanupList(); + + size_t space_allocated = 0; + auto mem = Free(&space_allocated); + + // Policy is about to get deleted. + auto* p = alloc_policy_.get(); + ArenaMetricsCollector* collector = p ? p->metrics_collector : nullptr; + + if (alloc_policy_.is_user_owned_initial_block()) { + space_allocated += mem.size; + } else { + GetDeallocator(alloc_policy_.get(), &space_allocated)(mem); } - CleanupNode* node = &cleanup->nodes[cleanup->len++]; - - node->elem = elem; - node->cleanup = func; + if (collector) collector->OnDestroy(space_allocated); } -void ArenaImpl::AddCleanup(void* elem, void (*cleanup)(void*)) { - return AddCleanupInBlock(GetBlock(0), elem, cleanup); -} - -void* ArenaImpl::AllocateAligned(size_t n) { - GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n); // Must be already aligned. - - return AllocFromBlock(GetBlock(n), n); -} - -void* ArenaImpl::AllocateAlignedAndAddCleanup(size_t n, - void (*cleanup)(void*)) { - GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n); // Must be already aligned. - - Block* b = GetBlock(n); - void* mem = AllocFromBlock(b, n); - AddCleanupInBlock(b, mem, cleanup); +SerialArena::Memory ThreadSafeArena::Free(size_t* space_allocated) { + SerialArena::Memory mem = {nullptr, 0}; + auto deallocator = GetDeallocator(alloc_policy_.get(), space_allocated); + PerSerialArena([deallocator, &mem](SerialArena* a) { + if (mem.ptr) deallocator(mem); + mem = a->Free(deallocator); + }); return mem; } -inline GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE -ArenaImpl::Block* ArenaImpl::GetBlock(size_t n) { - Block* my_block = NULL; +uint64_t ThreadSafeArena::Reset() { + // Have to do this in a first pass, because some of the destructors might + // refer to memory in other blocks. + CleanupList(); - // If this thread already owns a block in this arena then try to use that. - // This fast path optimizes the case where multiple threads allocate from the - // same arena. - ThreadCache* tc = &thread_cache(); - if (tc->last_lifecycle_id_seen == lifecycle_id_) { - my_block = tc->last_block_used_; - if (my_block->avail() >= n) { - return my_block; + // Discard all blocks except the special block (if present). + size_t space_allocated = 0; + auto mem = Free(&space_allocated); + + AllocationPolicy* policy = alloc_policy_.get(); + if (policy) { + auto saved_policy = *policy; + if (alloc_policy_.is_user_owned_initial_block()) { + space_allocated += mem.size; + } else { + GetDeallocator(alloc_policy_.get(), &space_allocated)(mem); + mem.ptr = nullptr; + mem.size = 0; } - } - - // Check whether we own the last accessed block on this arena. - // This fast path optimizes the case where a single thread uses multiple - // arenas. - Block* b = reinterpret_cast(google::protobuf::internal::Acquire_Load(&hint_)); - if (b != NULL && b->owner == tc) { - my_block = b; - if (my_block->avail() >= n) { - return my_block; + ArenaMetricsCollector* collector = saved_policy.metrics_collector; + if (collector) collector->OnReset(space_allocated); + InitializeWithPolicy(mem.ptr, mem.size, saved_policy); + } else { + GOOGLE_DCHECK(!alloc_policy_.should_record_allocs()); + // Nullptr policy + if (alloc_policy_.is_user_owned_initial_block()) { + space_allocated += mem.size; + InitializeFrom(mem.ptr, mem.size); + } else { + GetDeallocator(alloc_policy_.get(), &space_allocated)(mem); + Init(); } } - return GetBlockSlow(tc, my_block, n); -} - -inline GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE -void* ArenaImpl::AllocFromBlock(Block* b, size_t n) { - GOOGLE_DCHECK_EQ(internal::AlignUpTo8(b->pos), b->pos); // Must be already aligned. - GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n); // Must be already aligned. - GOOGLE_DCHECK_GE(b->avail(), n); - size_t p = b->pos; - b->pos = p + n; -#ifdef ADDRESS_SANITIZER - ASAN_UNPOISON_MEMORY_REGION(reinterpret_cast(b) + p, n); -#endif // ADDRESS_SANITIZER - return reinterpret_cast(b) + p; -} - -ArenaImpl::Block* ArenaImpl::GetBlockSlow(void* me, Block* my_full_block, - size_t n) { - ThreadInfo* info = - my_full_block ? my_full_block->thread_info : GetThreadInfo(me, n); - GOOGLE_DCHECK(info != NULL); - Block* b = info->head; - if (b->avail() < n) { - Block* new_b = NewBlock(me, b, n); - new_b->thread_info = info; - new_b->next = b; - info->head = new_b; - b = new_b; - } - CacheBlock(b); - return b; -} - -uint64 ArenaImpl::SpaceAllocated() const { - return google::protobuf::internal::NoBarrier_Load(&space_allocated_); -} - -uint64 ArenaImpl::SpaceUsed() const { - ThreadInfo* info = - reinterpret_cast(google::protobuf::internal::Acquire_Load(&threads_)); - uint64 space_used = 0; - - for ( ; info; info = info->next) { - // Remove the overhead of the ThreadInfo itself. - space_used -= sizeof(ThreadInfo); - for (Block* b = info->head; b; b = b->next) { - space_used += (b->pos - kHeaderSize); - } - } - - return space_used; -} - -uint64 ArenaImpl::FreeBlocks() { - uint64 space_allocated = 0; - // By omitting an Acquire barrier we ensure that any user code that doesn't - // properly synchronize Reset() or the destructor will throw a TSAN warning. - ThreadInfo* info = - reinterpret_cast(google::protobuf::internal::NoBarrier_Load(&threads_)); - - while (info) { - // This is inside the block we are freeing, so we need to read it now. - ThreadInfo* next_info = info->next; - for (Block* b = info->head; b; ) { - // This is inside the block we are freeing, so we need to read it now. - Block* next_block = b->next; - space_allocated += (b->size); - -#ifdef ADDRESS_SANITIZER - // This memory was provided by the underlying allocator as unpoisoned, so - // return it in an unpoisoned state. - ASAN_UNPOISON_MEMORY_REGION(reinterpret_cast(b), b->size); -#endif // ADDRESS_SANITIZER - - if (b != initial_block_) { - options_.block_dealloc(b, b->size); - } - - b = next_block; - } - info = next_info; - } return space_allocated; } -void ArenaImpl::CleanupList() { - // By omitting an Acquire barrier we ensure that any user code that doesn't - // properly synchronize Reset() or the destructor will throw a TSAN warning. - ThreadInfo* info = - reinterpret_cast(google::protobuf::internal::NoBarrier_Load(&threads_)); - - for ( ; info; info = info->next) { - CleanupChunk* list = info->cleanup; - while (list) { - size_t n = list->len; - CleanupNode* node = &list->nodes[list->len - 1]; - for (size_t i = 0; i < n; i++, node--) { - node->cleanup(node->elem); - } - list = list->next; - } +std::pair +ThreadSafeArena::AllocateAlignedWithCleanup(size_t n, + const std::type_info* type) { + SerialArena* arena; + if (PROTOBUF_PREDICT_TRUE(!alloc_policy_.should_record_allocs() && + GetSerialArenaFast(&arena))) { + return arena->AllocateAlignedWithCleanup(n, alloc_policy_.get()); + } else { + return AllocateAlignedWithCleanupFallback(n, type); } } -ArenaImpl::ThreadInfo* ArenaImpl::NewThreadInfo(Block* b) { - GOOGLE_DCHECK(FindThreadInfo(b->owner) == NULL); - ThreadInfo* info = - reinterpret_cast(AllocFromBlock(b, sizeof(ThreadInfo))); - b->thread_info = info; - info->owner = b->owner; - info->head = b; - info->cleanup = NULL; - return info; +void ThreadSafeArena::AddCleanup(void* elem, void (*cleanup)(void*)) { + SerialArena* arena; + if (PROTOBUF_PREDICT_FALSE(!GetSerialArenaFast(&arena))) { + arena = GetSerialArenaFallback(&thread_cache()); + } + arena->AddCleanup(elem, cleanup, AllocPolicy()); } -ArenaImpl::ThreadInfo* ArenaImpl::FindThreadInfo(void* me) { - ThreadInfo* info = - reinterpret_cast(google::protobuf::internal::Acquire_Load(&threads_)); - for ( ; info; info = info->next) { - if (info->owner == me) { - return info; +PROTOBUF_NOINLINE +void* ThreadSafeArena::AllocateAlignedFallback(size_t n, + const std::type_info* type) { + if (alloc_policy_.should_record_allocs()) { + alloc_policy_.RecordAlloc(type, n); + SerialArena* arena; + if (PROTOBUF_PREDICT_TRUE(GetSerialArenaFast(&arena))) { + return arena->AllocateAligned(n, alloc_policy_.get()); + } + } + return GetSerialArenaFallback(&thread_cache()) + ->AllocateAligned(n, alloc_policy_.get()); +} + +PROTOBUF_NOINLINE +std::pair +ThreadSafeArena::AllocateAlignedWithCleanupFallback( + size_t n, const std::type_info* type) { + if (alloc_policy_.should_record_allocs()) { + alloc_policy_.RecordAlloc(type, n); + SerialArena* arena; + if (GetSerialArenaFast(&arena)) { + return arena->AllocateAlignedWithCleanup(n, alloc_policy_.get()); + } + } + return GetSerialArenaFallback(&thread_cache()) + ->AllocateAlignedWithCleanup(n, alloc_policy_.get()); +} + +uint64_t ThreadSafeArena::SpaceAllocated() const { + SerialArena* serial = threads_.load(std::memory_order_acquire); + uint64_t res = 0; + for (; serial; serial = serial->next()) { + res += serial->SpaceAllocated(); + } + return res; +} + +uint64_t ThreadSafeArena::SpaceUsed() const { + SerialArena* serial = threads_.load(std::memory_order_acquire); + uint64_t space_used = 0; + for (; serial; serial = serial->next()) { + space_used += serial->SpaceUsed(); + } + return space_used - (alloc_policy_.get() ? sizeof(AllocationPolicy) : 0); +} + +void ThreadSafeArena::CleanupList() { + PerSerialArena([](SerialArena* a) { a->CleanupList(); }); +} + +PROTOBUF_NOINLINE +SerialArena* ThreadSafeArena::GetSerialArenaFallback(void* me) { + // Look for this SerialArena in our linked list. + SerialArena* serial = threads_.load(std::memory_order_acquire); + for (; serial; serial = serial->next()) { + if (serial->owner() == me) { + break; } } - return NULL; -} + if (!serial) { + // This thread doesn't have any SerialArena, which also means it doesn't + // have any blocks yet. So we'll allocate its first block now. + serial = SerialArena::New( + AllocateMemory(alloc_policy_.get(), 0, kSerialArenaSize), me); -ArenaImpl::ThreadInfo* ArenaImpl::GetThreadInfo(void* me, size_t n) { - ThreadInfo* info = FindThreadInfo(me); - - if (!info) { - // This thread doesn't have any ThreadInfo, which also means it doesn't have - // any blocks yet. So we'll allocate its first block now. - Block* b = NewBlock(me, NULL, sizeof(ThreadInfo) + n); - info = NewThreadInfo(b); - - google::protobuf::internal::AtomicWord head; + SerialArena* head = threads_.load(std::memory_order_relaxed); do { - head = google::protobuf::internal::NoBarrier_Load(&threads_); - info->next = reinterpret_cast(head); - } while (google::protobuf::internal::Release_CompareAndSwap( - &threads_, head, reinterpret_cast(info)) != head); + serial->set_next(head); + } while (!threads_.compare_exchange_weak( + head, serial, std::memory_order_release, std::memory_order_relaxed)); } - return info; + CacheSerialArena(serial); + return serial; } } // namespace internal -void Arena::CallDestructorHooks() { - uint64 space_allocated = impl_.SpaceAllocated(); - // Call the reset hook - if (on_arena_reset_ != NULL) { - on_arena_reset_(this, hooks_cookie_, space_allocated); - } - - // Call the destruction hook - if (on_arena_destruction_ != NULL) { - on_arena_destruction_(this, hooks_cookie_, space_allocated); - } +PROTOBUF_FUNC_ALIGN(32) +void* Arena::AllocateAlignedNoHook(size_t n) { + return impl_.AllocateAligned(n, nullptr); } -void Arena::OnArenaAllocation(const std::type_info* allocated_type, - size_t n) const { - if (on_arena_allocation_ != NULL) { - on_arena_allocation_(allocated_type, n, hooks_cookie_); - } +PROTOBUF_FUNC_ALIGN(32) +void* Arena::AllocateAlignedWithHook(size_t n, const std::type_info* type) { + return impl_.AllocateAligned(n, type); +} + +PROTOBUF_FUNC_ALIGN(32) +std::pair +Arena::AllocateAlignedWithCleanup(size_t n, const std::type_info* type) { + return impl_.AllocateAlignedWithCleanup(n, type); } } // namespace protobuf } // namespace google + +#include diff --git a/3rdparty/protobuf/src/google/protobuf/arena.h b/3rdparty/protobuf/src/google/protobuf/arena.h index 32be9a1726..6dd6467f58 100644 --- a/3rdparty/protobuf/src/google/protobuf/arena.h +++ b/3rdparty/protobuf/src/google/protobuf/arena.h @@ -33,14 +33,14 @@ #ifndef GOOGLE_PROTOBUF_ARENA_H__ #define GOOGLE_PROTOBUF_ARENA_H__ + #include +#include +#include #ifdef max #undef max // Visual Studio defines this macro #endif -#if LANG_CXX11 -#include -#endif -#if defined(_MSC_VER) && !_HAS_EXCEPTIONS +#if defined(_MSC_VER) && !defined(_LIBCPP_STD_VER) && !_HAS_EXCEPTIONS // Work around bugs in MSVC header when _HAS_EXCEPTIONS=0. #include #include @@ -51,38 +51,72 @@ using type_info = ::type_info; #include #endif +#include #include -#include +#include + +#include + +#ifdef SWIG +#error "You cannot SWIG proto headers" +#endif namespace google { namespace protobuf { -class Arena; // defined below -class Message; // message.h +struct ArenaOptions; // defined below +class Arena; // defined below +class Message; // defined in message.h +class MessageLite; +template +class Map; + +namespace arena_metrics { + +void EnableArenaMetrics(ArenaOptions* options); + +} // namespace arena_metrics + +namespace TestUtil { +class ReflectionTester; // defined in test_util.h +} // namespace TestUtil namespace internal { -struct ArenaStringPtr; // arenastring.h -class LazyField; // lazy_field.h -template -class GenericTypeHandler; // repeated_field.h +struct ArenaStringPtr; // defined in arenastring.h +class InlinedStringField; // defined in inlined_string_field.h +class LazyField; // defined in lazy_field.h +class EpsCopyInputStream; // defined in parse_context.h + +template +class GenericTypeHandler; // defined in repeated_field.h + +inline PROTOBUF_ALWAYS_INLINE +void* AlignTo(void* ptr, size_t align) { + return reinterpret_cast( + (reinterpret_cast(ptr) + align - 1) & (~align + 1)); +} // Templated cleanup methods. -template void arena_destruct_object(void* object) { +template +void arena_destruct_object(void* object) { reinterpret_cast(object)->~T(); } -template void arena_delete_object(void* object) { + +template +struct ObjectDestructor { + constexpr static void (*destructor)(void*) = &arena_destruct_object; +}; + +template +struct ObjectDestructor { + constexpr static void (*destructor)(void*) = nullptr; +}; + +template +void arena_delete_object(void* object) { delete reinterpret_cast(object); } -inline void arena_free(void* object, size_t size) { -#if defined(__GXX_DELETE_WITH_SIZE__) || defined(__cpp_sized_deallocation) - ::operator delete(object, size); -#else - (void)size; - ::operator delete(object); -#endif -} - } // namespace internal // ArenaOptions provides optional additional parameters to arena construction @@ -117,49 +151,45 @@ struct ArenaOptions { // from the arena. By default, it contains a ptr to a wrapper function that // calls free. void (*block_dealloc)(void*, size_t); - // Hooks for adding external functionality such as user-specific metrics - // collection, specific debugging abilities, etc. - // Init hook may return a pointer to a cookie to be stored in the arena. - // reset and destruction hooks will then be called with the same cookie - // pointer. This allows us to save an external object per arena instance and - // use it on the other hooks (Note: It is just as legal for init to return - // NULL and not use the cookie feature). - // on_arena_reset and on_arena_destruction also receive the space used in - // the arena just before the reset. - void* (*on_arena_init)(Arena* arena); - void (*on_arena_reset)(Arena* arena, void* cookie, uint64 space_used); - void (*on_arena_destruction)(Arena* arena, void* cookie, uint64 space_used); - - // type_info is promised to be static - its lifetime extends to - // match program's lifetime (It is given by typeid operator). - // Note: typeid(void) will be passed as allocated_type every time we - // intentionally want to avoid monitoring an allocation. (i.e. internal - // allocations for managing the arena) - void (*on_arena_allocation)(const std::type_info* allocated_type, - uint64 alloc_size, void* cookie); ArenaOptions() - : start_block_size(kDefaultStartBlockSize), - max_block_size(kDefaultMaxBlockSize), + : start_block_size(internal::AllocationPolicy::kDefaultStartBlockSize), + max_block_size(internal::AllocationPolicy::kDefaultMaxBlockSize), initial_block(NULL), initial_block_size(0), - block_alloc(&::operator new), - block_dealloc(&internal::arena_free), - on_arena_init(NULL), - on_arena_reset(NULL), - on_arena_destruction(NULL), - on_arena_allocation(NULL) {} + block_alloc(nullptr), + block_dealloc(nullptr), + make_metrics_collector(nullptr) {} private: - // Constants define default starting block size and max block size for - // arena allocator behavior -- see descriptions above. - static const size_t kDefaultStartBlockSize = 256; - static const size_t kDefaultMaxBlockSize = 8192; + // If make_metrics_collector is not nullptr, it will be called at Arena init + // time. It may return a pointer to a collector instance that will be notified + // of interesting events related to the arena. + internal::ArenaMetricsCollector* (*make_metrics_collector)(); + + internal::ArenaMetricsCollector* MetricsCollector() const { + return make_metrics_collector ? (*make_metrics_collector)() : nullptr; + } + + internal::AllocationPolicy AllocationPolicy() const { + internal::AllocationPolicy res; + res.start_block_size = start_block_size; + res.max_block_size = max_block_size; + res.block_alloc = block_alloc; + res.block_dealloc = block_dealloc; + res.metrics_collector = MetricsCollector(); + return res; + } + + friend void arena_metrics::EnableArenaMetrics(ArenaOptions*); + + friend class Arena; + friend class ArenaOptionsTestFriend; }; // Support for non-RTTI environments. (The metrics hooks API uses type // information.) -#ifndef GOOGLE_PROTOBUF_NO_RTTI +#if PROTOBUF_RTTI #define RTTI_TYPE_ID(type) (&typeid(type)) #else #define RTTI_TYPE_ID(type) (NULL) @@ -185,14 +215,15 @@ struct ArenaOptions { // any special requirements on the type T, and will invoke the object's // destructor when the arena is destroyed. // -// The arena message allocation protocol, required by CreateMessage, is as -// follows: +// The arena message allocation protocol, required by +// CreateMessage(Arena* arena, Args&&... args), is as follows: // -// - The type T must have (at least) two constructors: a constructor with no -// arguments, called when a T is allocated on the heap; and a constructor with -// a google::protobuf::Arena* argument, called when a T is allocated on an arena. If the -// second constructor is called with a NULL arena pointer, it must be -// equivalent to invoking the first (no-argument) constructor. +// - The type T must have (at least) two constructors: a constructor callable +// with `args` (without `arena`), called when a T is allocated on the heap; +// and a constructor callable with `Arena* arena, Args&&... args`, called when +// a T is allocated on an arena. If the second constructor is called with a +// NULL arena pointer, it must be equivalent to invoking the first +// (`args`-only) constructor. // // - The type T must have a particular type trait: a nested type // |InternalArenaConstructable_|. This is usually a typedef to |void|. If no @@ -205,23 +236,28 @@ struct ArenaOptions { // present on the type, then its destructor is always called when the // containing arena is destroyed. // -// - One- and two-user-argument forms of CreateMessage() also exist that -// forward these constructor arguments to T's constructor: for example, -// CreateMessage(Arena*, arg1, arg2) forwards to a constructor T(Arena*, -// arg1, arg2). -// // This protocol is implemented by all arena-enabled proto2 message classes as -// well as RepeatedPtrField. -// -// Do NOT subclass Arena. This class will be marked as final when C++11 is -// enabled. -class LIBPROTOBUF_EXPORT Arena { +// well as protobuf container types like RepeatedPtrField and Map. The protocol +// is internal to protobuf and is not guaranteed to be stable. Non-proto types +// should not rely on this protocol. +class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { public: - // Arena constructor taking custom options. See ArenaOptions below for + // Default constructor with sensible default options, tuned for average + // use-cases. + inline Arena() : impl_() {} + + // Construct an arena with default options, except for the supplied + // initial block. It is more efficient to use this constructor + // instead of passing ArenaOptions if the only configuration needed + // by the caller is supplying an initial block. + inline Arena(char* initial_block, size_t initial_block_size) + : impl_(initial_block, initial_block_size) {} + + // Arena constructor taking custom options. See ArenaOptions above for // descriptions of the options available. - explicit Arena(const ArenaOptions& options) : impl_(options) { - Init(options); - } + explicit Arena(const ArenaOptions& options) + : impl_(options.initial_block, options.initial_block_size, + options.AllocationPolicy()) {} // Block overhead. Use this as a guide for how much to over-allocate the // initial block if you want an allocation of size N to fit inside it. @@ -229,29 +265,14 @@ class LIBPROTOBUF_EXPORT Arena { // WARNING: if you allocate multiple objects, it is difficult to guarantee // that a series of allocations will fit in the initial block, especially if // Arena changes its alignment guarantees in the future! - static const size_t kBlockOverhead = internal::ArenaImpl::kHeaderSize; + static const size_t kBlockOverhead = + internal::ThreadSafeArena::kBlockHeaderSize + + internal::ThreadSafeArena::kSerialArenaSize; - // Default constructor with sensible default options, tuned for average - // use-cases. - Arena() : impl_(ArenaOptions()) { Init(ArenaOptions()); } + inline ~Arena() {} - ~Arena() { - if (on_arena_reset_ != NULL || on_arena_destruction_ != NULL) { - CallDestructorHooks(); - } - } - - void Init(const ArenaOptions& options) { - on_arena_allocation_ = options.on_arena_allocation; - on_arena_reset_ = options.on_arena_reset; - on_arena_destruction_ = options.on_arena_destruction; - // Call the initialization hook - if (options.on_arena_init != NULL) { - hooks_cookie_ = options.on_arena_init(this); - } else { - hooks_cookie_ = NULL; - } - } + // TODO(protobuf-team): Fix callers to use constructor and delete this method. + void Init(const ArenaOptions&) {} // API to create proto2 message objects on the arena. If the arena passed in // is NULL, then a heap allocated object is returned. Type T must be a message @@ -263,69 +284,15 @@ class LIBPROTOBUF_EXPORT Arena { // // This function also accepts any type T that satisfies the arena message // allocation protocol, documented above. -#if LANG_CXX11 template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE static T* CreateMessage( - ::google::protobuf::Arena* arena, Args&&... args) { + PROTOBUF_ALWAYS_INLINE static T* CreateMessage(Arena* arena, Args&&... args) { static_assert( InternalHelper::is_arena_constructable::value, "CreateMessage can only construct types that are ArenaConstructable"); - if (arena == NULL) { - return new T(NULL, std::forward(args)...); - } else { - return arena->CreateMessageInternal(std::forward(args)...); - } - } -#endif - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* CreateMessage(::google::protobuf::Arena* arena) { -#if LANG_CXX11 - static_assert( - InternalHelper::is_arena_constructable::value, - "CreateMessage can only construct types that are ArenaConstructable"); -#endif - if (arena == NULL) { - return new T; - } else { - return arena->CreateMessageInternal(); - } - } - - // One-argument form of CreateMessage. This is useful for constructing objects - // that implement the arena message construction protocol described above but - // take additional constructor arguments. - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* CreateMessage(::google::protobuf::Arena* arena, const Arg& arg) { -#if LANG_CXX11 - static_assert( - InternalHelper::is_arena_constructable::value, - "CreateMessage can only construct types that are ArenaConstructable"); -#endif - if (arena == NULL) { - return new T(NULL, arg); - } else { - return arena->CreateMessageInternal(arg); - } - } - - // Two-argument form of CreateMessage. This is useful for constructing objects - // that implement the arena message construction protocol described above but - // take additional constructor arguments. - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* CreateMessage(::google::protobuf::Arena* arena, - const Arg1& arg1, - const Arg2& arg2) { -#if LANG_CXX11 - static_assert( - InternalHelper::is_arena_constructable::value, - "CreateMessage can only construct types that are ArenaConstructable"); -#endif - if (arena == NULL) { - return new T(NULL, arg1, arg2); - } else { - return arena->CreateMessageInternal(arg1, arg2); - } + // We must delegate to CreateMaybeMessage() and NOT CreateMessageInternal() + // because protobuf generated classes specialize CreateMaybeMessage() and we + // need to use that specialization for code size reasons. + return Arena::CreateMaybeMessage(arena, static_cast(args)...); } // API to create any objects on the arena. Note that only the object will @@ -343,152 +310,10 @@ class LIBPROTOBUF_EXPORT Arena { // (unless the destructor is trivial). Hence, from T's point of view, it is as // if the object were allocated on the heap (except that the underlying memory // is obtained from the arena). -#if LANG_CXX11 template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* Create(::google::protobuf::Arena* arena, Args&&... args) { - if (arena == NULL) { - return new T(std::forward(args)...); - } else { - return arena->CreateInternal(google::protobuf::internal::has_trivial_destructor::value, - std::forward(args)...); - } - } -#endif - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* Create(::google::protobuf::Arena* arena) { - if (arena == NULL) { - return new T(); - } else { - return arena->CreateInternal(google::protobuf::internal::has_trivial_destructor::value); - } - } - - // Version of the above with one constructor argument for the created object. - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* Create(::google::protobuf::Arena* arena, const Arg& arg) { - if (arena == NULL) { - return new T(arg); - } else { - return arena->CreateInternal(google::protobuf::internal::has_trivial_destructor::value, - arg); - } - } - - // Version of the above with two constructor arguments for the created object. - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* Create(::google::protobuf::Arena* arena, const Arg1& arg1, const Arg2& arg2) { - if (arena == NULL) { - return new T(arg1, arg2); - } else { - return arena->CreateInternal(google::protobuf::internal::has_trivial_destructor::value, - arg1, arg2); - } - } - - // Version of the above with three constructor arguments for the created - // object. - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* Create(::google::protobuf::Arena* arena, - const Arg1& arg1, - const Arg2& arg2, - const Arg3& arg3) { - if (arena == NULL) { - return new T(arg1, arg2, arg3); - } else { - return arena->CreateInternal(google::protobuf::internal::has_trivial_destructor::value, - arg1, arg2, arg3); - } - } - - // Version of the above with four constructor arguments for the created - // object. - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* Create(::google::protobuf::Arena* arena, - const Arg1& arg1, const Arg2& arg2, - const Arg3& arg3, const Arg4& arg4) { - if (arena == NULL) { - return new T(arg1, arg2, arg3, arg4); - } else { - return arena->CreateInternal(google::protobuf::internal::has_trivial_destructor::value, - arg1, arg2, arg3, arg4); - } - } - - // Version of the above with five constructor arguments for the created - // object. - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* Create(::google::protobuf::Arena* arena, - const Arg1& arg1, const Arg2& arg2, - const Arg3& arg3, const Arg4& arg4, - const Arg5& arg5) { - if (arena == NULL) { - return new T(arg1, arg2, arg3, arg4, arg5); - } else { - return arena->CreateInternal(google::protobuf::internal::has_trivial_destructor::value, - arg1, arg2, arg3, arg4, arg5); - } - } - - // Version of the above with six constructor arguments for the created - // object. - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* Create(::google::protobuf::Arena* arena, - const Arg1& arg1, const Arg2& arg2, - const Arg3& arg3, const Arg4& arg4, - const Arg5& arg5, const Arg6& arg6) { - if (arena == NULL) { - return new T(arg1, arg2, arg3, arg4, arg5, arg6); - } else { - return arena->CreateInternal(google::protobuf::internal::has_trivial_destructor::value, - arg1, arg2, arg3, arg4, arg5, arg6); - } - } - - // Version of the above with seven constructor arguments for the created - // object. - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* Create(::google::protobuf::Arena* arena, - const Arg1& arg1, const Arg2& arg2, - const Arg3& arg3, const Arg4& arg4, - const Arg5& arg5, const Arg6& arg6, - const Arg7& arg7) { - if (arena == NULL) { - return new T(arg1, arg2, arg3, arg4, arg5, arg6, arg7); - } else { - return arena->CreateInternal(google::protobuf::internal::has_trivial_destructor::value, - arg1, arg2, arg3, arg4, arg5, arg6, arg7); - } - } - - // Version of the above with eight constructor arguments for the created - // object. - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* Create(::google::protobuf::Arena* arena, - const Arg1& arg1, const Arg2& arg2, - const Arg3& arg3, const Arg4& arg4, - const Arg5& arg5, const Arg6& arg6, - const Arg7& arg7, const Arg8& arg8) { - if (arena == NULL) { - return new T(arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8); - } else { - return arena->CreateInternal( - google::protobuf::internal::has_trivial_destructor::value, - arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8); - } + PROTOBUF_NDEBUG_INLINE static T* Create(Arena* arena, Args&&... args) { + return CreateInternal(arena, std::is_convertible(), + static_cast(args)...); } // Create an array of object type T on the arena *without* invoking the @@ -497,10 +322,14 @@ class LIBPROTOBUF_EXPORT Arena { // To ensure safe uses, this function checks at compile time // (when compiled as C++11) that T is trivially default-constructible and // trivially destructible. - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* CreateArray(::google::protobuf::Arena* arena, size_t num_elements) { - GOOGLE_CHECK_LE(num_elements, - std::numeric_limits::max() / sizeof(T)) + template + PROTOBUF_NDEBUG_INLINE static T* CreateArray(Arena* arena, + size_t num_elements) { + static_assert(std::is_trivial::value, + "CreateArray requires a trivially constructible type"); + static_assert(std::is_trivially_destructible::value, + "CreateArray requires a trivially destructible type"); + GOOGLE_CHECK_LE(num_elements, std::numeric_limits::max() / sizeof(T)) << "Requested size is too large to fit into size_t."; if (arena == NULL) { return static_cast(::operator new[](num_elements * sizeof(T))); @@ -509,41 +338,31 @@ class LIBPROTOBUF_EXPORT Arena { } } + // The following are routines are for monitoring. They will approximate the + // total sum allocated and used memory, but the exact value is an + // implementation deal. For instance allocated space depends on growth + // policies. Do not use these in unit tests. // Returns the total space allocated by the arena, which is the sum of the - // sizes of the underlying blocks. This method is relatively fast; a counter - // is kept as blocks are allocated. - uint64 SpaceAllocated() const { return impl_.SpaceAllocated(); } + // sizes of the underlying blocks. + uint64_t SpaceAllocated() const { return impl_.SpaceAllocated(); } // Returns the total space used by the arena. Similar to SpaceAllocated but // does not include free space and block overhead. The total space returned // may not include space used by other threads executing concurrently with // the call to this method. - uint64 SpaceUsed() const { return impl_.SpaceUsed(); } - // DEPRECATED. Please use SpaceAllocated() and SpaceUsed(). - // - // Combines SpaceAllocated and SpaceUsed. Returns a pair of - // . - std::pair SpaceAllocatedAndUsed() const { - return std::make_pair(SpaceAllocated(), SpaceUsed()); - } + uint64_t SpaceUsed() const { return impl_.SpaceUsed(); } // Frees all storage allocated by this arena after calling destructors // registered with OwnDestructor() and freeing objects registered with Own(). // Any objects allocated on this arena are unusable after this call. It also // returns the total space used by the arena which is the sums of the sizes // of the allocated blocks. This method is not thread-safe. - GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE uint64 Reset() { - // Call the reset hook - if (on_arena_reset_ != NULL) { - on_arena_reset_(this, hooks_cookie_, impl_.SpaceAllocated()); - } - return impl_.Reset(); - } + uint64_t Reset() { return impl_.Reset(); } // Adds |object| to a list of heap-allocated objects to be freed with |delete| // when the arena is destroyed or reset. - template GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE - void Own(T* object) { - OwnInternal(object, google::protobuf::internal::is_convertible()); + template + PROTOBUF_ALWAYS_INLINE void Own(T* object) { + OwnInternal(object, std::is_convertible()); } // Adds |object| to a list of objects whose destructors will be manually @@ -551,8 +370,8 @@ class LIBPROTOBUF_EXPORT Arena { // that it does not free the underlying memory with |delete|; hence, it is // normally only used for objects that are placement-newed into // arena-allocated memory. - template GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE - void OwnDestructor(T* object) { + template + PROTOBUF_ALWAYS_INLINE void OwnDestructor(T* object) { if (object != NULL) { impl_.AddCleanup(object, &internal::arena_destruct_object); } @@ -562,102 +381,208 @@ class LIBPROTOBUF_EXPORT Arena { // will be manually called when the arena is destroyed or reset. This differs // from OwnDestructor() in that any member function may be specified, not only // the class destructor. - GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE void OwnCustomDestructor( - void* object, void (*destruct)(void*)) { + PROTOBUF_ALWAYS_INLINE void OwnCustomDestructor(void* object, + void (*destruct)(void*)) { impl_.AddCleanup(object, destruct); } // Retrieves the arena associated with |value| if |value| is an arena-capable - // message, or NULL otherwise. This differs from value->GetArena() in that the - // latter is a virtual call, while this method is a templated call that - // resolves at compile-time. - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static ::google::protobuf::Arena* GetArena(const T* value) { - return GetArenaInternal(value, is_arena_constructable()); + // message, or NULL otherwise. If possible, the call resolves at compile time. + // Note that we can often devirtualize calls to `value->GetArena()` so usually + // calling this method is unnecessary. + template + PROTOBUF_ALWAYS_INLINE static Arena* GetArena(const T* value) { + return GetArenaInternal(value); } template class InternalHelper { + public: + // Provides access to protected GetOwningArena to generated messages. + static Arena* GetOwningArena(const T* p) { return p->GetOwningArena(); } + + // Provides access to protected GetArenaForAllocation to generated messages. + static Arena* GetArenaForAllocation(const T* p) { + return GetArenaForAllocationInternal( + p, std::is_convertible()); + } + + // Creates message-owned arena. + static Arena* CreateMessageOwnedArena() { + return new Arena(internal::MessageOwned{}); + } + + // Checks whether the given arena is message-owned. + static bool IsMessageOwnedArena(Arena* arena) { + return arena->IsMessageOwned(); + } + + private: + static Arena* GetArenaForAllocationInternal( + const T* p, std::true_type /*is_derived_from*/) { + return p->GetArenaForAllocation(); + } + + static Arena* GetArenaForAllocationInternal( + const T* p, std::false_type /*is_derived_from*/) { + return GetArenaForAllocationForNonMessage( + p, typename is_arena_constructable::type()); + } + + static Arena* GetArenaForAllocationForNonMessage( + const T* p, std::true_type /*is_arena_constructible*/) { + return p->GetArena(); + } + + static Arena* GetArenaForAllocationForNonMessage( + const T* p, std::false_type /*is_arena_constructible*/) { + return GetArenaForAllocationForNonMessageNonArenaConstructible( + p, typename has_get_arena::type()); + } + + static Arena* GetArenaForAllocationForNonMessageNonArenaConstructible( + const T* p, std::true_type /*has_get_arena*/) { + return p->GetArena(); + } + + static Arena* GetArenaForAllocationForNonMessageNonArenaConstructible( + const T* /* p */, std::false_type /*has_get_arena*/) { + return nullptr; + } + template static char DestructorSkippable(const typename U::DestructorSkippable_*); template static double DestructorSkippable(...); - typedef google::protobuf::internal::integral_constant< + typedef std::integral_constant< bool, sizeof(DestructorSkippable(static_cast(0))) == sizeof(char) || - google::protobuf::internal::has_trivial_destructor::value> + std::is_trivially_destructible::value> is_destructor_skippable; - template + template static char ArenaConstructable( const typename U::InternalArenaConstructable_*); - template + template static double ArenaConstructable(...); - typedef google::protobuf::internal::integral_constant( - static_cast(0))) == - sizeof(char)> + typedef std::integral_constant( + static_cast(0))) == + sizeof(char)> is_arena_constructable; -#if LANG_CXX11 + template () + .GetArena())>::value, + int>::type = 0> + static char HasGetArena(decltype(&U::GetArena)); + template + static double HasGetArena(...); + + typedef std::integral_constant(nullptr)) == + sizeof(char)> + has_get_arena; + template static T* Construct(void* ptr, Args&&... args) { - return new (ptr) T(std::forward(args)...); + return new (ptr) T(static_cast(args)...); } -#else - template - static T* Construct(void* ptr, const Arg1& arg1) { - return new (ptr) T(arg1); - } - template - static T* Construct(void* ptr, const Arg1& arg1, const Arg2& arg2) { - return new (ptr) T(arg1, arg2); - } - template - static T* Construct(void* ptr, const Arg1& arg1, - const Arg2& arg2, const Arg3& arg3) { - return new (ptr) T(arg1, arg2, arg3); - } -#endif // LANG_CXX11 - static Arena* GetArena(const T* p) { return p->GetArenaNoVirtual(); } + static inline PROTOBUF_ALWAYS_INLINE T* New() { + return new T(nullptr); + } + + static Arena* GetArena(const T* p) { return p->GetArena(); } friend class Arena; + friend class TestUtil::ReflectionTester; }; - // Helper typetrait that indicates support for arenas in a type T at compile + // Helper typetraits that indicates support for arenas in a type T at compile // time. This is public only to allow construction of higher-level templated - // utilities. is_arena_constructable::value is true if the message type T - // has arena support enabled, and false otherwise. + // utilities. + // + // is_arena_constructable::value is true if the message type T has arena + // support enabled, and false otherwise. + // + // is_destructor_skippable::value is true if the message type T has told + // the arena that it is safe to skip the destructor, and false otherwise. // // This is inside Arena because only Arena has the friend relationships // necessary to see the underlying generated code traits. template struct is_arena_constructable : InternalHelper::is_arena_constructable {}; + template + struct is_destructor_skippable : InternalHelper::is_destructor_skippable { + }; private: - void CallDestructorHooks(); - void OnArenaAllocation(const std::type_info* allocated_type, size_t n) const; - inline void AllocHook(const std::type_info* allocated_type, size_t n) const { - if (GOOGLE_PREDICT_FALSE(hooks_cookie_ != NULL)) { - OnArenaAllocation(allocated_type, n); + internal::ThreadSafeArena impl_; + + template + struct has_get_arena : InternalHelper::has_get_arena {}; + + // Constructor solely used by message-owned arena. + inline Arena(internal::MessageOwned) : impl_(internal::MessageOwned{}) {} + + // Checks whether this arena is message-owned. + PROTOBUF_ALWAYS_INLINE bool IsMessageOwned() const { + return impl_.IsMessageOwned(); + } + + template + PROTOBUF_NDEBUG_INLINE static T* CreateMessageInternal(Arena* arena, + Args&&... args) { + static_assert( + InternalHelper::is_arena_constructable::value, + "CreateMessage can only construct types that are ArenaConstructable"); + if (arena == NULL) { + return new T(nullptr, static_cast(args)...); + } else { + return arena->DoCreateMessage(static_cast(args)...); } } - // Allocate and also optionally call on_arena_allocation callback with the - // allocated type info when the hooks are in place in ArenaOptions and - // the cookie is not null. - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - void* AllocateInternal(bool skip_explicit_ownership) { - const size_t n = internal::AlignUpTo8(sizeof(T)); - AllocHook(RTTI_TYPE_ID(T), n); - // Monitor allocation if needed. - if (skip_explicit_ownership) { - return impl_.AllocateAligned(n); + // This specialization for no arguments is necessary, because its behavior is + // slightly different. When the arena pointer is nullptr, it calls T() + // instead of T(nullptr). + template + PROTOBUF_NDEBUG_INLINE static T* CreateMessageInternal(Arena* arena) { + static_assert( + InternalHelper::is_arena_constructable::value, + "CreateMessage can only construct types that are ArenaConstructable"); + if (arena == NULL) { + // Generated arena constructor T(Arena*) is protected. Call via + // InternalHelper. + return InternalHelper::New(); } else { - return impl_.AllocateAlignedAndAddCleanup( - n, &internal::arena_destruct_object); + return arena->DoCreateMessage(); + } + } + + // Allocate and also optionally call collector with the allocated type info + // when allocation recording is enabled. + PROTOBUF_NDEBUG_INLINE void* AllocateInternal(size_t size, size_t align, + void (*destructor)(void*), + const std::type_info* type) { + // Monitor allocation if needed. + if (destructor == nullptr) { + return AllocateAlignedWithHook(size, align, type); + } else { + if (align <= 8) { + auto res = AllocateAlignedWithCleanup(internal::AlignUpTo8(size), type); + res.second->elem = res.first; + res.second->cleanup = destructor; + return res.first; + } else { + auto res = AllocateAlignedWithCleanup(size + align - 8, type); + auto ptr = internal::AlignTo(res.first, align); + res.second->elem = ptr; + res.second->cleanup = destructor; + return ptr; + } } } @@ -666,218 +591,132 @@ class LIBPROTOBUF_EXPORT Arena { // as it can cause confusing API usages, and end up having double free in // user code. These are used only internally from LazyField and Repeated // fields, since they are designed to work in all mode combinations. - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static Msg* CreateMaybeMessage(Arena* arena, google::protobuf::internal::true_type) { - return CreateMessage(arena); + template + PROTOBUF_ALWAYS_INLINE static Msg* DoCreateMaybeMessage(Arena* arena, + std::true_type, + Args&&... args) { + return CreateMessageInternal(arena, std::forward(args)...); } - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* CreateMaybeMessage(Arena* arena, google::protobuf::internal::false_type) { - return Create(arena); + template + PROTOBUF_ALWAYS_INLINE static T* DoCreateMaybeMessage(Arena* arena, + std::false_type, + Args&&... args) { + return Create(arena, std::forward(args)...); } - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* CreateMaybeMessage(Arena* arena) { - return CreateMaybeMessage(arena, is_arena_constructable()); + template + PROTOBUF_ALWAYS_INLINE static T* CreateMaybeMessage(Arena* arena, + Args&&... args) { + return DoCreateMaybeMessage(arena, is_arena_constructable(), + std::forward(args)...); } // Just allocate the required size for the given type assuming the // type has a trivial constructor. - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - T* CreateInternalRawArray(size_t num_elements) { - GOOGLE_CHECK_LE(num_elements, - std::numeric_limits::max() / sizeof(T)) - << "Requested size is too large to fit into size_t."; - const size_t n = internal::AlignUpTo8(sizeof(T) * num_elements); - // Monitor allocation if needed. - AllocHook(RTTI_TYPE_ID(T), n); - return static_cast(impl_.AllocateAligned(n)); - } - -#if LANG_CXX11 - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - T* CreateInternal(bool skip_explicit_ownership, Args&&... args) { - return new (AllocateInternal(skip_explicit_ownership)) - T(std::forward(args)...); - } -#else - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - T* CreateInternal(bool skip_explicit_ownership) { - return new (AllocateInternal(skip_explicit_ownership)) T(); - } - - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - T* CreateInternal(bool skip_explicit_ownership, const Arg& arg) { - return new (AllocateInternal(skip_explicit_ownership)) T(arg); - } - - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - T* CreateInternal(bool skip_explicit_ownership, - const Arg1& arg1, - const Arg2& arg2) { - return new (AllocateInternal(skip_explicit_ownership)) T(arg1, arg2); - } - - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - T* CreateInternal(bool skip_explicit_ownership, - const Arg1& arg1, - const Arg2& arg2, - const Arg3& arg3) { - return new (AllocateInternal(skip_explicit_ownership)) - T(arg1, arg2, arg3); - } - - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - T* CreateInternal(bool skip_explicit_ownership, - const Arg1& arg1, - const Arg2& arg2, - const Arg3& arg3, - const Arg4& arg4) { - return new (AllocateInternal(skip_explicit_ownership)) - T(arg1, arg2, arg3, arg4); - } - - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - T* CreateInternal(bool skip_explicit_ownership, - const Arg1& arg1, - const Arg2& arg2, - const Arg3& arg3, - const Arg4& arg4, - const Arg5& arg5) { - return new (AllocateInternal(skip_explicit_ownership)) - T(arg1, arg2, arg3, arg4, arg5); - } - - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - T* CreateInternal(bool skip_explicit_ownership, - const Arg1& arg1, - const Arg2& arg2, - const Arg3& arg3, - const Arg4& arg4, - const Arg5& arg5, - const Arg6& arg6) { - return new (AllocateInternal(skip_explicit_ownership)) - T(arg1, arg2, arg3, arg4, arg5, arg6); - } - - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - T* CreateInternal(bool skip_explicit_ownership, - const Arg1& arg1, - const Arg2& arg2, - const Arg3& arg3, - const Arg4& arg4, - const Arg5& arg5, - const Arg6& arg6, - const Arg7& arg7) { - return new (AllocateInternal(skip_explicit_ownership)) - T(arg1, arg2, arg3, arg4, arg5, arg6, arg7); - } - - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - T* CreateInternal(bool skip_explicit_ownership, - const Arg1& arg1, - const Arg2& arg2, - const Arg3& arg3, - const Arg4& arg4, - const Arg5& arg5, - const Arg6& arg6, - const Arg7& arg7, - const Arg8& arg8) { - return new (AllocateInternal(skip_explicit_ownership)) - T(arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8); - } -#endif -#if LANG_CXX11 - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE T* CreateMessageInternal( - Args&&... args) { - return InternalHelper::Construct( - AllocateInternal(InternalHelper::is_destructor_skippable::value), - this, std::forward(args)...); - } -#endif template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE T* CreateMessageInternal() { - return InternalHelper::Construct( - AllocateInternal(InternalHelper::is_destructor_skippable::value), - this); + PROTOBUF_NDEBUG_INLINE T* CreateInternalRawArray(size_t num_elements) { + GOOGLE_CHECK_LE(num_elements, std::numeric_limits::max() / sizeof(T)) + << "Requested size is too large to fit into size_t."; + // We count on compiler to realize that if sizeof(T) is a multiple of + // 8 AlignUpTo can be elided. + const size_t n = sizeof(T) * num_elements; + return static_cast( + AllocateAlignedWithHook(n, alignof(T), RTTI_TYPE_ID(T))); } - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - T* CreateMessageInternal(const Arg& arg) { + template + PROTOBUF_NDEBUG_INLINE T* DoCreateMessage(Args&&... args) { return InternalHelper::Construct( - AllocateInternal(InternalHelper::is_destructor_skippable::value), - this, arg); - } - - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - T* CreateMessageInternal(const Arg1& arg1, const Arg2& arg2) { - return InternalHelper::Construct( - AllocateInternal(InternalHelper::is_destructor_skippable::value), - this, arg1, arg2); + AllocateInternal(sizeof(T), alignof(T), + internal::ObjectDestructor< + InternalHelper::is_destructor_skippable::value, + T>::destructor, + RTTI_TYPE_ID(T)), + this, std::forward(args)...); } // CreateInArenaStorage is used to implement map field. Without it, - // google::protobuf::Map need to call generated message's protected arena constructor, - // which needs to declare google::protobuf::Map as friend of generated message. - template - static void CreateInArenaStorage(T* ptr, Arena* arena) { + // Map need to call generated message's protected arena constructor, + // which needs to declare Map as friend of generated message. + template + static void CreateInArenaStorage(T* ptr, Arena* arena, Args&&... args) { CreateInArenaStorageInternal(ptr, arena, - typename is_arena_constructable::type()); - RegisterDestructorInternal( - ptr, arena, - typename InternalHelper::is_destructor_skippable::type()); + typename is_arena_constructable::type(), + std::forward(args)...); + if (arena != nullptr) { + RegisterDestructorInternal( + ptr, arena, + typename InternalHelper::is_destructor_skippable::type()); + } + } + + template + static void CreateInArenaStorageInternal(T* ptr, Arena* arena, + std::true_type, Args&&... args) { + InternalHelper::Construct(ptr, arena, std::forward(args)...); + } + template + static void CreateInArenaStorageInternal(T* ptr, Arena* /* arena */, + std::false_type, Args&&... args) { + new (ptr) T(std::forward(args)...); } template - static void CreateInArenaStorageInternal( - T* ptr, Arena* arena, google::protobuf::internal::true_type) { - InternalHelper::Construct(ptr, arena); - } + static void RegisterDestructorInternal(T* /* ptr */, Arena* /* arena */, + std::true_type) {} template - static void CreateInArenaStorageInternal( - T* ptr, Arena* /* arena */, google::protobuf::internal::false_type) { - new (ptr) T(); - } - - template - static void RegisterDestructorInternal( - T* /* ptr */, Arena* /* arena */, google::protobuf::internal::true_type) {} - template - static void RegisterDestructorInternal( - T* ptr, Arena* arena, google::protobuf::internal::false_type) { + static void RegisterDestructorInternal(T* ptr, Arena* arena, + std::false_type) { arena->OwnDestructor(ptr); } + // These implement Create(). The second parameter has type 'true_type' if T is + // a subtype of Message and 'false_type' otherwise. + template + PROTOBUF_ALWAYS_INLINE static T* CreateInternal(Arena* arena, std::true_type, + Args&&... args) { + if (arena == nullptr) { + return new T(std::forward(args)...); + } else { + auto destructor = + internal::ObjectDestructor::value, + T>::destructor; + T* result = + new (arena->AllocateInternal(sizeof(T), alignof(T), destructor, + RTTI_TYPE_ID(T))) + T(std::forward(args)...); + return result; + } + } + template + PROTOBUF_ALWAYS_INLINE static T* CreateInternal(Arena* arena, std::false_type, + Args&&... args) { + if (arena == nullptr) { + return new T(std::forward(args)...); + } else { + auto destructor = + internal::ObjectDestructor::value, + T>::destructor; + return new (arena->AllocateInternal(sizeof(T), alignof(T), destructor, + RTTI_TYPE_ID(T))) + T(std::forward(args)...); + } + } + // These implement Own(), which registers an object for deletion (destructor // call and operator delete()). The second parameter has type 'true_type' if T - // is a subtype of ::google::protobuf::Message and 'false_type' otherwise. Collapsing + // is a subtype of Message and 'false_type' otherwise. Collapsing // all template instantiations to one for generic Message reduces code size, // using the virtual destructor instead. - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - void OwnInternal(T* object, google::protobuf::internal::true_type) { + template + PROTOBUF_ALWAYS_INLINE void OwnInternal(T* object, std::true_type) { if (object != NULL) { - impl_.AddCleanup(object, - &internal::arena_delete_object< ::google::protobuf::Message>); + impl_.AddCleanup(object, &internal::arena_delete_object); } } - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - void OwnInternal(T* object, google::protobuf::internal::false_type) { + template + PROTOBUF_ALWAYS_INLINE void OwnInternal(T* object, std::false_type) { if (object != NULL) { impl_.AddCleanup(object, &internal::arena_delete_object); } @@ -885,42 +724,88 @@ class LIBPROTOBUF_EXPORT Arena { // Implementation for GetArena(). Only message objects with // InternalArenaConstructable_ tags can be associated with an arena, and such - // objects must implement a GetArenaNoVirtual() method. - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static ::google::protobuf::Arena* GetArenaInternal( - const T* value, google::protobuf::internal::true_type) { + // objects must implement a GetArena() method. + template ::value, int>::type = 0> + PROTOBUF_ALWAYS_INLINE static Arena* GetArenaInternal(const T* value) { return InternalHelper::GetArena(value); } + template ::value && + has_get_arena::value, + int>::type = 0> + PROTOBUF_ALWAYS_INLINE static Arena* GetArenaInternal(const T* value) { + return value->GetArena(); + } + template ::value && + !has_get_arena::value, + int>::type = 0> + PROTOBUF_ALWAYS_INLINE static Arena* GetArenaInternal(const T* value) { + (void)value; + return nullptr; + } template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static ::google::protobuf::Arena* GetArenaInternal( - const T* /* value */, google::protobuf::internal::false_type) { - return NULL; + PROTOBUF_ALWAYS_INLINE static Arena* GetOwningArena(const T* value) { + return GetOwningArenaInternal( + value, std::is_convertible()); + } + + // Implementation for GetOwningArena(). All and only message objects have + // GetOwningArena() method. + template + PROTOBUF_ALWAYS_INLINE static Arena* GetOwningArenaInternal( + const T* value, std::true_type) { + return InternalHelper::GetOwningArena(value); + } + template + PROTOBUF_ALWAYS_INLINE static Arena* GetOwningArenaInternal( + const T* /* value */, std::false_type) { + return nullptr; } // For friends of arena. - void* AllocateAligned(size_t n) { - AllocHook(NULL, n); - return impl_.AllocateAligned(internal::AlignUpTo8(n)); + void* AllocateAligned(size_t n, size_t align = 8) { + if (align <= 8) { + return AllocateAlignedNoHook(internal::AlignUpTo8(n)); + } else { + // We are wasting space by over allocating align - 8 bytes. Compared + // to a dedicated function that takes current alignment in consideration. + // Such a scheme would only waste (align - 8)/2 bytes on average, but + // requires a dedicated function in the outline arena allocation + // functions. Possibly re-evaluate tradeoffs later. + return internal::AlignTo(AllocateAlignedNoHook(n + align - 8), align); + } } - internal::ArenaImpl impl_; + void* AllocateAlignedWithHook(size_t n, size_t align, + const std::type_info* type) { + if (align <= 8) { + return AllocateAlignedWithHook(internal::AlignUpTo8(n), type); + } else { + // We are wasting space by over allocating align - 8 bytes. Compared + // to a dedicated function that takes current alignment in consideration. + // Such a schemee would only waste (align - 8)/2 bytes on average, but + // requires a dedicated function in the outline arena allocation + // functions. Possibly re-evaluate tradeoffs later. + return internal::AlignTo(AllocateAlignedWithHook(n + align - 8, type), + align); + } + } - void* (*on_arena_init_)(Arena* arena); - void (*on_arena_allocation_)(const std::type_info* allocated_type, - uint64 alloc_size, void* cookie); - void (*on_arena_reset_)(Arena* arena, void* cookie, uint64 space_used); - void (*on_arena_destruction_)(Arena* arena, void* cookie, uint64 space_used); - - // The arena may save a cookie it receives from the external on_init hook - // and then use it when calling the on_reset and on_destruction hooks. - void* hooks_cookie_; + void* AllocateAlignedNoHook(size_t n); + void* AllocateAlignedWithHook(size_t n, const std::type_info* type); + std::pair + AllocateAlignedWithCleanup(size_t n, const std::type_info* type); template - friend class ::google::protobuf::internal::GenericTypeHandler; + friend class internal::GenericTypeHandler; friend struct internal::ArenaStringPtr; // For AllocateAligned. - friend class internal::LazyField; // For CreateMaybeMessage. + friend class internal::InlinedStringField; // For AllocateAligned. + friend class internal::LazyField; // For CreateMaybeMessage. + friend class internal::EpsCopyInputStream; // For parser performance + friend class MessageLite; template friend class Map; }; @@ -929,6 +814,8 @@ class LIBPROTOBUF_EXPORT Arena { #undef RTTI_TYPE_ID } // namespace protobuf - } // namespace google + +#include + #endif // GOOGLE_PROTOBUF_ARENA_H__ diff --git a/3rdparty/protobuf/src/google/protobuf/arena_impl.h b/3rdparty/protobuf/src/google/protobuf/arena_impl.h index 6cc7096bfb..2ffac319b4 100644 --- a/3rdparty/protobuf/src/google/protobuf/arena_impl.h +++ b/3rdparty/protobuf/src/google/protobuf/arena_impl.h @@ -33,85 +33,168 @@ #ifndef GOOGLE_PROTOBUF_ARENA_IMPL_H__ #define GOOGLE_PROTOBUF_ARENA_IMPL_H__ +#include #include +#include -#include -#include #include #include -#include -#include -#include +#ifdef ADDRESS_SANITIZER +#include +#endif // ADDRESS_SANITIZER + +#include + namespace google { - namespace protobuf { namespace internal { -inline size_t AlignUpTo8(size_t n) { +inline constexpr size_t AlignUpTo8(size_t n) { // Align n to next multiple of 8 (from Hacker's Delight, Chapter 3.) - return (n + 7) & -8; + return (n + 7) & static_cast(-8); } -// This class provides the core Arena memory allocation library. Different -// implementations only need to implement the public interface below. -// Arena is not a template type as that would only be useful if all protos -// in turn would be templates, which will/cannot happen. However separating -// the memory allocation part from the cruft of the API users expect we can -// use #ifdef the select the best implementation based on hardware / OS. -class LIBPROTOBUF_EXPORT ArenaImpl { +using LifecycleIdAtomic = uint64_t; + +// MetricsCollector collects stats for a particular arena. +class PROTOBUF_EXPORT ArenaMetricsCollector { public: - struct Options { - size_t start_block_size; - size_t max_block_size; - char* initial_block; - size_t initial_block_size; - void* (*block_alloc)(size_t); - void (*block_dealloc)(void*, size_t); + ArenaMetricsCollector(bool record_allocs) : record_allocs_(record_allocs) {} - template - explicit Options(const O& options) - : start_block_size(options.start_block_size), - max_block_size(options.max_block_size), - initial_block(options.initial_block), - initial_block_size(options.initial_block_size), - block_alloc(options.block_alloc), - block_dealloc(options.block_dealloc) {} - }; + // Invoked when the arena is about to be destroyed. This method will + // typically finalize any metric collection and delete the collector. + // space_allocated is the space used by the arena. + virtual void OnDestroy(uint64_t space_allocated) = 0; - template - explicit ArenaImpl(const O& options) : options_(options) { - if (options_.initial_block != NULL && options_.initial_block_size > 0) { - GOOGLE_CHECK_GE(options_.initial_block_size, sizeof(Block)) - << ": Initial block size too small for header."; - initial_block_ = reinterpret_cast(options_.initial_block); - } else { - initial_block_ = NULL; - } + // OnReset() is called when the associated arena is reset. + // space_allocated is the space used by the arena just before the reset. + virtual void OnReset(uint64_t space_allocated) = 0; - Init(); + // OnAlloc is called when an allocation happens. + // type_info is promised to be static - its lifetime extends to + // match program's lifetime (It is given by typeid operator). + // Note: typeid(void) will be passed as allocated_type every time we + // intentionally want to avoid monitoring an allocation. (i.e. internal + // allocations for managing the arena) + virtual void OnAlloc(const std::type_info* allocated_type, + uint64_t alloc_size) = 0; + + // Does OnAlloc() need to be called? If false, metric collection overhead + // will be reduced since we will not do extra work per allocation. + bool RecordAllocs() { return record_allocs_; } + + protected: + // This class is destructed by the call to OnDestroy(). + ~ArenaMetricsCollector() = default; + const bool record_allocs_; +}; + +struct AllocationPolicy { + static constexpr size_t kDefaultStartBlockSize = 256; + static constexpr size_t kDefaultMaxBlockSize = 8192; + + size_t start_block_size = kDefaultStartBlockSize; + size_t max_block_size = kDefaultMaxBlockSize; + void* (*block_alloc)(size_t) = nullptr; + void (*block_dealloc)(void*, size_t) = nullptr; + ArenaMetricsCollector* metrics_collector = nullptr; + + bool IsDefault() const { + return start_block_size == kDefaultMaxBlockSize && + max_block_size == kDefaultMaxBlockSize && block_alloc == nullptr && + block_dealloc == nullptr && metrics_collector == nullptr; + } +}; + +// Tagged pointer to an AllocationPolicy. +class TaggedAllocationPolicyPtr { + public: + constexpr TaggedAllocationPolicyPtr() : policy_(0) {} + + explicit TaggedAllocationPolicyPtr(AllocationPolicy* policy) + : policy_(reinterpret_cast(policy)) {} + + void set_policy(AllocationPolicy* policy) { + auto bits = policy_ & kTagsMask; + policy_ = reinterpret_cast(policy) | bits; } - // Destructor deletes all owned heap allocated objects, and destructs objects - // that have non-trivial destructors, except for proto2 message objects whose - // destructors can be skipped. Also, frees all blocks except the initial block - // if it was passed in. - ~ArenaImpl(); + AllocationPolicy* get() { + return reinterpret_cast(policy_ & kPtrMask); + } + const AllocationPolicy* get() const { + return reinterpret_cast(policy_ & kPtrMask); + } - uint64 Reset(); + AllocationPolicy& operator*() { return *get(); } + const AllocationPolicy& operator*() const { return *get(); } - uint64 SpaceAllocated() const; - uint64 SpaceUsed() const; + AllocationPolicy* operator->() { return get(); } + const AllocationPolicy* operator->() const { return get(); } - void* AllocateAligned(size_t n); + bool is_user_owned_initial_block() const { + return static_cast(get_mask()); + } + void set_is_user_owned_initial_block(bool v) { + set_mask(v); + } - void* AllocateAlignedAndAddCleanup(size_t n, void (*cleanup)(void*)); + bool should_record_allocs() const { + return static_cast(get_mask()); + } + void set_should_record_allocs(bool v) { set_mask(v); } - // Add object pointer and cleanup function pointer to the list. - void AddCleanup(void* elem, void (*cleanup)(void*)); + uintptr_t get_raw() const { return policy_; } + + inline void RecordAlloc(const std::type_info* allocated_type, + size_t n) const { + get()->metrics_collector->OnAlloc(allocated_type, n); + } private: + enum : uintptr_t { + kUserOwnedInitialBlock = 1, + kRecordAllocs = 2, + }; + + static constexpr uintptr_t kTagsMask = 7; + static constexpr uintptr_t kPtrMask = ~kTagsMask; + + template + uintptr_t get_mask() const { + return policy_ & kMask; + } + template + void set_mask(bool v) { + if (v) { + policy_ |= kMask; + } else { + policy_ &= ~kMask; + } + } + uintptr_t policy_; +}; + +// A simple arena allocator. Calls to allocate functions must be properly +// serialized by the caller, hence this class cannot be used as a general +// purpose allocator in a multi-threaded program. It serves as a building block +// for ThreadSafeArena, which provides a thread-safe arena allocator. +// +// This class manages +// 1) Arena bump allocation + owning memory blocks. +// 2) Maintaining a cleanup list. +// It delagetes the actual memory allocation back to ThreadSafeArena, which +// contains the information on block growth policy and backing memory allocation +// used. +class PROTOBUF_EXPORT SerialArena { + public: + struct Memory { + void* ptr; + size_t size; + }; + // Node contains the ptr of the object to be cleaned up and the associated // cleanup function ptr. struct CleanupNode { @@ -119,124 +202,361 @@ class LIBPROTOBUF_EXPORT ArenaImpl { void (*cleanup)(void*); // Function pointer to the destructor or deleter. }; - // Cleanup uses a chunked linked list, to reduce pointer chasing. - struct CleanupChunk { - static size_t SizeOf(size_t i) { - return sizeof(CleanupChunk) + (sizeof(CleanupNode) * (i - 1)); + void CleanupList(); + uint64_t SpaceAllocated() const { + return space_allocated_.load(std::memory_order_relaxed); + } + uint64_t SpaceUsed() const; + + bool HasSpace(size_t n) { return n <= static_cast(limit_ - ptr_); } + + void* AllocateAligned(size_t n, const AllocationPolicy* policy) { + GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n); // Must be already aligned. + GOOGLE_DCHECK_GE(limit_, ptr_); + if (PROTOBUF_PREDICT_FALSE(!HasSpace(n))) { + return AllocateAlignedFallback(n, policy); } - size_t len; // Number of elements currently present. - size_t size; // Total elements in the list. - CleanupChunk* next; // Next node in the list. - CleanupNode nodes[1]; // True length is |size|. - }; + return AllocateFromExisting(n); + } - struct Block; + private: + void* AllocateFromExisting(size_t n) { + void* ret = ptr_; + ptr_ += n; +#ifdef ADDRESS_SANITIZER + ASAN_UNPOISON_MEMORY_REGION(ret, n); +#endif // ADDRESS_SANITIZER + return ret; + } - // Tracks per-thread info. ThreadInfos are kept in a linked list. - struct ThreadInfo { - void *owner; // &ThreadCache of this thread; - Block* head; // Head of linked list of blocks. - CleanupChunk* cleanup; // Head of cleanup list. - ThreadInfo* next; // Next ThreadInfo in this linked list. - }; + public: + // Allocate space if the current region provides enough space. + bool MaybeAllocateAligned(size_t n, void** out) { + GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n); // Must be already aligned. + GOOGLE_DCHECK_GE(limit_, ptr_); + if (PROTOBUF_PREDICT_FALSE(!HasSpace(n))) return false; + *out = AllocateFromExisting(n); + return true; + } + + std::pair AllocateAlignedWithCleanup( + size_t n, const AllocationPolicy* policy) { + GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n); // Must be already aligned. + if (PROTOBUF_PREDICT_FALSE(!HasSpace(n + kCleanupSize))) { + return AllocateAlignedWithCleanupFallback(n, policy); + } + return AllocateFromExistingWithCleanupFallback(n); + } + + private: + std::pair AllocateFromExistingWithCleanupFallback( + size_t n) { + void* ret = ptr_; + ptr_ += n; + limit_ -= kCleanupSize; +#ifdef ADDRESS_SANITIZER + ASAN_UNPOISON_MEMORY_REGION(ret, n); + ASAN_UNPOISON_MEMORY_REGION(limit_, kCleanupSize); +#endif // ADDRESS_SANITIZER + return CreatePair(ret, reinterpret_cast(limit_)); + } + + public: + void AddCleanup(void* elem, void (*cleanup)(void*), + const AllocationPolicy* policy) { + auto res = AllocateAlignedWithCleanup(0, policy); + res.second->elem = elem; + res.second->cleanup = cleanup; + } + + void* owner() const { return owner_; } + SerialArena* next() const { return next_; } + void set_next(SerialArena* next) { next_ = next; } + + private: + friend class ThreadSafeArena; + friend class ArenaBenchmark; + + // Creates a new SerialArena inside mem using the remaining memory as for + // future allocations. + static SerialArena* New(SerialArena::Memory mem, void* owner); + // Free SerialArena returning the memory passed in to New + template + Memory Free(Deallocator deallocator); // Blocks are variable length malloc-ed objects. The following structure // describes the common header for all blocks. struct Block { - void* owner; // &ThreadCache of thread that owns this block. - ThreadInfo* thread_info; // ThreadInfo of thread that owns this block. - Block* next; // Next block in arena (may have different owner) - // ((char*) &block) + pos is next available byte. It is always - // aligned at a multiple of 8 bytes. - size_t pos; - size_t size; // total size of the block. - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - size_t avail() const { return size - pos; } + Block(Block* next, size_t size) : next(next), size(size), start(nullptr) {} + + char* Pointer(size_t n) { + GOOGLE_DCHECK(n <= size); + return reinterpret_cast(this) + n; + } + + Block* const next; + const size_t size; + CleanupNode* start; // data follows }; - struct ThreadCache { + void* owner_; // &ThreadCache of this thread; + Block* head_; // Head of linked list of blocks. + SerialArena* next_; // Next SerialArena in this linked list. + size_t space_used_ = 0; // Necessary for metrics. + std::atomic space_allocated_; + + // Next pointer to allocate from. Always 8-byte aligned. Points inside + // head_ (and head_->pos will always be non-canonical). We keep these + // here to reduce indirection. + char* ptr_; + char* limit_; + + // Constructor is private as only New() should be used. + inline SerialArena(Block* b, void* owner); + void* AllocateAlignedFallback(size_t n, const AllocationPolicy* policy); + std::pair AllocateAlignedWithCleanupFallback( + size_t n, const AllocationPolicy* policy); + void AllocateNewBlock(size_t n, const AllocationPolicy* policy); + + std::pair CreatePair(void* ptr, CleanupNode* node) { + return {ptr, node}; + } + + public: + static constexpr size_t kBlockHeaderSize = AlignUpTo8(sizeof(Block)); + static constexpr size_t kCleanupSize = AlignUpTo8(sizeof(CleanupNode)); +}; + +// Tag type used to invoke the constructor of message-owned arena. +// Only message-owned arenas use this constructor for creation. +// Such constructors are internal implementation details of the library. +struct MessageOwned { + explicit MessageOwned() = default; +}; + +// This class provides the core Arena memory allocation library. Different +// implementations only need to implement the public interface below. +// Arena is not a template type as that would only be useful if all protos +// in turn would be templates, which will/cannot happen. However separating +// the memory allocation part from the cruft of the API users expect we can +// use #ifdef the select the best implementation based on hardware / OS. +class PROTOBUF_EXPORT ThreadSafeArena { + public: + ThreadSafeArena() { Init(); } + + // Constructor solely used by message-owned arena. + ThreadSafeArena(internal::MessageOwned) : tag_and_id_(kMessageOwnedArena) { + Init(); + } + + ThreadSafeArena(char* mem, size_t size) { InitializeFrom(mem, size); } + + explicit ThreadSafeArena(void* mem, size_t size, + const AllocationPolicy& policy) { + InitializeWithPolicy(mem, size, policy); + } + + // Destructor deletes all owned heap allocated objects, and destructs objects + // that have non-trivial destructors, except for proto2 message objects whose + // destructors can be skipped. Also, frees all blocks except the initial block + // if it was passed in. + ~ThreadSafeArena(); + + uint64_t Reset(); + + uint64_t SpaceAllocated() const; + uint64_t SpaceUsed() const; + + void* AllocateAligned(size_t n, const std::type_info* type) { + SerialArena* arena; + if (PROTOBUF_PREDICT_TRUE(!alloc_policy_.should_record_allocs() && + GetSerialArenaFast(&arena))) { + return arena->AllocateAligned(n, AllocPolicy()); + } else { + return AllocateAlignedFallback(n, type); + } + } + + // This function allocates n bytes if the common happy case is true and + // returns true. Otherwise does nothing and returns false. This strange + // semantics is necessary to allow callers to program functions that only + // have fallback function calls in tail position. This substantially improves + // code for the happy path. + PROTOBUF_NDEBUG_INLINE bool MaybeAllocateAligned(size_t n, void** out) { + SerialArena* a; + if (PROTOBUF_PREDICT_TRUE(!alloc_policy_.should_record_allocs() && + GetSerialArenaFromThreadCache(&a))) { + return a->MaybeAllocateAligned(n, out); + } + return false; + } + + std::pair AllocateAlignedWithCleanup( + size_t n, const std::type_info* type); + + // Add object pointer and cleanup function pointer to the list. + void AddCleanup(void* elem, void (*cleanup)(void*)); + + // Checks whether this arena is message-owned. + PROTOBUF_ALWAYS_INLINE bool IsMessageOwned() const { + return tag_and_id_ & kMessageOwnedArena; + } + + private: + // Unique for each arena. Changes on Reset(). + uint64_t tag_and_id_ = 0; + // The LSB of tag_and_id_ indicates if the arena is message-owned. + enum : uint64_t { kMessageOwnedArena = 1 }; + + TaggedAllocationPolicyPtr alloc_policy_; // Tagged pointer to AllocPolicy. + + // Pointer to a linked list of SerialArena. + std::atomic threads_; + std::atomic hint_; // Fast thread-local block access + + const AllocationPolicy* AllocPolicy() const { return alloc_policy_.get(); } + void InitializeFrom(void* mem, size_t size); + void InitializeWithPolicy(void* mem, size_t size, AllocationPolicy policy); + void* AllocateAlignedFallback(size_t n, const std::type_info* type); + std::pair + AllocateAlignedWithCleanupFallback(size_t n, const std::type_info* type); + + void Init(); + void SetInitialBlock(void* mem, size_t size); + + // Delete or Destruct all objects owned by the arena. + void CleanupList(); + + inline uint64_t LifeCycleId() const { + return tag_and_id_ & ~kMessageOwnedArena; + } + + inline void CacheSerialArena(SerialArena* serial) { + thread_cache().last_serial_arena = serial; + thread_cache().last_lifecycle_id_seen = tag_and_id_; + // TODO(haberman): evaluate whether we would gain efficiency by getting rid + // of hint_. It's the only write we do to ThreadSafeArena in the allocation + // path, which will dirty the cache line. + + hint_.store(serial, std::memory_order_release); + } + + PROTOBUF_NDEBUG_INLINE bool GetSerialArenaFast(SerialArena** arena) { + if (GetSerialArenaFromThreadCache(arena)) return true; + + // Check whether we own the last accessed SerialArena on this arena. This + // fast path optimizes the case where a single thread uses multiple arenas. + ThreadCache* tc = &thread_cache(); + SerialArena* serial = hint_.load(std::memory_order_acquire); + if (PROTOBUF_PREDICT_TRUE(serial != NULL && serial->owner() == tc)) { + *arena = serial; + return true; + } + return false; + } + + PROTOBUF_NDEBUG_INLINE bool GetSerialArenaFromThreadCache( + SerialArena** arena) { + // If this thread already owns a block in this arena then try to use that. + // This fast path optimizes the case where multiple threads allocate from + // the same arena. + ThreadCache* tc = &thread_cache(); + if (PROTOBUF_PREDICT_TRUE(tc->last_lifecycle_id_seen == tag_and_id_)) { + *arena = tc->last_serial_arena; + return true; + } + return false; + } + SerialArena* GetSerialArenaFallback(void* me); + + template + void PerSerialArena(Functor fn) { + // By omitting an Acquire barrier we ensure that any user code that doesn't + // properly synchronize Reset() or the destructor will throw a TSAN warning. + SerialArena* serial = threads_.load(std::memory_order_relaxed); + + for (; serial; serial = serial->next()) fn(serial); + } + + // Releases all memory except the first block which it returns. The first + // block might be owned by the user and thus need some extra checks before + // deleting. + SerialArena::Memory Free(size_t* space_allocated); + +#ifdef _MSC_VER +#pragma warning(disable : 4324) +#endif + struct alignas(64) ThreadCache { #if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL) // If we are using the ThreadLocalStorage class to store the ThreadCache, // then the ThreadCache's default constructor has to be responsible for // initializing it. - ThreadCache() : last_lifecycle_id_seen(-1), last_block_used_(NULL) {} + ThreadCache() + : next_lifecycle_id(0), + last_lifecycle_id_seen(-1), + last_serial_arena(NULL) {} #endif + // Number of per-thread lifecycle IDs to reserve. Must be power of two. + // To reduce contention on a global atomic, each thread reserves a batch of + // IDs. The following number is calculated based on a stress test with + // ~6500 threads all frequently allocating a new arena. + static constexpr size_t kPerThreadIds = 256; + // Next lifecycle ID available to this thread. We need to reserve a new + // batch, if `next_lifecycle_id & (kPerThreadIds - 1) == 0`. + uint64_t next_lifecycle_id; // The ThreadCache is considered valid as long as this matches the // lifecycle_id of the arena being used. - int64 last_lifecycle_id_seen; - Block* last_block_used_; + uint64_t last_lifecycle_id_seen; + SerialArena* last_serial_arena; }; - static google::protobuf::internal::SequenceNumber lifecycle_id_generator_; + + // Lifecycle_id can be highly contended variable in a situation of lots of + // arena creation. Make sure that other global variables are not sharing the + // cacheline. +#ifdef _MSC_VER +#pragma warning(disable : 4324) +#endif + struct alignas(64) CacheAlignedLifecycleIdGenerator { + std::atomic id; + }; + static CacheAlignedLifecycleIdGenerator lifecycle_id_generator_; #if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL) - // Android ndk does not support GOOGLE_THREAD_LOCAL keyword so we use a custom thread - // local storage class we implemented. - // iOS also does not support the GOOGLE_THREAD_LOCAL keyword. + // iOS does not support __thread keyword so we use a custom thread local + // storage class we implemented. static ThreadCache& thread_cache(); #elif defined(PROTOBUF_USE_DLLS) // Thread local variables cannot be exposed through DLL interface but we can // wrap them in static functions. static ThreadCache& thread_cache(); #else - static GOOGLE_THREAD_LOCAL ThreadCache thread_cache_; + static PROTOBUF_THREAD_LOCAL ThreadCache thread_cache_; static ThreadCache& thread_cache() { return thread_cache_; } #endif - void Init(); - - // Free all blocks and return the total space used which is the sums of sizes - // of the all the allocated blocks. - uint64 FreeBlocks(); - - void AddCleanupInBlock(Block* b, void* elem, void (*func)(void*)); - CleanupChunk* ExpandCleanupList(CleanupChunk* cleanup, Block* b); - // Delete or Destruct all objects owned by the arena. - void CleanupList(); - - inline void CacheBlock(Block* block) { - thread_cache().last_block_used_ = block; - thread_cache().last_lifecycle_id_seen = lifecycle_id_; - // TODO(haberman): evaluate whether we would gain efficiency by getting rid - // of hint_. It's the only write we do to ArenaImpl in the allocation path, - // which will dirty the cache line. - google::protobuf::internal::Release_Store(&hint_, reinterpret_cast(block)); - } - - google::protobuf::internal::AtomicWord threads_; // Pointer to a linked list of ThreadInfo. - google::protobuf::internal::AtomicWord hint_; // Fast thread-local block access - google::protobuf::internal::AtomicWord space_allocated_; // Sum of sizes of all allocated blocks. - - Block *initial_block_; // If non-NULL, points to the block that came from - // user data. - - // Returns a block owned by this thread. - Block* GetBlock(size_t n); - Block* GetBlockSlow(void* me, Block* my_full_block, size_t n); - Block* NewBlock(void* me, Block* my_last_block, size_t min_bytes); - void InitBlock(Block* b, void *me, size_t size); - static void* AllocFromBlock(Block* b, size_t n); - ThreadInfo* NewThreadInfo(Block* b); - ThreadInfo* FindThreadInfo(void* me); - ThreadInfo* GetThreadInfo(void* me, size_t n); - - int64 lifecycle_id_; // Unique for each arena. Changes on Reset(). - - Options options_; - - GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ArenaImpl); + GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ThreadSafeArena); + // All protos have pointers back to the arena hence Arena must have + // pointer stability. + ThreadSafeArena(ThreadSafeArena&&) = delete; + ThreadSafeArena& operator=(ThreadSafeArena&&) = delete; public: - // kHeaderSize is sizeof(Block), aligned up to the nearest multiple of 8 to - // protect the invariant that pos is always at a multiple of 8. - static const size_t kHeaderSize = (sizeof(Block) + 7) & -8; -#if LANG_CXX11 - static_assert(kHeaderSize % 8 == 0, "kHeaderSize must be a multiple of 8."); -#endif + // kBlockHeaderSize is sizeof(Block), aligned up to the nearest multiple of 8 + // to protect the invariant that pos is always at a multiple of 8. + static constexpr size_t kBlockHeaderSize = SerialArena::kBlockHeaderSize; + static constexpr size_t kSerialArenaSize = + (sizeof(SerialArena) + 7) & static_cast(-8); + static_assert(kBlockHeaderSize % 8 == 0, + "kBlockHeaderSize must be a multiple of 8."); + static_assert(kSerialArenaSize % 8 == 0, + "kSerialArenaSize must be a multiple of 8."); }; } // namespace internal } // namespace protobuf - } // namespace google + +#include + #endif // GOOGLE_PROTOBUF_ARENA_IMPL_H__ diff --git a/3rdparty/protobuf/src/google/protobuf/arenastring.cc b/3rdparty/protobuf/src/google/protobuf/arenastring.cc index 7f33a0c865..169f52729d 100644 --- a/3rdparty/protobuf/src/google/protobuf/arenastring.cc +++ b/3rdparty/protobuf/src/google/protobuf/arenastring.cc @@ -28,16 +28,259 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// The ArenaString implementation is not included in the open-source release. Do -// not include this file in the distribution. - #include +#include +#include +#include +#include +#include +#include +#include +#include + +// clang-format off +#include +// clang-format on + namespace google { namespace protobuf { namespace internal { +const std::string& LazyString::Init() const { + static WrappedMutex mu{GOOGLE_PROTOBUF_LINKER_INITIALIZED}; + mu.Lock(); + const std::string* res = inited_.load(std::memory_order_acquire); + if (res == nullptr) { + auto init_value = init_value_; + res = ::new (static_cast(string_buf_)) + std::string(init_value.ptr, init_value.size); + inited_.store(res, std::memory_order_release); + } + mu.Unlock(); + return *res; +} + + +std::string* ArenaStringPtr::SetAndReturnNewString() { + std::string* new_string = new std::string(); + tagged_ptr_.Set(new_string); + return new_string; +} + +void ArenaStringPtr::DestroyNoArenaSlowPath() { delete UnsafeMutablePointer(); } + +void ArenaStringPtr::Set(const std::string* default_value, + ConstStringParam value, ::google::protobuf::Arena* arena) { + if (IsDefault(default_value)) { + tagged_ptr_.Set(Arena::Create(arena, value)); + } else { + UnsafeMutablePointer()->assign(value.data(), value.length()); + } +} + +void ArenaStringPtr::Set(const std::string* default_value, std::string&& value, + ::google::protobuf::Arena* arena) { + if (IsDefault(default_value)) { + if (arena == nullptr) { + tagged_ptr_.Set(new std::string(std::move(value))); + } else { + tagged_ptr_.Set(Arena::Create(arena, std::move(value))); + } + } else if (IsDonatedString()) { + std::string* current = tagged_ptr_.Get(); + auto* s = new (current) std::string(std::move(value)); + arena->OwnDestructor(s); + tagged_ptr_.Set(s); + } else /* !IsDonatedString() */ { + *UnsafeMutablePointer() = std::move(value); + } +} + +void ArenaStringPtr::Set(EmptyDefault, ConstStringParam value, + ::google::protobuf::Arena* arena) { + Set(&GetEmptyStringAlreadyInited(), value, arena); +} + +void ArenaStringPtr::Set(EmptyDefault, std::string&& value, + ::google::protobuf::Arena* arena) { + Set(&GetEmptyStringAlreadyInited(), std::move(value), arena); +} + +void ArenaStringPtr::Set(NonEmptyDefault, ConstStringParam value, + ::google::protobuf::Arena* arena) { + Set(nullptr, value, arena); +} + +void ArenaStringPtr::Set(NonEmptyDefault, std::string&& value, + ::google::protobuf::Arena* arena) { + Set(nullptr, std::move(value), arena); +} + +std::string* ArenaStringPtr::Mutable(EmptyDefault, ::google::protobuf::Arena* arena) { + if (!IsDonatedString() && !IsDefault(&GetEmptyStringAlreadyInited())) { + return UnsafeMutablePointer(); + } else { + return MutableSlow(arena); + } +} + +std::string* ArenaStringPtr::Mutable(const LazyString& default_value, + ::google::protobuf::Arena* arena) { + if (!IsDonatedString() && !IsDefault(nullptr)) { + return UnsafeMutablePointer(); + } else { + return MutableSlow(arena, default_value); + } +} + +std::string* ArenaStringPtr::MutableNoCopy(const std::string* default_value, + ::google::protobuf::Arena* arena) { + if (!IsDonatedString() && !IsDefault(default_value)) { + return UnsafeMutablePointer(); + } else { + GOOGLE_DCHECK(IsDefault(default_value)); + // Allocate empty. The contents are not relevant. + std::string* new_string = Arena::Create(arena); + tagged_ptr_.Set(new_string); + return new_string; + } +} + +template +std::string* ArenaStringPtr::MutableSlow(::google::protobuf::Arena* arena, + const Lazy&... lazy_default) { + const std::string* const default_value = + sizeof...(Lazy) == 0 ? &GetEmptyStringAlreadyInited() : nullptr; + GOOGLE_DCHECK(IsDefault(default_value)); + std::string* new_string = + Arena::Create(arena, lazy_default.get()...); + tagged_ptr_.Set(new_string); + return new_string; +} + +std::string* ArenaStringPtr::Release(const std::string* default_value, + ::google::protobuf::Arena* arena) { + if (IsDefault(default_value)) { + return nullptr; + } else { + return ReleaseNonDefault(default_value, arena); + } +} + +std::string* ArenaStringPtr::ReleaseNonDefault(const std::string* default_value, + ::google::protobuf::Arena* arena) { + GOOGLE_DCHECK(!IsDefault(default_value)); + + if (!IsDonatedString()) { + std::string* released; + if (arena != nullptr) { + released = new std::string; + released->swap(*UnsafeMutablePointer()); + } else { + released = UnsafeMutablePointer(); + } + tagged_ptr_.Set(const_cast(default_value)); + return released; + } else /* IsDonatedString() */ { + GOOGLE_DCHECK(arena != nullptr); + std::string* released = new std::string(Get()); + tagged_ptr_.Set(const_cast(default_value)); + return released; + } +} + +void ArenaStringPtr::SetAllocated(const std::string* default_value, + std::string* value, ::google::protobuf::Arena* arena) { + // Release what we have first. + if (arena == nullptr && !IsDefault(default_value)) { + delete UnsafeMutablePointer(); + } + if (value == nullptr) { + tagged_ptr_.Set(const_cast(default_value)); + } else { +#ifdef NDEBUG + tagged_ptr_.Set(value); + if (arena != nullptr) { + arena->Own(value); + } +#else + // On debug builds, copy the string so the address differs. delete will + // fail if value was a stack-allocated temporary/etc., which would have + // failed when arena ran its cleanup list. + std::string* new_value = Arena::Create(arena, *value); + delete value; + tagged_ptr_.Set(new_value); +#endif + } +} + +void ArenaStringPtr::Destroy(const std::string* default_value, + ::google::protobuf::Arena* arena) { + if (arena == nullptr) { + GOOGLE_DCHECK(!IsDonatedString()); + if (!IsDefault(default_value)) { + delete UnsafeMutablePointer(); + } + } +} + +void ArenaStringPtr::Destroy(EmptyDefault, ::google::protobuf::Arena* arena) { + Destroy(&GetEmptyStringAlreadyInited(), arena); +} + +void ArenaStringPtr::Destroy(NonEmptyDefault, ::google::protobuf::Arena* arena) { + Destroy(nullptr, arena); +} + +void ArenaStringPtr::ClearToEmpty() { + if (IsDefault(&GetEmptyStringAlreadyInited())) { + // Already set to default -- do nothing. + } else { + // Unconditionally mask away the tag. + // + // UpdateDonatedString uses assign when capacity is larger than the new + // value, which is trivially true in the donated string case. + // const_cast(PtrValue())->clear(); + tagged_ptr_.Get()->clear(); + } +} + +void ArenaStringPtr::ClearToDefault(const LazyString& default_value, + ::google::protobuf::Arena* arena) { + (void)arena; + if (IsDefault(nullptr)) { + // Already set to default -- do nothing. + } else if (!IsDonatedString()) { + UnsafeMutablePointer()->assign(default_value.get()); + } +} + +inline void SetStrWithHeapBuffer(std::string* str, ArenaStringPtr* s) { + TaggedPtr res; + res.Set(str); + s->UnsafeSetTaggedPointer(res); +} + +const char* EpsCopyInputStream::ReadArenaString(const char* ptr, + ArenaStringPtr* s, + Arena* arena) { + GOOGLE_DCHECK(arena != nullptr); + + int size = ReadSize(&ptr); + if (!ptr) return nullptr; + + auto* str = Arena::Create(arena); + ptr = ReadString(ptr, size, str); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); + + SetStrWithHeapBuffer(str, s); + + return ptr; +} } // namespace internal } // namespace protobuf } // namespace google + +#include diff --git a/3rdparty/protobuf/src/google/protobuf/arenastring.h b/3rdparty/protobuf/src/google/protobuf/arenastring.h index c9d045a159..38c36378cd 100644 --- a/3rdparty/protobuf/src/google/protobuf/arenastring.h +++ b/3rdparty/protobuf/src/google/protobuf/arenastring.h @@ -32,307 +32,389 @@ #define GOOGLE_PROTOBUF_ARENASTRING_H__ #include +#include +#include -#include -#include -#include #include -#include +#include +#include +#include + +#include + +#ifdef SWIG +#error "You cannot SWIG proto headers" +#endif -// This is the implementation of arena string fields written for the open-source -// release. The ArenaStringPtr struct below is an internal implementation class -// and *should not be used* by user code. It is used to collect string -// operations together into one place and abstract away the underlying -// string-field pointer representation, so that (for example) an alternate -// implementation that knew more about ::std::string's internals could integrate more -// closely with the arena allocator. namespace google { namespace protobuf { namespace internal { -struct LIBPROTOBUF_EXPORT ArenaStringPtr { - inline void Set(const ::std::string* default_value, - const ::std::string& value, ::google::protobuf::Arena* arena) { - if (ptr_ == default_value) { - CreateInstance(arena, &value); - } else { - *ptr_ = value; - } +template +class ExplicitlyConstructed; + +class SwapFieldHelper; + +// Lazy string instance to support string fields with non-empty default. +// These are initialized on the first call to .get(). +class PROTOBUF_EXPORT LazyString { + public: + // We explicitly make LazyString an aggregate so that MSVC can do constant + // initialization on it without marking it `constexpr`. + // We do not want to use `constexpr` because it makes it harder to have extern + // storage for it and causes library bloat. + struct InitValue { + const char* ptr; + size_t size; + }; + // We keep a union of the initialization value and the std::string to save on + // space. We don't need the string array after Init() is done. + union { + mutable InitValue init_value_; + alignas(std::string) mutable char string_buf_[sizeof(std::string)]; + }; + mutable std::atomic inited_; + + const std::string& get() const { + // This check generates less code than a call-once invocation. + auto* res = inited_.load(std::memory_order_acquire); + if (PROTOBUF_PREDICT_FALSE(res == nullptr)) return Init(); + return *res; } - inline void SetLite(const ::std::string* default_value, - const ::std::string& value, - ::google::protobuf::Arena* arena) { - Set(default_value, value, arena); + private: + // Initialize the string in `string_buf_`, update `inited_` and return it. + // We return it here to avoid having to read it again in the inlined code. + const std::string& Init() const; +}; + +template +class TaggedPtr { + public: + TaggedPtr() = default; + explicit constexpr TaggedPtr(const ExplicitlyConstructed* ptr) + : ptr_(const_cast*>(ptr)) {} + + void SetTagged(T* p) { + Set(p); + ptr_ = reinterpret_cast(as_int() | 1); + } + void Set(T* p) { ptr_ = p; } + T* Get() const { return reinterpret_cast(as_int() & -2); } + bool IsTagged() const { return as_int() & 1; } + + // Returned value is only safe to dereference if IsTagged() == false. + // It is safe to compare. + T* UnsafeGet() const { return static_cast(ptr_); } + + bool IsNull() { return ptr_ == nullptr; } + + private: + uintptr_t as_int() const { return reinterpret_cast(ptr_); } + void* ptr_; +}; + +static_assert(std::is_trivial>::value, + "TaggedPtr must be trivial"); + +// This class encapsulates a pointer to a std::string with or without a donated +// buffer, tagged by bottom bit. It is a high-level wrapper that almost directly +// corresponds to the interface required by string fields in generated +// code. It replaces the old std::string* pointer in such cases. +// +// The object has different but similar code paths for when the default value is +// the empty string and when it is a non-empty string. +// The empty string is handled different throughout the library and there is a +// single global instance of it we can share. +// +// For fields with an empty string default value, there are three distinct +// states: +// +// - Pointer set to 'String' tag (LSB is 0), equal to +// &GetEmptyStringAlreadyInited(): field is set to its default value. Points +// to a true std::string*, but we do not own that std::string* (it's a +// globally shared instance). +// +// - Pointer set to 'String' tag (LSB is 0), but not equal to the global empty +// string: field points to a true std::string* instance that we own. This +// instance is either on the heap or on the arena (i.e. registered on +// free()/destructor-call list) as appropriate. +// +// - Pointer set to 'DonatedString' tag (LSB is 1): points to a std::string +// instance with a buffer on the arena (arena is never nullptr in this case). +// +// For fields with a non-empty string default value, there are three distinct +// states: +// +// - Pointer set to 'String' tag (LSB is 0), equal to `nullptr`: +// Field is in "default" mode and does not point to any actual instance. +// Methods that might need to create an instance of the object will pass a +// `const LazyString&` for it. +// +// - Pointer set to 'String' tag (LSB is 0), but not equal to `nullptr`: +// field points to a true std::string* instance that we own. This instance is +// either on the heap or on the arena (i.e. registered on +// free()/destructor-call list) as appropriate. +// +// - Pointer set to 'DonatedString' tag (LSB is 1): points to a std::string +// instance with a buffer on the arena (arena is never nullptr in this case). +// +// Generated code and reflection code both ensure that ptr_ is never null for +// fields with an empty default. +// Because ArenaStringPtr is used in oneof unions, its constructor is a NOP and +// so the field is always manually initialized via method calls. +// +// Side-note: why pass information about the default on every API call? Because +// we don't want to hold it in a member variable, or else this would go into +// every proto message instance. This would be a huge waste of space, since the +// default instance pointer is typically a global (static class field). We want +// the generated code to be as efficient as possible, and if we take +// the default value information as a parameter that's in practice taken from a +// static class field, and compare ptr_ to the default value, we end up with a +// single "cmp %reg, GLOBAL" in the resulting machine code. (Note that this also +// requires the String tag to be 0 so we can avoid the mask before comparing.) +struct PROTOBUF_EXPORT ArenaStringPtr { + ArenaStringPtr() = default; + explicit constexpr ArenaStringPtr( + const ExplicitlyConstructed* default_value) + : tagged_ptr_(default_value) {} + + // Some methods below are overloaded on a `default_value` and on tags. + // The tagged overloads help reduce code size in the callers in generated + // code, while the `default_value` overloads are useful from reflection. + // By-value empty struct arguments are elided in the ABI. + struct EmptyDefault {}; + struct NonEmptyDefault {}; + + void Set(const std::string* default_value, ConstStringParam value, + ::google::protobuf::Arena* arena); + void Set(const std::string* default_value, std::string&& value, + ::google::protobuf::Arena* arena); + void Set(EmptyDefault, ConstStringParam value, ::google::protobuf::Arena* arena); + void Set(EmptyDefault, std::string&& value, ::google::protobuf::Arena* arena); + void Set(NonEmptyDefault, ConstStringParam value, ::google::protobuf::Arena* arena); + void Set(NonEmptyDefault, std::string&& value, ::google::protobuf::Arena* arena); + template + void Set(FirstParam p1, const char* str, ::google::protobuf::Arena* arena) { + Set(p1, ConstStringParam(str), arena); + } + template + void Set(FirstParam p1, const char* str, size_t size, + ::google::protobuf::Arena* arena) { + ConstStringParam sp{str, size}; // for string_view and `const string &` + Set(p1, sp, arena); + } + template + void Set(FirstParam p1, + std::reference_wrapper const_string_ref, + ::google::protobuf::Arena* arena) { + Set(p1, const_string_ref.get(), arena); + } + + template + void SetBytes(FirstParam p1, SecondParam&& p2, ::google::protobuf::Arena* arena) { + Set(p1, static_cast(p2), arena); + } + template + void SetBytes(FirstParam p1, const void* str, size_t size, + ::google::protobuf::Arena* arena) { + // must work whether ConstStringParam is string_view or `const string &` + ConstStringParam sp{static_cast(str), size}; + Set(p1, sp, arena); } // Basic accessors. - inline const ::std::string& Get() const { return *ptr_; } - - inline ::std::string* Mutable(const ::std::string* default_value, - ::google::protobuf::Arena* arena) { - if (ptr_ == default_value) { - CreateInstance(arena, default_value); - } - return ptr_; + PROTOBUF_NDEBUG_INLINE const std::string& Get() const { + // Unconditionally mask away the tag. + return *tagged_ptr_.Get(); + } + PROTOBUF_NDEBUG_INLINE const std::string* GetPointer() const { + // Unconditionally mask away the tag. + return tagged_ptr_.Get(); } - // Release returns a ::std::string* instance that is heap-allocated and is not - // Own()'d by any arena. If the field was not set, it returns NULL. The caller - // retains ownership. Clears this field back to NULL state. Used to implement - // release_() methods on generated classes. - inline ::std::string* Release(const ::std::string* default_value, - ::google::protobuf::Arena* arena) { - if (ptr_ == default_value) { - return NULL; - } - ::std::string* released = NULL; - if (arena != NULL) { - // ptr_ is owned by the arena. - released = new ::std::string; - released->swap(*ptr_); - } else { - released = ptr_; - } - ptr_ = const_cast< ::std::string* >(default_value); - return released; - } + // For fields with an empty default value. + std::string* Mutable(EmptyDefault, ::google::protobuf::Arena* arena); + // For fields with a non-empty default value. + std::string* Mutable(const LazyString& default_value, ::google::protobuf::Arena* arena); - // UnsafeArenaRelease returns a ::std::string*, but it may be arena-owned (i.e. - // have its destructor already registered) if arena != NULL. If the field was - // not set, this returns NULL. This method clears this field back to NULL - // state. Used to implement unsafe_arena_release_() methods on - // generated classes. - inline ::std::string* UnsafeArenaRelease(const ::std::string* default_value, - ::google::protobuf::Arena* /* arena */) { - if (ptr_ == default_value) { - return NULL; - } - ::std::string* released = ptr_; - ptr_ = const_cast< ::std::string* >(default_value); - return released; - } + // Release returns a std::string* instance that is heap-allocated and is not + // Own()'d by any arena. If the field is not set, this returns nullptr. The + // caller retains ownership. Clears this field back to nullptr state. Used to + // implement release_() methods on generated classes. + PROTOBUF_NODISCARD std::string* Release(const std::string* default_value, + ::google::protobuf::Arena* arena); + PROTOBUF_NODISCARD std::string* ReleaseNonDefault( + const std::string* default_value, ::google::protobuf::Arena* arena); - // Takes a string that is heap-allocated, and takes ownership. The string's - // destructor is registered with the arena. Used to implement + // Takes a std::string that is heap-allocated, and takes ownership. The + // std::string's destructor is registered with the arena. Used to implement // set_allocated_ in generated classes. - inline void SetAllocated(const ::std::string* default_value, - ::std::string* value, ::google::protobuf::Arena* arena) { - if (arena == NULL && ptr_ != default_value) { - Destroy(default_value, arena); - } - if (value != NULL) { - ptr_ = value; - if (arena != NULL) { - arena->Own(value); - } - } else { - ptr_ = const_cast< ::std::string* >(default_value); - } - } - - // Takes a string that has lifetime equal to the arena's lifetime. The arena - // must be non-null. It is safe only to pass this method a value returned by - // UnsafeArenaRelease() on another field of a message in the same arena. Used - // to implement unsafe_arena_set_allocated_ in generated classes. - inline void UnsafeArenaSetAllocated(const ::std::string* default_value, - ::std::string* value, - ::google::protobuf::Arena* /* arena */) { - if (value != NULL) { - ptr_ = value; - } else { - ptr_ = const_cast< ::std::string* >(default_value); - } - } + void SetAllocated(const std::string* default_value, std::string* value, + ::google::protobuf::Arena* arena); // Swaps internal pointers. Arena-safety semantics: this is guarded by the // logic in Swap()/UnsafeArenaSwap() at the message level, so this method is // 'unsafe' if called directly. - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE void Swap(ArenaStringPtr* other) { - std::swap(ptr_, other->ptr_); - } + inline PROTOBUF_NDEBUG_INLINE static void InternalSwap( + const std::string* default_value, ArenaStringPtr* rhs, Arena* rhs_arena, + ArenaStringPtr* lhs, Arena* lhs_arena); // Frees storage (if not on an arena). - inline void Destroy(const ::std::string* default_value, - ::google::protobuf::Arena* arena) { - if (arena == NULL && ptr_ != default_value) { - delete ptr_; - } - } + void Destroy(const std::string* default_value, ::google::protobuf::Arena* arena); + void Destroy(EmptyDefault, ::google::protobuf::Arena* arena); + void Destroy(NonEmptyDefault, ::google::protobuf::Arena* arena); - // Clears content, but keeps allocated string if arena != NULL, to avoid the - // overhead of heap operations. After this returns, the content (as seen by - // the user) will always be the empty string. Assumes that |default_value| - // is an empty string. - inline void ClearToEmpty(const ::std::string* default_value, - ::google::protobuf::Arena* /* arena */) { - if (ptr_ == default_value) { - // Already set to default (which is empty) -- do nothing. - } else { - ptr_->clear(); - } - } + // Clears content, but keeps allocated std::string, to avoid the overhead of + // heap operations. After this returns, the content (as seen by the user) will + // always be the empty std::string. Assumes that |default_value| is an empty + // std::string. + void ClearToEmpty(); - // Clears content, but keeps allocated string if arena != NULL, to avoid the - // overhead of heap operations. After this returns, the content (as seen by - // the user) will always be equal to |default_value|. - inline void ClearToDefault(const ::std::string* default_value, - ::google::protobuf::Arena* /* arena */) { - if (ptr_ == default_value) { - // Already set to default -- do nothing. - } else { - // Have another allocated string -- rather than throwing this away and - // resetting ptr_ to the canonical default string instance, we just reuse - // this instance. - *ptr_ = *default_value; - } - } + // Clears content, assuming that the current value is not the empty + // string default. + void ClearNonDefaultToEmpty(); + + // Clears content, but keeps allocated std::string if arena != nullptr, to + // avoid the overhead of heap operations. After this returns, the content + // (as seen by the user) will always be equal to |default_value|. + void ClearToDefault(const LazyString& default_value, ::google::protobuf::Arena* arena); // Called from generated code / reflection runtime only. Resets value to point - // to a default string pointer, with the semantics that this ArenaStringPtr - // does not own the pointed-to memory. Disregards initial value of ptr_ (so - // this is the *ONLY* safe method to call after construction or when - // reinitializing after becoming the active field in a oneof union). - inline void UnsafeSetDefault(const ::std::string* default_value) { - // Casting away 'const' is safe here: accessors ensure that ptr_ is only - // returned as a const if it is equal to default_value. - ptr_ = const_cast< ::std::string* >(default_value); + // to a default string pointer, with the semantics that this + // ArenaStringPtr does not own the pointed-to memory. Disregards initial value + // of ptr_ (so this is the *ONLY* safe method to call after construction or + // when reinitializing after becoming the active field in a oneof union). + inline void UnsafeSetDefault(const std::string* default_value); + + // Returns a mutable pointer, but doesn't initialize the string to the + // default value. + std::string* MutableNoArenaNoDefault(const std::string* default_value); + + // Get a mutable pointer with unspecified contents. + // Similar to `MutableNoArenaNoDefault`, but also handles the arena case. + // If the value was donated, the contents are discarded. + std::string* MutableNoCopy(const std::string* default_value, + ::google::protobuf::Arena* arena); + + // Destroy the string. Assumes `arena == nullptr`. + void DestroyNoArena(const std::string* default_value); + + // Internal setter used only at parse time to directly set a donated string + // value. + void UnsafeSetTaggedPointer(TaggedPtr value) { + tagged_ptr_ = value; } + // Generated code only! An optimization, in certain cases the generated + // code is certain we can obtain a std::string with no default checks and + // tag tests. + std::string* UnsafeMutablePointer() PROTOBUF_RETURNS_NONNULL; - // The 'NoArena' variants of methods below assume arena == NULL and are - // optimized to provide very little overhead relative to a raw string pointer - // (while still being in-memory compatible with other code that assumes - // ArenaStringPtr). Note the invariant that a class instance that has only - // ever been mutated by NoArena methods must *only* be in the String state - // (i.e., tag bits are not used), *NEVER* ArenaString. This allows all - // tagged-pointer manipulations to be avoided. - inline void SetNoArena(const ::std::string* default_value, - const ::std::string& value) { - if (ptr_ == default_value) { - CreateInstanceNoArena(&value); - } else { - *ptr_ = value; - } - } - -#if LANG_CXX11 - void SetNoArena(const ::std::string* default_value, ::std::string&& value) { - if (IsDefault(default_value)) { - ptr_ = new ::std::string(std::move(value)); - } else { - *ptr_ = std::move(value); - } - } -#endif - - void AssignWithDefault(const ::std::string* default_value, ArenaStringPtr value); - - inline const ::std::string& GetNoArena() const { return *ptr_; } - - inline ::std::string* MutableNoArena(const ::std::string* default_value) { - if (ptr_ == default_value) { - CreateInstanceNoArena(default_value); - } - return ptr_; - } - - inline ::std::string* ReleaseNoArena(const ::std::string* default_value) { - if (ptr_ == default_value) { - return NULL; - } else { - ::std::string* released = ptr_; - ptr_ = const_cast< ::std::string* >(default_value); - return released; - } - } - - inline void SetAllocatedNoArena(const ::std::string* default_value, - ::std::string* value) { - if (ptr_ != default_value) { - delete ptr_; - } - if (value != NULL) { - ptr_ = value; - } else { - ptr_ = const_cast< ::std::string* >(default_value); - } - } - - inline void DestroyNoArena(const ::std::string* default_value) { - if (ptr_ != default_value) { - delete ptr_; - } - } - - inline void ClearToEmptyNoArena(const ::std::string* default_value) { - if (ptr_ == default_value) { - // Nothing: already equal to default (which is the empty string). - } else { - ptr_->clear(); - } - } - - inline void ClearToDefaultNoArena(const ::std::string* default_value) { - if (ptr_ == default_value) { - // Nothing: already set to default. - } else { - // Reuse existing allocated instance. - *ptr_ = *default_value; - } - } - - // Internal accessor used only at parse time to provide direct access to the - // raw pointer from the shared parse routine (in the non-arenas case). The - // parse routine does the string allocation in order to save code size in the - // generated parsing code. - inline ::std::string** UnsafeRawStringPointer() { - return &ptr_; - } - - inline bool IsDefault(const ::std::string* default_value) const { - return ptr_ == default_value; + inline bool IsDefault(const std::string* default_value) const { + // Relies on the fact that kPtrTagString == 0, so if IsString(), ptr_ is the + // actual std::string pointer (and if !IsString(), ptr_ will never be equal + // to any aligned |default_value| pointer). The key is that we want to avoid + // masking in the fastpath const-pointer Get() case for non-arena code. + return tagged_ptr_.UnsafeGet() == default_value; } private: - ::std::string* ptr_; + TaggedPtr tagged_ptr_; - GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE - void CreateInstance(::google::protobuf::Arena* arena, - const ::std::string* initial_value) { - GOOGLE_DCHECK(initial_value != NULL); - ptr_ = new ::std::string(*initial_value); - if (arena != NULL) { - arena->Own(ptr_); - } - } - GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE - void CreateInstanceNoArena(const ::std::string* initial_value) { - GOOGLE_DCHECK(initial_value != NULL); - ptr_ = new ::std::string(*initial_value); + bool IsDonatedString() const { return false; } + + // Swaps tagged pointer without debug hardening. This is to allow python + // protobuf to maintain pointer stability even in DEBUG builds. + inline PROTOBUF_NDEBUG_INLINE static void UnsafeShallowSwap( + ArenaStringPtr* rhs, ArenaStringPtr* lhs) { + std::swap(lhs->tagged_ptr_, rhs->tagged_ptr_); } + + friend class ::google::protobuf::internal::SwapFieldHelper; + + // Slow paths. + + // MutableSlow requires that !IsString() || IsDefault + // Variadic to support 0 args for EmptyDefault and 1 arg for LazyString. + template + std::string* MutableSlow(::google::protobuf::Arena* arena, const Lazy&... lazy_default); + + // Sets value to a newly allocated string and returns it + std::string* SetAndReturnNewString(); + + // Destroys the non-default string value out-of-line + void DestroyNoArenaSlowPath(); + }; -} // namespace internal -} // namespace protobuf +inline void ArenaStringPtr::UnsafeSetDefault(const std::string* value) { + tagged_ptr_.Set(const_cast(value)); +} +// Make sure rhs_arena allocated rhs, and lhs_arena allocated lhs. +inline PROTOBUF_NDEBUG_INLINE void ArenaStringPtr::InternalSwap( // + const std::string* default_value, // + ArenaStringPtr* rhs, Arena* rhs_arena, // + ArenaStringPtr* lhs, Arena* lhs_arena) { + // Silence unused variable warnings in release buildls. + (void)default_value; + (void)rhs_arena; + (void)lhs_arena; + std::swap(lhs->tagged_ptr_, rhs->tagged_ptr_); +#ifdef PROTOBUF_FORCE_COPY_IN_SWAP + auto force_realloc = [default_value](ArenaStringPtr* p, Arena* arena) { + if (p->IsDefault(default_value)) return; + std::string* old_value = p->tagged_ptr_.Get(); + std::string* new_value = + p->IsDonatedString() + ? Arena::Create(arena, *old_value) + : Arena::Create(arena, std::move(*old_value)); + if (arena == nullptr) delete old_value; + p->tagged_ptr_.Set(new_value); + }; + // Because, at this point, tagged_ptr_ has been swapped, arena should also be + // swapped. + force_realloc(lhs, rhs_arena); + force_realloc(rhs, lhs_arena); +#endif // PROTOBUF_FORCE_COPY_IN_SWAP +} +inline void ArenaStringPtr::ClearNonDefaultToEmpty() { + // Unconditionally mask away the tag. + tagged_ptr_.Get()->clear(); +} -namespace protobuf { -namespace internal { - -inline void ArenaStringPtr::AssignWithDefault(const ::std::string* default_value, - ArenaStringPtr value) { - const ::std::string* me = *UnsafeRawStringPointer(); - const ::std::string* other = *value.UnsafeRawStringPointer(); - // If the pointers are the same then do nothing. - if (me != other) { - SetNoArena(default_value, value.GetNoArena()); +inline std::string* ArenaStringPtr::MutableNoArenaNoDefault( + const std::string* default_value) { + // VERY IMPORTANT for performance and code size: this will reduce to a member + // variable load, a pointer check (against |default_value|, in practice a + // static global) and a branch to the slowpath (which calls operator new and + // the ctor). DO NOT add any tagged-pointer operations here. + if (IsDefault(default_value)) { + return SetAndReturnNewString(); + } else { + return UnsafeMutablePointer(); } } +inline void ArenaStringPtr::DestroyNoArena(const std::string* default_value) { + if (!IsDefault(default_value)) { + DestroyNoArenaSlowPath(); + } +} + +inline std::string* ArenaStringPtr::UnsafeMutablePointer() { + GOOGLE_DCHECK(!tagged_ptr_.IsTagged()); + GOOGLE_DCHECK(tagged_ptr_.UnsafeGet() != nullptr); + return tagged_ptr_.UnsafeGet(); +} + + } // namespace internal } // namespace protobuf - } // namespace google + +#include + #endif // GOOGLE_PROTOBUF_ARENASTRING_H__ diff --git a/3rdparty/protobuf/src/google/protobuf/descriptor.cc b/3rdparty/protobuf/src/google/protobuf/descriptor.cc index 3f54b84841..c8ce218a98 100644 --- a/3rdparty/protobuf/src/google/protobuf/descriptor.cc +++ b/3rdparty/protobuf/src/google/protobuf/descriptor.cc @@ -32,48 +32,53 @@ // Based on original Protocol Buffers design by // Sanjay Ghemawat, Jeff Dean, and others. -#include +#include + +#include +#include +#include +#include #include #include -#ifndef _SHARED_PTR_H -#include -#endif #include #include +#include +#include #include -#include -#include #include #include -#include -#include #include #include -#include +#include +#include +#include #include #include #include -#include -#include #include #include #include #include #include #include +#include #include - +#include #include #include +#include #undef PACKAGE // autoheader #defines this. :( -namespace google { +#include + +namespace google { namespace protobuf { -struct Symbol { +class Symbol { + public: enum Type { NULL_SYMBOL, MESSAGE, @@ -81,139 +86,264 @@ struct Symbol { ONEOF, ENUM, ENUM_VALUE, + ENUM_VALUE_OTHER_PARENT, SERVICE, METHOD, - PACKAGE - }; - Type type; - union { - const Descriptor* descriptor; - const FieldDescriptor* field_descriptor; - const OneofDescriptor* oneof_descriptor; - const EnumDescriptor* enum_descriptor; - const EnumValueDescriptor* enum_value_descriptor; - const ServiceDescriptor* service_descriptor; - const MethodDescriptor* method_descriptor; - const FileDescriptor* package_file_descriptor; + PACKAGE, + QUERY_KEY }; - inline Symbol() : type(NULL_SYMBOL) { descriptor = NULL; } - inline bool IsNull() const { return type == NULL_SYMBOL; } - inline bool IsType() const { return type == MESSAGE || type == ENUM; } - inline bool IsAggregate() const { - return type == MESSAGE || type == PACKAGE || type == ENUM || - type == SERVICE; + Symbol() : ptr_(nullptr) {} + + // Every object we store derives from internal::SymbolBase, where we store the + // symbol type enum. + // Storing in the object can be done without using more space in most cases, + // while storing it in the Symbol type would require 8 bytes. +#define DEFINE_MEMBERS(TYPE, TYPE_CONSTANT, FIELD) \ + explicit Symbol(TYPE* value) : ptr_(value) { \ + value->symbol_type_ = TYPE_CONSTANT; \ + } \ + const TYPE* FIELD() const { \ + return type() == TYPE_CONSTANT ? static_cast(ptr_) : nullptr; \ } -#define CONSTRUCTOR(TYPE, TYPE_CONSTANT, FIELD) \ - inline explicit Symbol(const TYPE* value) { \ - type = TYPE_CONSTANT; \ - this->FIELD = value; \ + DEFINE_MEMBERS(Descriptor, MESSAGE, descriptor) + DEFINE_MEMBERS(FieldDescriptor, FIELD, field_descriptor) + DEFINE_MEMBERS(OneofDescriptor, ONEOF, oneof_descriptor) + DEFINE_MEMBERS(EnumDescriptor, ENUM, enum_descriptor) + DEFINE_MEMBERS(ServiceDescriptor, SERVICE, service_descriptor) + DEFINE_MEMBERS(MethodDescriptor, METHOD, method_descriptor) + + // We use a special node for FileDescriptor. + // It is potentially added to the table with multiple different names, so we + // need a separate place to put the name. + struct Package : internal::SymbolBase { + const std::string* name; + const FileDescriptor* file; + }; + DEFINE_MEMBERS(Package, PACKAGE, package_file_descriptor) + + // Enum values have two different parents. + // We use two different identitied for the same object to determine the two + // different insertions in the map. + static Symbol EnumValue(EnumValueDescriptor* value, int n) { + Symbol s; + internal::SymbolBase* ptr; + if (n == 0) { + ptr = static_cast*>(value); + ptr->symbol_type_ = ENUM_VALUE; + } else { + ptr = static_cast*>(value); + ptr->symbol_type_ = ENUM_VALUE_OTHER_PARENT; + } + s.ptr_ = ptr; + return s; } - CONSTRUCTOR(Descriptor, MESSAGE, descriptor) - CONSTRUCTOR(FieldDescriptor, FIELD, field_descriptor) - CONSTRUCTOR(OneofDescriptor, ONEOF, oneof_descriptor) - CONSTRUCTOR(EnumDescriptor, ENUM, enum_descriptor) - CONSTRUCTOR(EnumValueDescriptor, ENUM_VALUE, enum_value_descriptor) - CONSTRUCTOR(ServiceDescriptor, SERVICE, service_descriptor) - CONSTRUCTOR(MethodDescriptor, METHOD, method_descriptor) - CONSTRUCTOR(FileDescriptor, PACKAGE, package_file_descriptor) -#undef CONSTRUCTOR + const EnumValueDescriptor* enum_value_descriptor() const { + return type() == ENUM_VALUE + ? static_cast( + static_cast*>(ptr_)) + : type() == ENUM_VALUE_OTHER_PARENT + ? static_cast( + static_cast*>(ptr_)) + : nullptr; + } + + // Not a real symbol. + // Only used for heterogeneous lookups and never actually inserted in the + // tables. + struct QueryKey : internal::SymbolBase { + StringPiece name; + const void* parent; + int field_number; + }; + DEFINE_MEMBERS(QueryKey, QUERY_KEY, query_key); +#undef DEFINE_MEMBERS + + Type type() const { + return ptr_ == nullptr ? NULL_SYMBOL + : static_cast(ptr_->symbol_type_); + } + bool IsNull() const { return type() == NULL_SYMBOL; } + bool IsType() const { return type() == MESSAGE || type() == ENUM; } + bool IsAggregate() const { + return type() == MESSAGE || type() == PACKAGE || type() == ENUM || + type() == SERVICE; + } const FileDescriptor* GetFile() const { - switch (type) { - case NULL_SYMBOL: - return NULL; + switch (type()) { case MESSAGE: - return descriptor->file(); + return descriptor()->file(); case FIELD: - return field_descriptor->file(); + return field_descriptor()->file(); case ONEOF: - return oneof_descriptor->containing_type()->file(); + return oneof_descriptor()->containing_type()->file(); case ENUM: - return enum_descriptor->file(); + return enum_descriptor()->file(); case ENUM_VALUE: - return enum_value_descriptor->type()->file(); + return enum_value_descriptor()->type()->file(); case SERVICE: - return service_descriptor->file(); + return service_descriptor()->file(); case METHOD: - return method_descriptor->service()->file(); + return method_descriptor()->service()->file(); case PACKAGE: - return package_file_descriptor; + return package_file_descriptor()->file; + default: + return nullptr; } - return NULL; } + + StringPiece full_name() const { + switch (type()) { + case MESSAGE: + return descriptor()->full_name(); + case FIELD: + return field_descriptor()->full_name(); + case ONEOF: + return oneof_descriptor()->full_name(); + case ENUM: + return enum_descriptor()->full_name(); + case ENUM_VALUE: + return enum_value_descriptor()->full_name(); + case SERVICE: + return service_descriptor()->full_name(); + case METHOD: + return method_descriptor()->full_name(); + case PACKAGE: + return *package_file_descriptor()->name; + case QUERY_KEY: + return query_key()->name; + default: + GOOGLE_CHECK(false); + } + return ""; + } + + std::pair parent_name_key() const { + const auto or_file = [&](const void* p) { return p ? p : GetFile(); }; + switch (type()) { + case MESSAGE: + return {or_file(descriptor()->containing_type()), descriptor()->name()}; + case FIELD: { + auto* field = field_descriptor(); + return {or_file(field->is_extension() ? field->extension_scope() + : field->containing_type()), + field->name()}; + } + case ONEOF: + return {oneof_descriptor()->containing_type(), + oneof_descriptor()->name()}; + case ENUM: + return {or_file(enum_descriptor()->containing_type()), + enum_descriptor()->name()}; + case ENUM_VALUE: + return {or_file(enum_value_descriptor()->type()->containing_type()), + enum_value_descriptor()->name()}; + case ENUM_VALUE_OTHER_PARENT: + return {enum_value_descriptor()->type(), + enum_value_descriptor()->name()}; + case SERVICE: + return {GetFile(), service_descriptor()->name()}; + case METHOD: + return {method_descriptor()->service(), method_descriptor()->name()}; + case QUERY_KEY: + return {query_key()->parent, query_key()->name}; + default: + GOOGLE_CHECK(false); + } + return {}; + } + + std::pair parent_number_key() const { + switch (type()) { + case FIELD: + return {field_descriptor()->containing_type(), + field_descriptor()->number()}; + case ENUM_VALUE: + return {enum_value_descriptor()->type(), + enum_value_descriptor()->number()}; + case QUERY_KEY: + return {query_key()->parent, query_key()->field_number}; + default: + GOOGLE_CHECK(false); + } + return {}; + } + + private: + const internal::SymbolBase* ptr_; }; const FieldDescriptor::CppType -FieldDescriptor::kTypeToCppTypeMap[MAX_TYPE + 1] = { - static_cast(0), // 0 is reserved for errors + FieldDescriptor::kTypeToCppTypeMap[MAX_TYPE + 1] = { + static_cast(0), // 0 is reserved for errors - CPPTYPE_DOUBLE, // TYPE_DOUBLE - CPPTYPE_FLOAT, // TYPE_FLOAT - CPPTYPE_INT64, // TYPE_INT64 - CPPTYPE_UINT64, // TYPE_UINT64 - CPPTYPE_INT32, // TYPE_INT32 - CPPTYPE_UINT64, // TYPE_FIXED64 - CPPTYPE_UINT32, // TYPE_FIXED32 - CPPTYPE_BOOL, // TYPE_BOOL - CPPTYPE_STRING, // TYPE_STRING - CPPTYPE_MESSAGE, // TYPE_GROUP - CPPTYPE_MESSAGE, // TYPE_MESSAGE - CPPTYPE_STRING, // TYPE_BYTES - CPPTYPE_UINT32, // TYPE_UINT32 - CPPTYPE_ENUM, // TYPE_ENUM - CPPTYPE_INT32, // TYPE_SFIXED32 - CPPTYPE_INT64, // TYPE_SFIXED64 - CPPTYPE_INT32, // TYPE_SINT32 - CPPTYPE_INT64, // TYPE_SINT64 + CPPTYPE_DOUBLE, // TYPE_DOUBLE + CPPTYPE_FLOAT, // TYPE_FLOAT + CPPTYPE_INT64, // TYPE_INT64 + CPPTYPE_UINT64, // TYPE_UINT64 + CPPTYPE_INT32, // TYPE_INT32 + CPPTYPE_UINT64, // TYPE_FIXED64 + CPPTYPE_UINT32, // TYPE_FIXED32 + CPPTYPE_BOOL, // TYPE_BOOL + CPPTYPE_STRING, // TYPE_STRING + CPPTYPE_MESSAGE, // TYPE_GROUP + CPPTYPE_MESSAGE, // TYPE_MESSAGE + CPPTYPE_STRING, // TYPE_BYTES + CPPTYPE_UINT32, // TYPE_UINT32 + CPPTYPE_ENUM, // TYPE_ENUM + CPPTYPE_INT32, // TYPE_SFIXED32 + CPPTYPE_INT64, // TYPE_SFIXED64 + CPPTYPE_INT32, // TYPE_SINT32 + CPPTYPE_INT64, // TYPE_SINT64 }; -const char * const FieldDescriptor::kTypeToName[MAX_TYPE + 1] = { - "ERROR", // 0 is reserved for errors +const char* const FieldDescriptor::kTypeToName[MAX_TYPE + 1] = { + "ERROR", // 0 is reserved for errors - "double", // TYPE_DOUBLE - "float", // TYPE_FLOAT - "int64", // TYPE_INT64 - "uint64", // TYPE_UINT64 - "int32", // TYPE_INT32 - "fixed64", // TYPE_FIXED64 - "fixed32", // TYPE_FIXED32 - "bool", // TYPE_BOOL - "string", // TYPE_STRING - "group", // TYPE_GROUP - "message", // TYPE_MESSAGE - "bytes", // TYPE_BYTES - "uint32", // TYPE_UINT32 - "enum", // TYPE_ENUM - "sfixed32", // TYPE_SFIXED32 - "sfixed64", // TYPE_SFIXED64 - "sint32", // TYPE_SINT32 - "sint64", // TYPE_SINT64 + "double", // TYPE_DOUBLE + "float", // TYPE_FLOAT + "int64", // TYPE_INT64 + "uint64", // TYPE_UINT64 + "int32", // TYPE_INT32 + "fixed64", // TYPE_FIXED64 + "fixed32", // TYPE_FIXED32 + "bool", // TYPE_BOOL + "string", // TYPE_STRING + "group", // TYPE_GROUP + "message", // TYPE_MESSAGE + "bytes", // TYPE_BYTES + "uint32", // TYPE_UINT32 + "enum", // TYPE_ENUM + "sfixed32", // TYPE_SFIXED32 + "sfixed64", // TYPE_SFIXED64 + "sint32", // TYPE_SINT32 + "sint64", // TYPE_SINT64 }; -const char * const FieldDescriptor::kCppTypeToName[MAX_CPPTYPE + 1] = { - "ERROR", // 0 is reserved for errors +const char* const FieldDescriptor::kCppTypeToName[MAX_CPPTYPE + 1] = { + "ERROR", // 0 is reserved for errors - "int32", // CPPTYPE_INT32 - "int64", // CPPTYPE_INT64 - "uint32", // CPPTYPE_UINT32 - "uint64", // CPPTYPE_UINT64 - "double", // CPPTYPE_DOUBLE - "float", // CPPTYPE_FLOAT - "bool", // CPPTYPE_BOOL - "enum", // CPPTYPE_ENUM - "string", // CPPTYPE_STRING - "message", // CPPTYPE_MESSAGE + "int32", // CPPTYPE_INT32 + "int64", // CPPTYPE_INT64 + "uint32", // CPPTYPE_UINT32 + "uint64", // CPPTYPE_UINT64 + "double", // CPPTYPE_DOUBLE + "float", // CPPTYPE_FLOAT + "bool", // CPPTYPE_BOOL + "enum", // CPPTYPE_ENUM + "string", // CPPTYPE_STRING + "message", // CPPTYPE_MESSAGE }; -const char * const FieldDescriptor::kLabelToName[MAX_LABEL + 1] = { - "ERROR", // 0 is reserved for errors +const char* const FieldDescriptor::kLabelToName[MAX_LABEL + 1] = { + "ERROR", // 0 is reserved for errors - "optional", // LABEL_OPTIONAL - "required", // LABEL_REQUIRED - "repeated", // LABEL_REPEATED + "optional", // LABEL_OPTIONAL + "required", // LABEL_REQUIRED + "repeated", // LABEL_REPEATED }; const char* FileDescriptor::SyntaxName(FileDescriptor::Syntax syntax) { @@ -226,12 +356,12 @@ const char* FileDescriptor::SyntaxName(FileDescriptor::Syntax syntax) { return "unknown"; } GOOGLE_LOG(FATAL) << "can't reach here."; - return NULL; + return nullptr; } -static const char * const kNonLinkedWeakMessageReplacementName = "google.protobuf.Empty"; +static const char* const kNonLinkedWeakMessageReplacementName = "google.protobuf.Empty"; -#if !defined(_MSC_VER) || _MSC_VER >= 1900 +#if !defined(_MSC_VER) || (_MSC_VER >= 1900 && _MSC_VER < 1912) const int FieldDescriptor::kMaxNumber; const int FieldDescriptor::kFirstReservedNumber; const int FieldDescriptor::kLastReservedNumber; @@ -248,19 +378,19 @@ char ToLower(char ch) { return (ch >= 'A' && ch <= 'Z') ? (ch - 'A' + 'a') : ch; } -string ToCamelCase(const string& input, bool lower_first) { +std::string ToCamelCase(const std::string& input, bool lower_first) { bool capitalize_next = !lower_first; - string result; + std::string result; result.reserve(input.size()); - for (int i = 0; i < input.size(); i++) { - if (input[i] == '_') { + for (char character : input) { + if (character == '_') { capitalize_next = true; } else if (capitalize_next) { - result.push_back(ToUpper(input[i])); + result.push_back(ToUpper(character)); capitalize_next = false; } else { - result.push_back(input[i]); + result.push_back(character); } } @@ -272,38 +402,38 @@ string ToCamelCase(const string& input, bool lower_first) { return result; } -string ToJsonName(const string& input) { +std::string ToJsonName(const std::string& input) { bool capitalize_next = false; - string result; + std::string result; result.reserve(input.size()); - for (int i = 0; i < input.size(); i++) { - if (input[i] == '_') { + for (char character : input) { + if (character == '_') { capitalize_next = true; } else if (capitalize_next) { - result.push_back(ToUpper(input[i])); + result.push_back(ToUpper(character)); capitalize_next = false; } else { - result.push_back(input[i]); + result.push_back(character); } } return result; } -string EnumValueToPascalCase(const string& input) { +std::string EnumValueToPascalCase(const std::string& input) { bool next_upper = true; - string result; + std::string result; result.reserve(input.size()); - for (int i = 0; i < input.size(); i++) { - if (input[i] == '_') { + for (char character : input) { + if (character == '_') { next_upper = true; } else { if (next_upper) { - result.push_back(ToUpper(input[i])); + result.push_back(ToUpper(character)); } else { - result.push_back(ToLower(input[i])); + result.push_back(ToLower(character)); } next_upper = false; } @@ -317,16 +447,16 @@ class PrefixRemover { public: PrefixRemover(StringPiece prefix) { // Strip underscores and lower-case the prefix. - for (int i = 0; i < prefix.size(); i++) { - if (prefix[i] != '_') { - prefix_ += ascii_tolower(prefix[i]); + for (char character : prefix) { + if (character != '_') { + prefix_ += ascii_tolower(character); } } } // Tries to remove the enum prefix from this enum value. // If this is not possible, returns the input verbatim. - string MaybeRemove(StringPiece str) { + std::string MaybeRemove(StringPiece str) { // We can't just lowercase and strip str and look for a prefix. // We need to properly recognize the difference between: // @@ -346,14 +476,14 @@ class PrefixRemover { } if (ascii_tolower(str[i]) != prefix_[j++]) { - return str.as_string(); + return std::string(str); } } // If we didn't make it through the prefix, we've failed to strip the // prefix. if (j < prefix_.size()) { - return str.as_string(); + return std::string(str); } // Skip underscores between prefix and further characters. @@ -363,47 +493,41 @@ class PrefixRemover { // Enum label can't be the empty string. if (i == str.size()) { - return str.as_string(); + return std::string(str); } // We successfully stripped the prefix. str.remove_prefix(i); - return str.as_string(); + return std::string(str); } private: - string prefix_; + std::string prefix_; }; -// A DescriptorPool contains a bunch of hash_maps to implement the +// A DescriptorPool contains a bunch of hash-maps to implement the // various Find*By*() methods. Since hashtable lookups are O(1), it's -// most efficient to construct a fixed set of large hash_maps used by +// most efficient to construct a fixed set of large hash-maps used by // all objects in the pool rather than construct one or more small -// hash_maps for each object. +// hash-maps for each object. // -// The keys to these hash_maps are (parent, name) or (parent, number) -// pairs. Unfortunately STL doesn't provide hash functions for pair<>, -// so we must invent our own. -// -// TODO(kenton): Use StringPiece rather than const char* in keys? It would -// be a lot cleaner but we'd just have to convert it back to const char* -// for the open source release. +// The keys to these hash-maps are (parent, name) or (parent, number) pairs. -typedef std::pair PointerStringPair; +typedef std::pair PointerStringPair; -struct PointerStringPairEqual { - inline bool operator()(const PointerStringPair& a, - const PointerStringPair& b) const { - return a.first == b.first && strcmp(a.second, b.second) == 0; - } -}; +typedef std::pair DescriptorIntPair; -template +#define HASH_MAP std::unordered_map +#define HASH_SET std::unordered_set +#define HASH_FXN hash + +template struct PointerIntegerPairHash { size_t operator()(const PairType& p) const { - // FIXME(kenton): What is the best way to compute this hash? I have - // no idea! This seems a bit better than an XOR. - return reinterpret_cast(p.first) * ((1 << 16) - 1) + p.second; + static const size_t prime1 = 16777499; + static const size_t prime2 = 16777619; + return reinterpret_cast(p.first) * prime1 ^ + static_cast(p.second) * prime2; } #ifdef _MSC_VER @@ -412,21 +536,16 @@ struct PointerIntegerPairHash { static const size_t min_buckets = 8; #endif inline bool operator()(const PairType& a, const PairType& b) const { - return a.first < b.first || - (a.first == b.first && a.second < b.second); + return a < b; } }; -typedef std::pair DescriptorIntPair; -typedef std::pair EnumIntPair; - struct PointerStringPairHash { size_t operator()(const PointerStringPair& p) const { - // FIXME(kenton): What is the best way to compute this hash? I have - // no idea! This seems a bit better than an XOR. - hash cstring_hash; - return reinterpret_cast(p.first) * ((1 << 16) - 1) + - cstring_hash(p.second); + static const size_t prime = 16777619; + hash string_hash; + return reinterpret_cast(p.first) * prime ^ + static_cast(string_hash(p.second)); } #ifdef _MSC_VER @@ -436,77 +555,497 @@ struct PointerStringPairHash { #endif inline bool operator()(const PointerStringPair& a, const PointerStringPair& b) const { - if (a.first < b.first) return true; - if (a.first > b.first) return false; - return strcmp(a.second, b.second) < 0; + return a < b; } }; const Symbol kNullSymbol; -typedef hash_map, streq> - SymbolsByNameMap; -typedef hash_map - SymbolsByParentMap; -typedef hash_map, streq> - FilesByNameMap; -typedef hash_map - FieldsByNameMap; -typedef hash_map > - FieldsByNumberMap; -typedef hash_map > - EnumValuesByNumberMap; -// This is a map rather than a hash_map, since we use it to iterate +struct SymbolByFullNameHash { + size_t operator()(Symbol s) const { + return HASH_FXN{}(s.full_name()); + } +}; +struct SymbolByFullNameEq { + bool operator()(Symbol a, Symbol b) const { + return a.full_name() == b.full_name(); + } +}; +using SymbolsByNameSet = + HASH_SET; + +struct SymbolByParentHash { + size_t operator()(Symbol s) const { + return PointerStringPairHash{}(s.parent_name_key()); + } +}; +struct SymbolByParentEq { + bool operator()(Symbol a, Symbol b) const { + return a.parent_name_key() == b.parent_name_key(); + } +}; +using SymbolsByParentSet = + HASH_SET; + +typedef HASH_MAP> + FilesByNameMap; + +typedef HASH_MAP + FieldsByNameMap; + +struct FieldsByNumberHash { + size_t operator()(Symbol s) const { + return PointerIntegerPairHash>{}( + s.parent_number_key()); + } +}; +struct FieldsByNumberEq { + bool operator()(Symbol a, Symbol b) const { + return a.parent_number_key() == b.parent_number_key(); + } +}; +using FieldsByNumberSet = + HASH_SET; +using EnumValuesByNumberSet = FieldsByNumberSet; + +// This is a map rather than a hash-map, since we use it to iterate // through all the extensions that extend a given Descriptor, and an // ordered data structure that implements lower_bound is convenient // for that. typedef std::map - ExtensionsGroupedByDescriptorMap; -typedef hash_map LocationsByPathMap; + ExtensionsGroupedByDescriptorMap; +typedef HASH_MAP + LocationsByPathMap; -std::set* allowed_proto3_extendees_ = NULL; -GOOGLE_PROTOBUF_DECLARE_ONCE(allowed_proto3_extendees_init_); - -void DeleteAllowedProto3Extendee() { - delete allowed_proto3_extendees_; -} - -void InitAllowedProto3Extendee() { - allowed_proto3_extendees_ = new std::set; +std::set* NewAllowedProto3Extendee() { + auto allowed_proto3_extendees = new std::set; const char* kOptionNames[] = { - "FileOptions", "MessageOptions", "FieldOptions", "EnumOptions", + "FileOptions", "MessageOptions", "FieldOptions", "EnumOptions", "EnumValueOptions", "ServiceOptions", "MethodOptions", "OneofOptions"}; - for (int i = 0; i < GOOGLE_ARRAYSIZE(kOptionNames); ++i) { + for (const char* option_name : kOptionNames) { // descriptor.proto has a different package name in opensource. We allow // both so the opensource protocol compiler can also compile internal // proto3 files with custom options. See: b/27567912 - allowed_proto3_extendees_->insert(string("google.protobuf.") + - kOptionNames[i]); + allowed_proto3_extendees->insert(std::string("google.protobuf.") + + option_name); // Split the word to trick the opensource processing scripts so they - // will keep the origial package name. - allowed_proto3_extendees_->insert(string("proto") + "2." + kOptionNames[i]); + // will keep the original package name. + allowed_proto3_extendees->insert(std::string("proto") + "2." + option_name); } - - google::protobuf::internal::OnShutdown(&DeleteAllowedProto3Extendee); + return allowed_proto3_extendees; } // Checks whether the extendee type is allowed in proto3. // Only extensions to descriptor options are allowed. We use name comparison // instead of comparing the descriptor directly because the extensions may be // defined in a different pool. -bool AllowedExtendeeInProto3(const string& name) { - ::google::protobuf::GoogleOnceInit(&allowed_proto3_extendees_init_, &InitAllowedProto3Extendee); - return allowed_proto3_extendees_->find(name) != - allowed_proto3_extendees_->end(); +bool AllowedExtendeeInProto3(const std::string& name) { + static auto allowed_proto3_extendees = + internal::OnShutdownDelete(NewAllowedProto3Extendee()); + return allowed_proto3_extendees->find(name) != + allowed_proto3_extendees->end(); } +// This bump allocator arena is optimized for the use case of this file. It is +// mostly optimized for memory usage, since these objects are expected to live +// for the entirety of the program. +// +// Some differences from other arenas: +// - It has a fixed number of non-trivial types it can hold. This allows +// tracking the allocations with a single byte. In contrast, google::protobuf::Arena +// uses 16 bytes per non-trivial object created. +// - It has some extra metadata for rollbacks. This is necessary for +// implementing the API below. This metadata is flushed at the end and would +// not cause persistent memory usage. +// - It tries to squeeze every byte of out the blocks. If an allocation is too +// large for the current block we move the block to a secondary area where we +// can still use it for smaller objects. This complicates rollback logic but +// makes it much more memory efficient. +// +// The allocation strategy is as follows: +// - Memory is allocated from the front, with a forced 8 byte alignment. +// - Metadata is allocated from the back, one byte per element. +// - The metadata encodes one of two things: +// * For types we want to track, the index into KnownTypes. +// * For raw memory blocks, the size of the block (in 8 byte increments +// to allow for a larger limit). +// - When the raw data is too large to represent in the metadata byte, we +// allocate this memory separately in the heap and store an OutOfLineAlloc +// object instead. These come from large array allocations and alike. +// +// Blocks are kept in 3 areas: +// - `current_` is the one we are currently allocating from. When we need to +// allocate a block that doesn't fit there, we make a new block and move the +// old `current_` to one of the areas below. +// - Blocks that have no more usable space left (ie less than 9 bytes) are +// stored in `full_blocks_`. +// - Blocks that have some usable space are categorized in +// `small_size_blocks_` depending on how much space they have left. +// See `kSmallSizes` to see which sizes we track. +// +class TableArena { + public: + // Allocate a block on `n` bytes, with no destructor information saved. + void* AllocateMemory(uint32_t n) { + uint32_t tag = SizeToRawTag(n) + kFirstRawTag; + if (tag > 255) { + // We can't fit the size, use an OutOfLineAlloc. + return Create(OutOfLineAlloc{::operator new(n), n})->ptr; + } + + return AllocRawInternal(n, static_cast(tag)); + } + + // Allocate and construct an element of type `T` as if by + // `T(std::forward(args...))`. + // The object is registered for destruction, if its destructor is not trivial. + template + T* Create(Args&&... args) { + static_assert(alignof(T) <= 8, ""); + return ::new (AllocRawInternal(sizeof(T), TypeTag(KnownTypes{}))) + T(std::forward(args)...); + } + + TableArena() {} + + TableArena(const TableArena&) = delete; + TableArena& operator=(const TableArena&) = delete; + + ~TableArena() { + // Uncomment this to debug usage statistics of the arena blocks. + // PrintUsageInfo(); + + for (Block* list : GetLists()) { + while (list != nullptr) { + Block* b = list; + list = list->next; + b->VisitBlock(DestroyVisitor{}); + b->Destroy(); + } + } + } + + + // This function exists for debugging only. + // It can be called from the destructor to dump some info in the tests to + // inspect the usage of the arena. + void PrintUsageInfo() const { + const auto print_histogram = [](Block* b, int size) { + std::map unused_space_count; + int count = 0; + for (; b != nullptr; b = b->next) { + ++unused_space_count[b->space_left()]; + ++count; + } + if (size > 0) { + fprintf(stderr, " Blocks `At least %d`", size); + } else { + fprintf(stderr, " Blocks `full`"); + } + fprintf(stderr, ": %d blocks.\n", count); + for (auto p : unused_space_count) { + fprintf(stderr, " space=%4u, count=%3u\n", p.first, p.second); + } + }; + + fprintf(stderr, "TableArena unused space histogram:\n"); + fprintf(stderr, " Current: %u\n", + current_ != nullptr ? current_->space_left() : 0); + print_histogram(full_blocks_, 0); + for (size_t i = 0; i < kSmallSizes.size(); ++i) { + print_histogram(small_size_blocks_[i], kSmallSizes[i]); + } + } + + // Current allocation count. + // This can be used for checkpointing. + size_t num_allocations() const { return num_allocations_; } + + // Rollback the latest allocations until we reach back to `checkpoint` + // num_allocations. + void RollbackTo(size_t checkpoint) { + while (num_allocations_ > checkpoint) { + GOOGLE_DCHECK(!rollback_info_.empty()); + auto& info = rollback_info_.back(); + Block* b = info.block; + + VisitAlloc(b->data(), &b->start_offset, &b->end_offset, DestroyVisitor{}, + KnownTypes{}); + if (--info.count == 0) { + rollback_info_.pop_back(); + } + --num_allocations_; + } + + // Reconstruct the lists and destroy empty blocks. + auto lists = GetLists(); + current_ = full_blocks_ = nullptr; + small_size_blocks_.fill(nullptr); + + for (Block* list : lists) { + while (list != nullptr) { + Block* b = list; + list = list->next; + + if (b->start_offset == 0) { + // This is empty, free it. + b->Destroy(); + } else { + RelocateToUsedList(b); + } + } + } + } + + // Clear all rollback information. Reduces memory usage. + // Trying to rollback past num_allocations() is now impossible. + void ClearRollbackData() { + rollback_info_.clear(); + rollback_info_.shrink_to_fit(); + } + + private: + static constexpr size_t RoundUp(size_t n) { return (n + 7) & ~7; } + + using Tag = unsigned char; + + void* AllocRawInternal(uint32_t size, Tag tag) { + GOOGLE_DCHECK_GT(size, 0); + size = RoundUp(size); + + Block* to_relocate = nullptr; + Block* to_use = nullptr; + + for (size_t i = 0; i < kSmallSizes.size(); ++i) { + if (small_size_blocks_[i] != nullptr && size <= kSmallSizes[i]) { + to_use = to_relocate = PopBlock(small_size_blocks_[i]); + break; + } + } + + if (to_relocate != nullptr) { + // We found one in the loop. + } else if (current_ != nullptr && size + 1 <= current_->space_left()) { + to_use = current_; + } else { + // No space left anywhere, make a new block. + to_relocate = current_; + // For now we hardcode the size to one page. Note that the maximum we can + // allocate in the block according to the limits of Tag is less than 2k, + // so this can fit anything that Tag can represent. + constexpr size_t kBlockSize = 4096; + to_use = current_ = ::new (::operator new(kBlockSize)) Block(kBlockSize); + GOOGLE_DCHECK_GE(current_->space_left(), size + 1); + } + + ++num_allocations_; + if (!rollback_info_.empty() && rollback_info_.back().block == to_use) { + ++rollback_info_.back().count; + } else { + rollback_info_.push_back({to_use, 1}); + } + + void* p = to_use->Allocate(size, tag); + if (to_relocate != nullptr) { + RelocateToUsedList(to_relocate); + } + return p; + } + + static void OperatorDelete(void* p, size_t s) { +#if defined(__GXX_DELETE_WITH_SIZE__) || defined(__cpp_sized_deallocation) + ::operator delete(p, s); +#else + ::operator delete(p); +#endif + } + + struct OutOfLineAlloc { + void* ptr; + uint32_t size; + }; + + template + struct TypeList { + static constexpr Tag kSize = static_cast(sizeof...(T)); + }; + + template + static void RunVisitor(char* p, uint16_t* start, Visitor visit) { + *start -= RoundUp(sizeof(T)); + visit(reinterpret_cast(p + *start)); + } + + // Visit the allocation at the passed location. + // It updates start/end to be after the visited object. + // This allows visiting a whole block by calling the function in a loop. + template + static void VisitAlloc(char* p, uint16_t* start, uint16_t* end, Visitor visit, + TypeList) { + const Tag tag = static_cast(p[*end]); + if (tag >= kFirstRawTag) { + // Raw memory. Skip it. + *start -= TagToSize(tag); + } else { + using F = void (*)(char*, uint16_t*, Visitor); + static constexpr F kFuncs[] = {&RunVisitor...}; + kFuncs[tag](p, start, visit); + } + ++*end; + } + + template + static constexpr Tag TypeTag(TypeList) { + return 0; + } + + template < + typename U, typename T, typename... Ts, + typename = typename std::enable_if::value>::type> + static constexpr Tag TypeTag(TypeList) { + return 1 + TypeTag(TypeList{}); + } + + template + static constexpr Tag TypeTag(TypeList<>) { + static_assert(std::is_trivially_destructible::value, ""); + return SizeToRawTag(sizeof(U)); + } + + using KnownTypes = + TypeList, std::array, + std::array, std::array, + FileDescriptorTables, SourceCodeInfo, FileOptions, + MessageOptions, FieldOptions, ExtensionRangeOptions, + OneofOptions, EnumOptions, EnumValueOptions, ServiceOptions, + MethodOptions>; + static constexpr Tag kFirstRawTag = KnownTypes::kSize; + + + struct DestroyVisitor { + template + void operator()(T* p) { + p->~T(); + } + void operator()(OutOfLineAlloc* p) { OperatorDelete(p->ptr, p->size); } + }; + + static uint32_t SizeToRawTag(size_t n) { return (RoundUp(n) / 8) - 1; } + + static uint32_t TagToSize(Tag tag) { + GOOGLE_DCHECK_GE(tag, kFirstRawTag); + return static_cast(tag - kFirstRawTag + 1) * 8; + } + + struct Block { + uint16_t start_offset; + uint16_t end_offset; + uint16_t capacity; + Block* next; + + // `allocated_size` is the total size of the memory block allocated. + // The `Block` structure is constructed at the start and the rest of the + // memory is used as the payload of the `Block`. + explicit Block(uint32_t allocated_size) { + start_offset = 0; + end_offset = capacity = + reinterpret_cast(this) + allocated_size - data(); + next = nullptr; + } + + char* data() { + return reinterpret_cast(this) + RoundUp(sizeof(Block)); + } + + uint32_t memory_used() { + return data() + capacity - reinterpret_cast(this); + } + uint32_t space_left() const { return end_offset - start_offset; } + + void* Allocate(uint32_t n, Tag tag) { + GOOGLE_DCHECK_LE(n + 1, space_left()); + void* p = data() + start_offset; + start_offset += n; + data()[--end_offset] = tag; + return p; + } + + void Destroy() { OperatorDelete(this, memory_used()); } + + void PrependTo(Block*& list) { + next = list; + list = this; + } + + template + void VisitBlock(Visitor visit) { + for (uint16_t s = start_offset, e = end_offset; s != 0;) { + VisitAlloc(data(), &s, &e, visit, KnownTypes{}); + } + } + }; + + Block* PopBlock(Block*& list) { + Block* res = list; + list = list->next; + return res; + } + + void RelocateToUsedList(Block* to_relocate) { + if (current_ == nullptr) { + current_ = to_relocate; + current_->next = nullptr; + return; + } else if (current_->space_left() < to_relocate->space_left()) { + std::swap(current_, to_relocate); + current_->next = nullptr; + } + + for (int i = kSmallSizes.size(); --i >= 0;) { + if (to_relocate->space_left() >= 1 + kSmallSizes[i]) { + to_relocate->PrependTo(small_size_blocks_[i]); + return; + } + } + + to_relocate->PrependTo(full_blocks_); + } + + static constexpr std::array kSmallSizes = { + {// Sizes for pointer arrays. + 8, 16, 24, 32, + // Sizes for string arrays (for descriptor names). + // The most common array sizes are 2 and 3. + 2 * sizeof(std::string), 3 * sizeof(std::string)}}; + + // Helper function to iterate all lists. + std::array GetLists() const { + std::array res; + res[0] = current_; + res[1] = full_blocks_; + std::copy(small_size_blocks_.begin(), small_size_blocks_.end(), &res[2]); + return res; + } + + Block* current_ = nullptr; + std::array small_size_blocks_ = {{}}; + Block* full_blocks_ = nullptr; + + size_t num_allocations_ = 0; + struct RollbackInfo { + Block* block; + size_t count; + }; + std::vector rollback_info_; +}; + +constexpr std::array TableArena::kSmallSizes; + } // anonymous namespace // =================================================================== @@ -556,42 +1095,47 @@ class DescriptorPool::Tables { // The stack of files which are currently being built. Used to detect // cyclic dependencies when loading files from a DescriptorDatabase. Not - // used when fallback_database_ == NULL. - std::vector pending_files_; + // used when fallback_database_ == nullptr. + std::vector pending_files_; // A set of files which we have tried to load from the fallback database // and encountered errors. We will not attempt to load them again during // execution of the current public API call, but for compatibility with // legacy clients, this is cleared at the beginning of each public API call. - // Not used when fallback_database_ == NULL. - hash_set known_bad_files_; + // Not used when fallback_database_ == nullptr. + HASH_SET known_bad_files_; // A set of symbols which we have tried to load from the fallback database // and encountered errors. We will not attempt to load them again during // execution of the current public API call, but for compatibility with // legacy clients, this is cleared at the beginning of each public API call. - hash_set known_bad_symbols_; + HASH_SET known_bad_symbols_; // The set of descriptors for which we've already loaded the full // set of extensions numbers from fallback_database_. - hash_set extensions_loaded_from_db_; + HASH_SET extensions_loaded_from_db_; + + // Maps type name to Descriptor::WellKnownType. This is logically global + // and const, but we make it a member here to simplify its construction and + // destruction. This only has 20-ish entries and is one per DescriptorPool, + // so the overhead is small. + HASH_MAP well_known_types_; // ----------------------------------------------------------------- // Finding items. // Find symbols. This returns a null Symbol (symbol.IsNull() is true) // if not found. - inline Symbol FindSymbol(const string& key) const; + inline Symbol FindSymbol(StringPiece key) const; // This implements the body of DescriptorPool::Find*ByName(). It should // really be a private method of DescriptorPool, but that would require // declaring Symbol in descriptor.h, which would drag all kinds of other // stuff into the header. Yay C++. - Symbol FindByNameHelper( - const DescriptorPool* pool, const string& name); + Symbol FindByNameHelper(const DescriptorPool* pool, StringPiece name); - // These return NULL if not found. - inline const FileDescriptor* FindFile(const string& key) const; + // These return nullptr if not found. + inline const FileDescriptor* FindFile(StringPiece key) const; inline const FieldDescriptor* FindExtension(const Descriptor* extendee, int number) const; inline void FindAllExtensions(const Descriptor* extendee, @@ -604,7 +1148,7 @@ class DescriptorPool::Tables { // the key already exists in the table. For AddSymbol(), the string passed // in must be one that was constructed using AllocateString(), as it will // be used as a key in the symbols_by_name_ map without copying. - bool AddSymbol(const string& full_name, Symbol symbol); + bool AddSymbol(const std::string& full_name, Symbol symbol); bool AddFile(const FileDescriptor* file); bool AddExtension(const FieldDescriptor* field); @@ -615,67 +1159,84 @@ class DescriptorPool::Tables { // destroyed. Note that the object's destructor will never be called, // so its fields must be plain old data (primitive data types and // pointers). All of the descriptor types are such objects. - template Type* Allocate(); + template + Type* Allocate(); // Allocate an array of objects which will be reclaimed when the // pool in destroyed. Again, destructors are never called. - template Type* AllocateArray(int count); + template + Type* AllocateArray(int count); // Allocate a string which will be destroyed when the pool is destroyed. // The string is initialized to the given value for convenience. - string* AllocateString(const string& value); + const std::string* AllocateString(StringPiece value); - // Allocate a GoogleOnceDynamic which will be destroyed when the pool is - // destroyed. - GoogleOnceDynamic* AllocateOnceDynamic(); + // Copy the input into a NUL terminated string whose lifetime is managed by + // the pool. + const char* Strdup(StringPiece value); + + // Allocates an array of strings which will be destroyed when the pool is + // destroyed. The array is initialized with the input values. + template + const std::string* AllocateStringArray(In&&... values); + + struct FieldNamesResult { + std::string* array; + int lowercase_index; + int camelcase_index; + int json_index; + }; + // Allocate all 5 names of the field: + // name, full name, lowercase, camelcase and json. + // This function will dedup the strings when possible. + // The resulting array contains `name` at index 0, `full_name` at index 1 and + // the other 3 indices are specified in the result. + FieldNamesResult AllocateFieldNames(const std::string& name, + const std::string& scope, + const std::string* opt_json_name); + + // Create an object that will be deleted when the pool is destroyed. + // The object is value initialized, and its destructor will be called if + // non-trivial. + template + Type* Create(); // Allocate a protocol message object. Some older versions of GCC have // trouble understanding explicit template instantiations in some cases, so // in those cases we have to pass a dummy pointer of the right type as the // parameter instead of specifying the type explicitly. - template Type* AllocateMessage(Type* dummy = NULL); + template + Type* AllocateMessage(Type* dummy = nullptr); // Allocate a FileDescriptorTables object. FileDescriptorTables* AllocateFileTables(); private: - std::vector strings_; // All strings in the pool. - std::vector messages_; // All messages in the pool. - std::vector - once_dynamics_; // All GoogleOnceDynamics in the pool. - std::vector - file_tables_; // All file tables in the pool. - std::vector allocations_; // All other memory allocated in the pool. + // All other memory allocated in the pool. Must be first as other objects can + // point into these. + TableArena arena_; - SymbolsByNameMap symbols_by_name_; - FilesByNameMap files_by_name_; + SymbolsByNameSet symbols_by_name_; + FilesByNameMap files_by_name_; ExtensionsGroupedByDescriptorMap extensions_; struct CheckPoint { explicit CheckPoint(const Tables* tables) - : strings_before_checkpoint(tables->strings_.size()), - messages_before_checkpoint(tables->messages_.size()), - once_dynamics_before_checkpoint(tables->once_dynamics_.size()), - file_tables_before_checkpoint(tables->file_tables_.size()), - allocations_before_checkpoint(tables->allocations_.size()), + : arena_before_checkpoint(tables->arena_.num_allocations()), pending_symbols_before_checkpoint( tables->symbols_after_checkpoint_.size()), pending_files_before_checkpoint( tables->files_after_checkpoint_.size()), pending_extensions_before_checkpoint( tables->extensions_after_checkpoint_.size()) {} - int strings_before_checkpoint; - int messages_before_checkpoint; - int once_dynamics_before_checkpoint; - int file_tables_before_checkpoint; - int allocations_before_checkpoint; + int arena_before_checkpoint; int pending_symbols_before_checkpoint; int pending_files_before_checkpoint; int pending_extensions_before_checkpoint; }; std::vector checkpoints_; - std::vector symbols_after_checkpoint_; - std::vector files_after_checkpoint_; + std::vector symbols_after_checkpoint_; + std::vector files_after_checkpoint_; std::vector extensions_after_checkpoint_; // Allocate some bytes which will be reclaimed when the pool is @@ -703,23 +1264,19 @@ class FileDescriptorTables { // ----------------------------------------------------------------- // Finding items. - // Find symbols. These return a null Symbol (symbol.IsNull() is true) - // if not found. + // Returns a null Symbol (symbol.IsNull() is true) if not found. inline Symbol FindNestedSymbol(const void* parent, - const string& name) const; - inline Symbol FindNestedSymbolOfType(const void* parent, - const string& name, - const Symbol::Type type) const; + StringPiece name) const; - // These return NULL if not found. - inline const FieldDescriptor* FindFieldByNumber( - const Descriptor* parent, int number) const; + // These return nullptr if not found. + inline const FieldDescriptor* FindFieldByNumber(const Descriptor* parent, + int number) const; inline const FieldDescriptor* FindFieldByLowercaseName( - const void* parent, const string& lowercase_name) const; + const void* parent, StringPiece lowercase_name) const; inline const FieldDescriptor* FindFieldByCamelcaseName( - const void* parent, const string& camelcase_name) const; + const void* parent, StringPiece camelcase_name) const; inline const EnumValueDescriptor* FindEnumValueByNumber( - const EnumDescriptor* parent, int number) const; + const EnumDescriptor* parent, int number) const; // This creates a new EnumValueDescriptor if not found, in a thread-safe way. inline const EnumValueDescriptor* FindEnumValueByNumberCreatingIfUnknown( const EnumDescriptor* parent, int number) const; @@ -731,22 +1288,22 @@ class FileDescriptorTables { // the key already exists in the table. For AddAliasUnderParent(), the // string passed in must be one that was constructed using AllocateString(), // as it will be used as a key in the symbols_by_parent_ map without copying. - bool AddAliasUnderParent(const void* parent, const string& name, + bool AddAliasUnderParent(const void* parent, const std::string& name, Symbol symbol); - bool AddFieldByNumber(const FieldDescriptor* field); - bool AddEnumValueByNumber(const EnumValueDescriptor* value); + bool AddFieldByNumber(FieldDescriptor* field); + bool AddEnumValueByNumber(EnumValueDescriptor* value); // Adds the field to the lowercase_name and camelcase_name maps. Never // fails because we allow duplicates; the first field by the name wins. void AddFieldByStylizedNames(const FieldDescriptor* field); // Populates p->first->locations_by_path_ from p->second. - // Unusual signature dictated by GoogleOnceDynamic. + // Unusual signature dictated by internal::call_once. static void BuildLocationsByPath( std::pair* p); // Returns the location denoted by the specified path through info, - // or NULL if not found. + // or nullptr if not found. // The value of info must be that of the corresponding FileDescriptor. // (Conceptually a pure function, but stateful as an optimisation.) const SourceCodeInfo_Location* GetSourceLocation( @@ -765,88 +1322,60 @@ class FileDescriptorTables { const FileDescriptorTables* tables); void FieldsByCamelcaseNamesLazyInitInternal() const; - SymbolsByParentMap symbols_by_parent_; + SymbolsByParentSet symbols_by_parent_; mutable FieldsByNameMap fields_by_lowercase_name_; - mutable FieldsByNameMap* fields_by_lowercase_name_tmp_; - mutable GoogleOnceDynamic fields_by_lowercase_name_once_; + std::unique_ptr fields_by_lowercase_name_tmp_; + mutable internal::once_flag fields_by_lowercase_name_once_; mutable FieldsByNameMap fields_by_camelcase_name_; - mutable FieldsByNameMap* fields_by_camelcase_name_tmp_; - mutable GoogleOnceDynamic fields_by_camelcase_name_once_; - FieldsByNumberMap fields_by_number_; // Not including extensions. - EnumValuesByNumberMap enum_values_by_number_; - mutable EnumValuesByNumberMap unknown_enum_values_by_number_ - GOOGLE_GUARDED_BY(unknown_enum_values_mu_); + std::unique_ptr fields_by_camelcase_name_tmp_; + mutable internal::once_flag fields_by_camelcase_name_once_; + FieldsByNumberSet fields_by_number_; // Not including extensions. + EnumValuesByNumberSet enum_values_by_number_; + mutable EnumValuesByNumberSet unknown_enum_values_by_number_ + PROTOBUF_GUARDED_BY(unknown_enum_values_mu_); // Populated on first request to save space, hence constness games. - mutable GoogleOnceDynamic locations_by_path_once_; + mutable internal::once_flag locations_by_path_once_; mutable LocationsByPathMap locations_by_path_; // Mutex to protect the unknown-enum-value map due to dynamic // EnumValueDescriptor creation on unknown values. - mutable Mutex unknown_enum_values_mu_; + mutable internal::WrappedMutex unknown_enum_values_mu_; }; -DescriptorPool::Tables::Tables() - // Start some hash_map and hash_set objects with a small # of buckets - : known_bad_files_(3), - known_bad_symbols_(3), - extensions_loaded_from_db_(3), - symbols_by_name_(3), - files_by_name_(3) {} - - -DescriptorPool::Tables::~Tables() { - GOOGLE_DCHECK(checkpoints_.empty()); - // Note that the deletion order is important, since the destructors of some - // messages may refer to objects in allocations_. - STLDeleteElements(&messages_); - for (int i = 0; i < allocations_.size(); i++) { - operator delete(allocations_[i]); - } - STLDeleteElements(&strings_); - STLDeleteElements(&file_tables_); - STLDeleteElements(&once_dynamics_); +DescriptorPool::Tables::Tables() { + well_known_types_.insert({ + {"google.protobuf.DoubleValue", Descriptor::WELLKNOWNTYPE_DOUBLEVALUE}, + {"google.protobuf.FloatValue", Descriptor::WELLKNOWNTYPE_FLOATVALUE}, + {"google.protobuf.Int64Value", Descriptor::WELLKNOWNTYPE_INT64VALUE}, + {"google.protobuf.UInt64Value", Descriptor::WELLKNOWNTYPE_UINT64VALUE}, + {"google.protobuf.Int32Value", Descriptor::WELLKNOWNTYPE_INT32VALUE}, + {"google.protobuf.UInt32Value", Descriptor::WELLKNOWNTYPE_UINT32VALUE}, + {"google.protobuf.StringValue", Descriptor::WELLKNOWNTYPE_STRINGVALUE}, + {"google.protobuf.BytesValue", Descriptor::WELLKNOWNTYPE_BYTESVALUE}, + {"google.protobuf.BoolValue", Descriptor::WELLKNOWNTYPE_BOOLVALUE}, + {"google.protobuf.Any", Descriptor::WELLKNOWNTYPE_ANY}, + {"google.protobuf.FieldMask", Descriptor::WELLKNOWNTYPE_FIELDMASK}, + {"google.protobuf.Duration", Descriptor::WELLKNOWNTYPE_DURATION}, + {"google.protobuf.Timestamp", Descriptor::WELLKNOWNTYPE_TIMESTAMP}, + {"google.protobuf.Value", Descriptor::WELLKNOWNTYPE_VALUE}, + {"google.protobuf.ListValue", Descriptor::WELLKNOWNTYPE_LISTVALUE}, + {"google.protobuf.Struct", Descriptor::WELLKNOWNTYPE_STRUCT}, + }); } +DescriptorPool::Tables::~Tables() { GOOGLE_DCHECK(checkpoints_.empty()); } + FileDescriptorTables::FileDescriptorTables() - // Initialize all the hash tables to start out with a small # of buckets. - : symbols_by_parent_(3), - fields_by_lowercase_name_(3), - fields_by_lowercase_name_tmp_(new FieldsByNameMap()), - fields_by_camelcase_name_(3), - fields_by_camelcase_name_tmp_(new FieldsByNameMap()), - fields_by_number_(3), - enum_values_by_number_(3), - unknown_enum_values_by_number_(3), - locations_by_path_(3) {} + : fields_by_lowercase_name_tmp_(new FieldsByNameMap()), + fields_by_camelcase_name_tmp_(new FieldsByNameMap()) {} FileDescriptorTables::~FileDescriptorTables() {} -namespace { - -FileDescriptorTables* file_descriptor_tables_ = NULL; -GOOGLE_PROTOBUF_DECLARE_ONCE(file_descriptor_tables_once_init_); - -void DeleteFileDescriptorTables() { - delete file_descriptor_tables_; - file_descriptor_tables_ = NULL; -} - -void InitFileDescriptorTables() { - file_descriptor_tables_ = new FileDescriptorTables(); - internal::OnShutdown(&DeleteFileDescriptorTables); -} - -inline void InitFileDescriptorTablesOnce() { - ::google::protobuf::GoogleOnceInit( - &file_descriptor_tables_once_init_, &InitFileDescriptorTables); -} - -} // anonymous namespace - inline const FileDescriptorTables& FileDescriptorTables::GetEmptyInstance() { - InitFileDescriptorTablesOnce(); - return *file_descriptor_tables_; + static auto file_descriptor_tables = + internal::OnShutdownDelete(new FileDescriptorTables()); + return *file_descriptor_tables; } void DescriptorPool::Tables::AddCheckpoint() { @@ -862,6 +1391,7 @@ void DescriptorPool::Tables::ClearLastCheckpoint() { symbols_after_checkpoint_.clear(); files_after_checkpoint_.clear(); extensions_after_checkpoint_.clear(); + arena_.ClearRollbackData(); } } @@ -869,19 +1399,18 @@ void DescriptorPool::Tables::RollbackToLastCheckpoint() { GOOGLE_DCHECK(!checkpoints_.empty()); const CheckPoint& checkpoint = checkpoints_.back(); - for (int i = checkpoint.pending_symbols_before_checkpoint; - i < symbols_after_checkpoint_.size(); - i++) { - symbols_by_name_.erase(symbols_after_checkpoint_[i]); + for (size_t i = checkpoint.pending_symbols_before_checkpoint; + i < symbols_after_checkpoint_.size(); i++) { + Symbol::QueryKey name; + name.name = symbols_after_checkpoint_[i]; + symbols_by_name_.erase(Symbol(&name)); } - for (int i = checkpoint.pending_files_before_checkpoint; - i < files_after_checkpoint_.size(); - i++) { + for (size_t i = checkpoint.pending_files_before_checkpoint; + i < files_after_checkpoint_.size(); i++) { files_by_name_.erase(files_after_checkpoint_[i]); } - for (int i = checkpoint.pending_extensions_before_checkpoint; - i < extensions_after_checkpoint_.size(); - i++) { + for (size_t i = checkpoint.pending_extensions_before_checkpoint; + i < extensions_after_checkpoint_.size(); i++) { extensions_.erase(extensions_after_checkpoint_[i]); } @@ -891,71 +1420,48 @@ void DescriptorPool::Tables::RollbackToLastCheckpoint() { extensions_after_checkpoint_.resize( checkpoint.pending_extensions_before_checkpoint); - STLDeleteContainerPointers( - strings_.begin() + checkpoint.strings_before_checkpoint, strings_.end()); - STLDeleteContainerPointers( - messages_.begin() + checkpoint.messages_before_checkpoint, - messages_.end()); - STLDeleteContainerPointers( - once_dynamics_.begin() + checkpoint.once_dynamics_before_checkpoint, - once_dynamics_.end()); - STLDeleteContainerPointers( - file_tables_.begin() + checkpoint.file_tables_before_checkpoint, - file_tables_.end()); - for (int i = checkpoint.allocations_before_checkpoint; - i < allocations_.size(); - i++) { - operator delete(allocations_[i]); - } - - strings_.resize(checkpoint.strings_before_checkpoint); - messages_.resize(checkpoint.messages_before_checkpoint); - once_dynamics_.resize(checkpoint.once_dynamics_before_checkpoint); - file_tables_.resize(checkpoint.file_tables_before_checkpoint); - allocations_.resize(checkpoint.allocations_before_checkpoint); + arena_.RollbackTo(checkpoint.arena_before_checkpoint); checkpoints_.pop_back(); } // ------------------------------------------------------------------- -inline Symbol DescriptorPool::Tables::FindSymbol(const string& key) const { - const Symbol* result = FindOrNull(symbols_by_name_, key.c_str()); - if (result == NULL) { - return kNullSymbol; - } else { - return *result; - } +inline Symbol DescriptorPool::Tables::FindSymbol(StringPiece key) const { + Symbol::QueryKey name; + name.name = key; + auto it = symbols_by_name_.find(Symbol(&name)); + return it == symbols_by_name_.end() ? kNullSymbol : *it; } inline Symbol FileDescriptorTables::FindNestedSymbol( - const void* parent, const string& name) const { - const Symbol* result = - FindOrNull(symbols_by_parent_, PointerStringPair(parent, name.c_str())); - if (result == NULL) { - return kNullSymbol; - } else { - return *result; + const void* parent, StringPiece name) const { + Symbol::QueryKey query; + query.name = name; + query.parent = parent; + auto it = symbols_by_parent_.find(Symbol(&query)); + return it == symbols_by_parent_.end() ? kNullSymbol : *it; +} + +Symbol DescriptorPool::Tables::FindByNameHelper(const DescriptorPool* pool, + StringPiece name) { + if (pool->mutex_ != nullptr) { + // Fast path: the Symbol is already cached. This is just a hash lookup. + ReaderMutexLock lock(pool->mutex_); + if (known_bad_symbols_.empty() && known_bad_files_.empty()) { + Symbol result = FindSymbol(name); + if (!result.IsNull()) return result; + } } -} - -inline Symbol FileDescriptorTables::FindNestedSymbolOfType( - const void* parent, const string& name, const Symbol::Type type) const { - Symbol result = FindNestedSymbol(parent, name); - if (result.type != type) return kNullSymbol; - return result; -} - -Symbol DescriptorPool::Tables::FindByNameHelper( - const DescriptorPool* pool, const string& name) { MutexLockMaybe lock(pool->mutex_); - known_bad_symbols_.clear(); - known_bad_files_.clear(); + if (pool->fallback_database_ != nullptr) { + known_bad_symbols_.clear(); + known_bad_files_.clear(); + } Symbol result = FindSymbol(name); - if (result.IsNull() && pool->underlay_ != NULL) { + if (result.IsNull() && pool->underlay_ != nullptr) { // Symbol not found; check the underlay. - result = - pool->underlay_->tables_->FindByNameHelper(pool->underlay_, name); + result = pool->underlay_->tables_->FindByNameHelper(pool->underlay_, name); } if (result.IsNull()) { @@ -969,19 +1475,31 @@ Symbol DescriptorPool::Tables::FindByNameHelper( } inline const FileDescriptor* DescriptorPool::Tables::FindFile( - const string& key) const { - return FindPtrOrNull(files_by_name_, key.c_str()); + StringPiece key) const { + return FindPtrOrNull(files_by_name_, key); } inline const FieldDescriptor* FileDescriptorTables::FindFieldByNumber( const Descriptor* parent, int number) const { - return FindPtrOrNull(fields_by_number_, std::make_pair(parent, number)); + // If `number` is within the sequential range, just index into the parent + // without doing a table lookup. + if (parent != nullptr && // + 1 <= number && number <= parent->sequential_field_limit_) { + return parent->field(number - 1); + } + + Symbol::QueryKey query; + query.parent = parent; + query.field_number = number; + + auto it = fields_by_number_.find(Symbol(&query)); + return it == fields_by_number_.end() ? nullptr : it->field_descriptor(); } const void* FileDescriptorTables::FindParentForFieldsByMap( const FieldDescriptor* field) const { if (field->is_extension()) { - if (field->extension_scope() == NULL) { + if (field->extension_scope() == nullptr) { return field->file(); } else { return field->extension_scope(); @@ -997,20 +1515,22 @@ void FileDescriptorTables::FieldsByLowercaseNamesLazyInitStatic( } void FileDescriptorTables::FieldsByLowercaseNamesLazyInitInternal() const { - for (FieldsByNumberMap::const_iterator it = fields_by_number_.begin(); - it != fields_by_number_.end(); it++) { - PointerStringPair lowercase_key(FindParentForFieldsByMap(it->second), - it->second->lowercase_name().c_str()); - InsertIfNotPresent(&fields_by_lowercase_name_, lowercase_key, it->second); + for (Symbol symbol : symbols_by_parent_) { + const FieldDescriptor* field = symbol.field_descriptor(); + if (!field) continue; + PointerStringPair lowercase_key(FindParentForFieldsByMap(field), + field->lowercase_name().c_str()); + InsertIfNotPresent(&fields_by_lowercase_name_, lowercase_key, field); } } inline const FieldDescriptor* FileDescriptorTables::FindFieldByLowercaseName( - const void* parent, const string& lowercase_name) const { - fields_by_lowercase_name_once_.Init( + const void* parent, StringPiece lowercase_name) const { + internal::call_once( + fields_by_lowercase_name_once_, &FileDescriptorTables::FieldsByLowercaseNamesLazyInitStatic, this); return FindPtrOrNull(fields_by_lowercase_name_, - PointerStringPair(parent, lowercase_name.c_str())); + PointerStringPair(parent, lowercase_name)); } void FileDescriptorTables::FieldsByCamelcaseNamesLazyInitStatic( @@ -1019,25 +1539,41 @@ void FileDescriptorTables::FieldsByCamelcaseNamesLazyInitStatic( } void FileDescriptorTables::FieldsByCamelcaseNamesLazyInitInternal() const { - for (FieldsByNumberMap::const_iterator it = fields_by_number_.begin(); - it != fields_by_number_.end(); it++) { - PointerStringPair camelcase_key(FindParentForFieldsByMap(it->second), - it->second->camelcase_name().c_str()); - InsertIfNotPresent(&fields_by_camelcase_name_, camelcase_key, it->second); + for (Symbol symbol : symbols_by_parent_) { + const FieldDescriptor* field = symbol.field_descriptor(); + if (!field) continue; + PointerStringPair camelcase_key(FindParentForFieldsByMap(field), + field->camelcase_name().c_str()); + InsertIfNotPresent(&fields_by_camelcase_name_, camelcase_key, field); } } inline const FieldDescriptor* FileDescriptorTables::FindFieldByCamelcaseName( - const void* parent, const string& camelcase_name) const { - fields_by_camelcase_name_once_.Init( - &FileDescriptorTables::FieldsByCamelcaseNamesLazyInitStatic, this); + const void* parent, StringPiece camelcase_name) const { + internal::call_once( + fields_by_camelcase_name_once_, + FileDescriptorTables::FieldsByCamelcaseNamesLazyInitStatic, this); return FindPtrOrNull(fields_by_camelcase_name_, - PointerStringPair(parent, camelcase_name.c_str())); + PointerStringPair(parent, camelcase_name)); } inline const EnumValueDescriptor* FileDescriptorTables::FindEnumValueByNumber( const EnumDescriptor* parent, int number) const { - return FindPtrOrNull(enum_values_by_number_, std::make_pair(parent, number)); + // If `number` is within the sequential range, just index into the parent + // without doing a table lookup. + const int base = parent->value(0)->number(); + if (base <= number && + number <= static_cast(base) + parent->sequential_value_limit_) { + return parent->value(number - base); + } + + Symbol::QueryKey query; + query.parent = parent; + query.field_number = number; + + auto it = enum_values_by_number_.find(Symbol(&query)); + return it == enum_values_by_number_.end() ? nullptr + : it->enum_value_descriptor(); } inline const EnumValueDescriptor* @@ -1045,54 +1581,60 @@ FileDescriptorTables::FindEnumValueByNumberCreatingIfUnknown( const EnumDescriptor* parent, int number) const { // First try, with map of compiled-in values. { - const EnumValueDescriptor* desc = - FindPtrOrNull(enum_values_by_number_, std::make_pair(parent, number)); - if (desc != NULL) { - return desc; + const auto* value = FindEnumValueByNumber(parent, number); + if (value != nullptr) { + return value; } } + + Symbol::QueryKey query; + query.parent = parent; + query.field_number = number; + // Second try, with reader lock held on unknown enum values: common case. { ReaderMutexLock l(&unknown_enum_values_mu_); - const EnumValueDescriptor* desc = FindPtrOrNull( - unknown_enum_values_by_number_, std::make_pair(parent, number)); - if (desc != NULL) { - return desc; + auto it = unknown_enum_values_by_number_.find(Symbol(&query)); + if (it != unknown_enum_values_by_number_.end() && + it->enum_value_descriptor() != nullptr) { + return it->enum_value_descriptor(); } } // If not found, try again with writer lock held, and create new descriptor if // necessary. { WriterMutexLock l(&unknown_enum_values_mu_); - const EnumValueDescriptor* desc = FindPtrOrNull( - unknown_enum_values_by_number_, std::make_pair(parent, number)); - if (desc != NULL) { - return desc; + auto it = unknown_enum_values_by_number_.find(Symbol(&query)); + if (it != unknown_enum_values_by_number_.end() && + it->enum_value_descriptor() != nullptr) { + return it->enum_value_descriptor(); } // Create an EnumValueDescriptor dynamically. We don't insert it into the // EnumDescriptor (it's not a part of the enum as originally defined), but // we do insert it into the table so that we can return the same pointer // later. - string enum_value_name = StringPrintf( - "UNKNOWN_ENUM_VALUE_%s_%d", parent->name().c_str(), number); - DescriptorPool::Tables* tables = - const_cast(DescriptorPool::generated_pool()-> - tables_.get()); - EnumValueDescriptor* result = tables->Allocate(); - result->name_ = tables->AllocateString(enum_value_name); - result->full_name_ = tables->AllocateString(parent->full_name() + - "." + enum_value_name); + std::string enum_value_name = StringPrintf("UNKNOWN_ENUM_VALUE_%s_%d", + parent->name().c_str(), number); + auto* pool = DescriptorPool::generated_pool(); + auto* tables = const_cast(pool->tables_.get()); + EnumValueDescriptor* result; + { + // Must lock the pool because we will do allocations in the shared arena. + MutexLockMaybe l2(pool->mutex_); + result = tables->Allocate(); + result->all_names_ = tables->AllocateStringArray( + enum_value_name, + StrCat(parent->full_name(), ".", enum_value_name)); + } result->number_ = number; result->type_ = parent; result->options_ = &EnumValueOptions::default_instance(); - InsertIfNotPresent(&unknown_enum_values_by_number_, - std::make_pair(parent, number), result); + unknown_enum_values_by_number_.insert(Symbol::EnumValue(result, 0)); return result; } } - inline const FieldDescriptor* DescriptorPool::Tables::FindExtension( const Descriptor* extendee, int number) const { return FindPtrOrNull(extensions_, std::make_pair(extendee, number)); @@ -1110,9 +1652,10 @@ inline void DescriptorPool::Tables::FindAllExtensions( // ------------------------------------------------------------------- -bool DescriptorPool::Tables::AddSymbol( - const string& full_name, Symbol symbol) { - if (InsertIfNotPresent(&symbols_by_name_, full_name.c_str(), symbol)) { +bool DescriptorPool::Tables::AddSymbol(const std::string& full_name, + Symbol symbol) { + GOOGLE_DCHECK_EQ(full_name, symbol.full_name()); + if (symbols_by_name_.insert(symbol).second) { symbols_after_checkpoint_.push_back(full_name.c_str()); return true; } else { @@ -1120,14 +1663,16 @@ bool DescriptorPool::Tables::AddSymbol( } } -bool FileDescriptorTables::AddAliasUnderParent( - const void* parent, const string& name, Symbol symbol) { - PointerStringPair by_parent_key(parent, name.c_str()); - return InsertIfNotPresent(&symbols_by_parent_, by_parent_key, symbol); +bool FileDescriptorTables::AddAliasUnderParent(const void* parent, + const std::string& name, + Symbol symbol) { + GOOGLE_DCHECK_EQ(name, symbol.parent_name_key().second); + GOOGLE_DCHECK_EQ(parent, symbol.parent_name_key().first); + return symbols_by_parent_.insert(symbol).second; } bool DescriptorPool::Tables::AddFile(const FileDescriptor* file) { - if (InsertIfNotPresent(&files_by_name_, file->name().c_str(), file)) { + if (InsertIfNotPresent(&files_by_name_, file->name(), file)) { files_after_checkpoint_.push_back(file->name().c_str()); return true; } else { @@ -1137,10 +1682,8 @@ bool DescriptorPool::Tables::AddFile(const FileDescriptor* file) { void FileDescriptorTables::FinalizeTables() { // Clean up the temporary maps used by AddFieldByStylizedNames(). - delete fields_by_lowercase_name_tmp_; - fields_by_lowercase_name_tmp_ = NULL; - delete fields_by_camelcase_name_tmp_; - fields_by_camelcase_name_tmp_ = NULL; + fields_by_lowercase_name_tmp_ = nullptr; + fields_by_camelcase_name_tmp_ = nullptr; } void FileDescriptorTables::AddFieldByStylizedNames( @@ -1155,31 +1698,46 @@ void FileDescriptorTables::AddFieldByStylizedNames( // entries from fields_by_number_. PointerStringPair lowercase_key(parent, field->lowercase_name().c_str()); - if (!InsertIfNotPresent(fields_by_lowercase_name_tmp_, lowercase_key, - field)) { + if (!InsertIfNotPresent(fields_by_lowercase_name_tmp_.get(), + lowercase_key, field)) { InsertIfNotPresent( &fields_by_lowercase_name_, lowercase_key, FindPtrOrNull(*fields_by_lowercase_name_tmp_, lowercase_key)); } PointerStringPair camelcase_key(parent, field->camelcase_name().c_str()); - if (!InsertIfNotPresent(fields_by_camelcase_name_tmp_, camelcase_key, - field)) { + if (!InsertIfNotPresent(fields_by_camelcase_name_tmp_.get(), + camelcase_key, field)) { InsertIfNotPresent( &fields_by_camelcase_name_, camelcase_key, FindPtrOrNull(*fields_by_camelcase_name_tmp_, camelcase_key)); } } -bool FileDescriptorTables::AddFieldByNumber(const FieldDescriptor* field) { - DescriptorIntPair key(field->containing_type(), field->number()); - return InsertIfNotPresent(&fields_by_number_, key, field); +bool FileDescriptorTables::AddFieldByNumber(FieldDescriptor* field) { + // Skip fields that are at the start of the sequence. + if (field->containing_type() != nullptr && field->number() >= 1 && + field->number() <= field->containing_type()->sequential_field_limit_) { + if (field->is_extension()) { + // Conflicts with the field that already exists in the sequential range. + return false; + } + // Only return true if the field at that index matches. Otherwise it + // conflicts with the existing field in the sequential range. + return field->containing_type()->field(field->number() - 1) == field; + } + + return fields_by_number_.insert(Symbol(field)).second; } -bool FileDescriptorTables::AddEnumValueByNumber( - const EnumValueDescriptor* value) { - EnumIntPair key(value->type(), value->number()); - return InsertIfNotPresent(&enum_values_by_number_, key, value); +bool FileDescriptorTables::AddEnumValueByNumber(EnumValueDescriptor* value) { + // Skip values that are at the start of the sequence. + const int base = value->type()->value(0)->number(); + if (base <= value->number() && + value->number() <= + static_cast(base) + value->type()->sequential_value_limit_) + return true; + return enum_values_by_number_.insert(Symbol::EnumValue(value, 0)).second; } bool DescriptorPool::Tables::AddExtension(const FieldDescriptor* field) { @@ -1194,51 +1752,125 @@ bool DescriptorPool::Tables::AddExtension(const FieldDescriptor* field) { // ------------------------------------------------------------------- -template +template Type* DescriptorPool::Tables::Allocate() { return reinterpret_cast(AllocateBytes(sizeof(Type))); } -template +template Type* DescriptorPool::Tables::AllocateArray(int count) { return reinterpret_cast(AllocateBytes(sizeof(Type) * count)); } -string* DescriptorPool::Tables::AllocateString(const string& value) { - string* result = new string(value); - strings_.push_back(result); +const std::string* DescriptorPool::Tables::AllocateString( + StringPiece value) { + return arena_.Create(value); +} + +const char* DescriptorPool::Tables::Strdup(StringPiece value) { + char* p = AllocateArray(static_cast(value.size() + 1)); + memcpy(p, value.data(), value.size()); + p[value.size()] = 0; + return p; +} + +template +const std::string* DescriptorPool::Tables::AllocateStringArray(In&&... values) { + auto& array = *arena_.Create>(); + array = {{std::string(std::forward(values))...}}; + return array.data(); +} + +DescriptorPool::Tables::FieldNamesResult +DescriptorPool::Tables::AllocateFieldNames(const std::string& name, + const std::string& scope, + const std::string* opt_json_name) { + std::string lowercase_name = name; + LowerString(&lowercase_name); + + std::string camelcase_name = ToCamelCase(name, /* lower_first = */ true); + std::string json_name; + if (opt_json_name != nullptr) { + json_name = *opt_json_name; + } else { + json_name = ToJsonName(name); + } + + const bool lower_eq_name = lowercase_name == name; + const bool camel_eq_name = camelcase_name == name; + const bool json_eq_name = json_name == name; + const bool json_eq_camel = json_name == camelcase_name; + + const int total_count = 2 + (lower_eq_name ? 0 : 1) + + (camel_eq_name ? 0 : 1) + + (json_eq_name || json_eq_camel ? 0 : 1); + FieldNamesResult result{nullptr, 0, 0, 0}; + // We use std::array to allow handling of the destruction of the strings. + switch (total_count) { + case 2: + result.array = arena_.Create>()->data(); + break; + case 3: + result.array = arena_.Create>()->data(); + break; + case 4: + result.array = arena_.Create>()->data(); + break; + case 5: + result.array = arena_.Create>()->data(); + break; + } + + result.array[0] = name; + if (scope.empty()) { + result.array[1] = name; + } else { + result.array[1] = StrCat(scope, ".", name); + } + int index = 2; + if (lower_eq_name) { + result.lowercase_index = 0; + } else { + result.lowercase_index = index; + result.array[index++] = std::move(lowercase_name); + } + + if (camel_eq_name) { + result.camelcase_index = 0; + } else { + result.camelcase_index = index; + result.array[index++] = std::move(camelcase_name); + } + + if (json_eq_name) { + result.json_index = 0; + } else if (json_eq_camel) { + result.json_index = result.camelcase_index; + } else { + result.json_index = index; + result.array[index] = std::move(json_name); + } + return result; } -GoogleOnceDynamic* DescriptorPool::Tables::AllocateOnceDynamic() { - GoogleOnceDynamic* result = new GoogleOnceDynamic(); - once_dynamics_.push_back(result); - return result; +template +Type* DescriptorPool::Tables::Create() { + return arena_.Create(); } -template +template Type* DescriptorPool::Tables::AllocateMessage(Type* /* dummy */) { - Type* result = new Type; - messages_.push_back(result); - return result; + return arena_.Create(); } FileDescriptorTables* DescriptorPool::Tables::AllocateFileTables() { - FileDescriptorTables* result = new FileDescriptorTables; - file_tables_.push_back(result); - return result; + return arena_.Create(); } void* DescriptorPool::Tables::AllocateBytes(int size) { - // TODO(kenton): Would it be worthwhile to implement this in some more - // sophisticated way? Probably not for the open source release, but for - // internal use we could easily plug in one of our existing memory pool - // allocators... - if (size == 0) return NULL; - - void* result = operator new(size); - allocations_.push_back(result); - return result; + if (size == 0) return nullptr; + return arena_.AllocateMemory(size); } void FileDescriptorTables::BuildLocationsByPath( @@ -1253,7 +1885,8 @@ const SourceCodeInfo_Location* FileDescriptorTables::GetSourceLocation( const std::vector& path, const SourceCodeInfo* info) const { std::pair p( std::make_pair(this, info)); - locations_by_path_once_.Init(&FileDescriptorTables::BuildLocationsByPath, &p); + internal::call_once(locations_by_path_once_, + FileDescriptorTables::BuildLocationsByPath, &p); return FindPtrOrNull(locations_by_path_, Join(path, ",")); } @@ -1263,45 +1896,44 @@ const SourceCodeInfo_Location* FileDescriptorTables::GetSourceLocation( DescriptorPool::ErrorCollector::~ErrorCollector() {} DescriptorPool::DescriptorPool() - : mutex_(NULL), - fallback_database_(NULL), - default_error_collector_(NULL), - underlay_(NULL), - tables_(new Tables), - enforce_dependencies_(true), - lazily_build_dependencies_(false), - allow_unknown_(false), - enforce_weak_(false), - disallow_enforce_utf8_(false) {} + : mutex_(nullptr), + fallback_database_(nullptr), + default_error_collector_(nullptr), + underlay_(nullptr), + tables_(new Tables), + enforce_dependencies_(true), + lazily_build_dependencies_(false), + allow_unknown_(false), + enforce_weak_(false), + disallow_enforce_utf8_(false) {} DescriptorPool::DescriptorPool(DescriptorDatabase* fallback_database, ErrorCollector* error_collector) - : mutex_(new Mutex), - fallback_database_(fallback_database), - default_error_collector_(error_collector), - underlay_(NULL), - tables_(new Tables), - enforce_dependencies_(true), - lazily_build_dependencies_(false), - allow_unknown_(false), - enforce_weak_(false), - disallow_enforce_utf8_(false) { -} + : mutex_(new internal::WrappedMutex), + fallback_database_(fallback_database), + default_error_collector_(error_collector), + underlay_(nullptr), + tables_(new Tables), + enforce_dependencies_(true), + lazily_build_dependencies_(false), + allow_unknown_(false), + enforce_weak_(false), + disallow_enforce_utf8_(false) {} DescriptorPool::DescriptorPool(const DescriptorPool* underlay) - : mutex_(NULL), - fallback_database_(NULL), - default_error_collector_(NULL), - underlay_(underlay), - tables_(new Tables), - enforce_dependencies_(true), - lazily_build_dependencies_(false), - allow_unknown_(false), - enforce_weak_(false), - disallow_enforce_utf8_(false) {} + : mutex_(nullptr), + fallback_database_(nullptr), + default_error_collector_(nullptr), + underlay_(underlay), + tables_(new Tables), + enforce_dependencies_(true), + lazily_build_dependencies_(false), + allow_unknown_(false), + enforce_weak_(false), + disallow_enforce_utf8_(false) {} DescriptorPool::~DescriptorPool() { - if (mutex_ != NULL) delete mutex_; + if (mutex_ != nullptr) delete mutex_; } // DescriptorPool::BuildFile() defined later. @@ -1311,17 +1943,18 @@ void DescriptorPool::InternalDontEnforceDependencies() { enforce_dependencies_ = false; } -void DescriptorPool::AddUnusedImportTrackFile(const string& file_name) { - unused_import_track_files_.insert(file_name); +void DescriptorPool::AddUnusedImportTrackFile(ConstStringParam file_name, + bool is_error) { + unused_import_track_files_[std::string(file_name)] = is_error; } void DescriptorPool::ClearUnusedImportTrackFiles() { unused_import_track_files_.clear(); } -bool DescriptorPool::InternalIsFileLoaded(const string& filename) const { +bool DescriptorPool::InternalIsFileLoaded(ConstStringParam filename) const { MutexLockMaybe lock(mutex_); - return tables_->FindFile(filename) != NULL; + return tables_->FindFile(filename) != nullptr; } // generated_pool ==================================================== @@ -1329,43 +1962,38 @@ bool DescriptorPool::InternalIsFileLoaded(const string& filename) const { namespace { -EncodedDescriptorDatabase* generated_database_ = NULL; -DescriptorPool* generated_pool_ = NULL; -GOOGLE_PROTOBUF_DECLARE_ONCE(generated_pool_init_); - -void DeleteGeneratedPool() { - delete generated_database_; - generated_database_ = NULL; - delete generated_pool_; - generated_pool_ = NULL; +EncodedDescriptorDatabase* GeneratedDatabase() { + static auto generated_database = + internal::OnShutdownDelete(new EncodedDescriptorDatabase()); + return generated_database; } -static void InitGeneratedPool() { - generated_database_ = new EncodedDescriptorDatabase; - generated_pool_ = new DescriptorPool(generated_database_); - generated_pool_->InternalSetLazilyBuildDependencies(); - - internal::OnShutdown(&DeleteGeneratedPool); -} - -inline void InitGeneratedPoolOnce() { - ::google::protobuf::GoogleOnceInit(&generated_pool_init_, &InitGeneratedPool); +DescriptorPool* NewGeneratedPool() { + auto generated_pool = new DescriptorPool(GeneratedDatabase()); + generated_pool->InternalSetLazilyBuildDependencies(); + return generated_pool; } } // anonymous namespace -const DescriptorPool* DescriptorPool::generated_pool() { - InitGeneratedPoolOnce(); - return generated_pool_; +DescriptorDatabase* DescriptorPool::internal_generated_database() { + return GeneratedDatabase(); } - - DescriptorPool* DescriptorPool::internal_generated_pool() { - InitGeneratedPoolOnce(); - return generated_pool_; + static DescriptorPool* generated_pool = + internal::OnShutdownDelete(NewGeneratedPool()); + return generated_pool; } +const DescriptorPool* DescriptorPool::generated_pool() { + const DescriptorPool* pool = internal_generated_pool(); + // Ensure that descriptor.proto has been registered in the generated pool. + DescriptorProto::descriptor(); + return pool; +} + + void DescriptorPool::InternalAddGeneratedFile( const void* encoded_file_descriptor, int size) { // So, this function is called in the process of initializing the @@ -1390,8 +2018,7 @@ void DescriptorPool::InternalAddGeneratedFile( // Therefore, when we parse one, we have to be very careful to avoid using // any descriptor-based operations, since this might cause infinite recursion // or deadlock. - InitGeneratedPoolOnce(); - GOOGLE_CHECK(generated_database_->Add(encoded_file_descriptor, size)); + GOOGLE_CHECK(GeneratedDatabase()->Add(encoded_file_descriptor, size)); } @@ -1401,149 +2028,195 @@ void DescriptorPool::InternalAddGeneratedFile( // there's any good way to factor it out. Think about this some time when // there's nothing more important to do (read: never). -const FileDescriptor* DescriptorPool::FindFileByName(const string& name) const { +const FileDescriptor* DescriptorPool::FindFileByName( + ConstStringParam name) const { MutexLockMaybe lock(mutex_); - tables_->known_bad_symbols_.clear(); - tables_->known_bad_files_.clear(); + if (fallback_database_ != nullptr) { + tables_->known_bad_symbols_.clear(); + tables_->known_bad_files_.clear(); + } const FileDescriptor* result = tables_->FindFile(name); - if (result != NULL) return result; - if (underlay_ != NULL) { + if (result != nullptr) return result; + if (underlay_ != nullptr) { result = underlay_->FindFileByName(name); - if (result != NULL) return result; + if (result != nullptr) return result; } if (TryFindFileInFallbackDatabase(name)) { result = tables_->FindFile(name); - if (result != NULL) return result; + if (result != nullptr) return result; } - return NULL; + return nullptr; } const FileDescriptor* DescriptorPool::FindFileContainingSymbol( - const string& symbol_name) const { + ConstStringParam symbol_name) const { MutexLockMaybe lock(mutex_); - tables_->known_bad_symbols_.clear(); - tables_->known_bad_files_.clear(); + if (fallback_database_ != nullptr) { + tables_->known_bad_symbols_.clear(); + tables_->known_bad_files_.clear(); + } Symbol result = tables_->FindSymbol(symbol_name); if (!result.IsNull()) return result.GetFile(); - if (underlay_ != NULL) { + if (underlay_ != nullptr) { const FileDescriptor* file_result = - underlay_->FindFileContainingSymbol(symbol_name); - if (file_result != NULL) return file_result; + underlay_->FindFileContainingSymbol(symbol_name); + if (file_result != nullptr) return file_result; } if (TryFindSymbolInFallbackDatabase(symbol_name)) { result = tables_->FindSymbol(symbol_name); if (!result.IsNull()) return result.GetFile(); } - return NULL; + return nullptr; } const Descriptor* DescriptorPool::FindMessageTypeByName( - const string& name) const { - Symbol result = tables_->FindByNameHelper(this, name); - return (result.type == Symbol::MESSAGE) ? result.descriptor : NULL; + ConstStringParam name) const { + return tables_->FindByNameHelper(this, name).descriptor(); } const FieldDescriptor* DescriptorPool::FindFieldByName( - const string& name) const { - Symbol result = tables_->FindByNameHelper(this, name); - if (result.type == Symbol::FIELD && - !result.field_descriptor->is_extension()) { - return result.field_descriptor; - } else { - return NULL; + ConstStringParam name) const { + if (const FieldDescriptor* field = + tables_->FindByNameHelper(this, name).field_descriptor()) { + if (!field->is_extension()) { + return field; + } } + return nullptr; } const FieldDescriptor* DescriptorPool::FindExtensionByName( - const string& name) const { - Symbol result = tables_->FindByNameHelper(this, name); - if (result.type == Symbol::FIELD && - result.field_descriptor->is_extension()) { - return result.field_descriptor; - } else { - return NULL; + ConstStringParam name) const { + if (const FieldDescriptor* field = + tables_->FindByNameHelper(this, name).field_descriptor()) { + if (field->is_extension()) { + return field; + } } + return nullptr; } const OneofDescriptor* DescriptorPool::FindOneofByName( - const string& name) const { - Symbol result = tables_->FindByNameHelper(this, name); - return (result.type == Symbol::ONEOF) ? result.oneof_descriptor : NULL; + ConstStringParam name) const { + return tables_->FindByNameHelper(this, name).oneof_descriptor(); } const EnumDescriptor* DescriptorPool::FindEnumTypeByName( - const string& name) const { - Symbol result = tables_->FindByNameHelper(this, name); - return (result.type == Symbol::ENUM) ? result.enum_descriptor : NULL; + ConstStringParam name) const { + return tables_->FindByNameHelper(this, name).enum_descriptor(); } const EnumValueDescriptor* DescriptorPool::FindEnumValueByName( - const string& name) const { - Symbol result = tables_->FindByNameHelper(this, name); - return (result.type == Symbol::ENUM_VALUE) ? - result.enum_value_descriptor : NULL; + ConstStringParam name) const { + return tables_->FindByNameHelper(this, name).enum_value_descriptor(); } const ServiceDescriptor* DescriptorPool::FindServiceByName( - const string& name) const { - Symbol result = tables_->FindByNameHelper(this, name); - return (result.type == Symbol::SERVICE) ? result.service_descriptor : NULL; + ConstStringParam name) const { + return tables_->FindByNameHelper(this, name).service_descriptor(); } const MethodDescriptor* DescriptorPool::FindMethodByName( - const string& name) const { - Symbol result = tables_->FindByNameHelper(this, name); - return (result.type == Symbol::METHOD) ? result.method_descriptor : NULL; + ConstStringParam name) const { + return tables_->FindByNameHelper(this, name).method_descriptor(); } const FieldDescriptor* DescriptorPool::FindExtensionByNumber( const Descriptor* extendee, int number) const { + if (extendee->extension_range_count() == 0) return nullptr; // A faster path to reduce lock contention in finding extensions, assuming // most extensions will be cache hit. - if (mutex_ != NULL) { + if (mutex_ != nullptr) { ReaderMutexLock lock(mutex_); const FieldDescriptor* result = tables_->FindExtension(extendee, number); - if (result != NULL) { + if (result != nullptr) { return result; } } MutexLockMaybe lock(mutex_); - tables_->known_bad_symbols_.clear(); - tables_->known_bad_files_.clear(); + if (fallback_database_ != nullptr) { + tables_->known_bad_symbols_.clear(); + tables_->known_bad_files_.clear(); + } const FieldDescriptor* result = tables_->FindExtension(extendee, number); - if (result != NULL) { + if (result != nullptr) { return result; } - if (underlay_ != NULL) { + if (underlay_ != nullptr) { result = underlay_->FindExtensionByNumber(extendee, number); - if (result != NULL) return result; + if (result != nullptr) return result; } if (TryFindExtensionInFallbackDatabase(extendee, number)) { result = tables_->FindExtension(extendee, number); - if (result != NULL) { + if (result != nullptr) { return result; } } - return NULL; + return nullptr; +} + +const FieldDescriptor* DescriptorPool::InternalFindExtensionByNumberNoLock( + const Descriptor* extendee, int number) const { + if (extendee->extension_range_count() == 0) return nullptr; + + const FieldDescriptor* result = tables_->FindExtension(extendee, number); + if (result != nullptr) { + return result; + } + + if (underlay_ != nullptr) { + result = underlay_->InternalFindExtensionByNumberNoLock(extendee, number); + if (result != nullptr) return result; + } + + return nullptr; +} + +const FieldDescriptor* DescriptorPool::FindExtensionByPrintableName( + const Descriptor* extendee, ConstStringParam printable_name) const { + if (extendee->extension_range_count() == 0) return nullptr; + const FieldDescriptor* result = FindExtensionByName(printable_name); + if (result != nullptr && result->containing_type() == extendee) { + return result; + } + if (extendee->options().message_set_wire_format()) { + // MessageSet extensions may be identified by type name. + const Descriptor* type = FindMessageTypeByName(printable_name); + if (type != nullptr) { + // Look for a matching extension in the foreign type's scope. + const int type_extension_count = type->extension_count(); + for (int i = 0; i < type_extension_count; i++) { + const FieldDescriptor* extension = type->extension(i); + if (extension->containing_type() == extendee && + extension->type() == FieldDescriptor::TYPE_MESSAGE && + extension->is_optional() && extension->message_type() == type) { + // Found it. + return extension; + } + } + } + } + return nullptr; } void DescriptorPool::FindAllExtensions( const Descriptor* extendee, std::vector* out) const { MutexLockMaybe lock(mutex_); - tables_->known_bad_symbols_.clear(); - tables_->known_bad_files_.clear(); + if (fallback_database_ != nullptr) { + tables_->known_bad_symbols_.clear(); + tables_->known_bad_files_.clear(); + } // Initialize tables_->extensions_ from the fallback database first // (but do this only once per descriptor). - if (fallback_database_ != NULL && + if (fallback_database_ != nullptr && tables_->extensions_loaded_from_db_.count(extendee) == 0) { std::vector numbers; if (fallback_database_->FindAllExtensionNumbers(extendee->full_name(), &numbers)) { - for (int i = 0; i < numbers.size(); ++i) { - int number = numbers[i]; - if (tables_->FindExtension(extendee, number) == NULL) { + for (int number : numbers) { + if (tables_->FindExtension(extendee, number) == nullptr) { TryFindExtensionInFallbackDatabase(extendee, number); } } @@ -1552,7 +2225,7 @@ void DescriptorPool::FindAllExtensions( } tables_->FindAllExtensions(extendee, out); - if (underlay_ != NULL) { + if (underlay_ != nullptr) { underlay_->FindAllExtensions(extendee, out); } } @@ -1560,253 +2233,199 @@ void DescriptorPool::FindAllExtensions( // ------------------------------------------------------------------- -const FieldDescriptor* -Descriptor::FindFieldByNumber(int key) const { - const FieldDescriptor* result = - file()->tables_->FindFieldByNumber(this, key); - if (result == NULL || result->is_extension()) { - return NULL; +const FieldDescriptor* Descriptor::FindFieldByNumber(int key) const { + const FieldDescriptor* result = file()->tables_->FindFieldByNumber(this, key); + if (result == nullptr || result->is_extension()) { + return nullptr; } else { return result; } } -const FieldDescriptor* -Descriptor::FindFieldByLowercaseName(const string& key) const { +const FieldDescriptor* Descriptor::FindFieldByLowercaseName( + ConstStringParam key) const { const FieldDescriptor* result = - file()->tables_->FindFieldByLowercaseName(this, key); - if (result == NULL || result->is_extension()) { - return NULL; + file()->tables_->FindFieldByLowercaseName(this, key); + if (result == nullptr || result->is_extension()) { + return nullptr; } else { return result; } } -const FieldDescriptor* -Descriptor::FindFieldByCamelcaseName(const string& key) const { +const FieldDescriptor* Descriptor::FindFieldByCamelcaseName( + ConstStringParam key) const { const FieldDescriptor* result = - file()->tables_->FindFieldByCamelcaseName(this, key); - if (result == NULL || result->is_extension()) { - return NULL; + file()->tables_->FindFieldByCamelcaseName(this, key); + if (result == nullptr || result->is_extension()) { + return nullptr; } else { return result; } } -const FieldDescriptor* -Descriptor::FindFieldByName(const string& key) const { - Symbol result = - file()->tables_->FindNestedSymbolOfType(this, key, Symbol::FIELD); - if (!result.IsNull() && !result.field_descriptor->is_extension()) { - return result.field_descriptor; - } else { - return NULL; - } +const FieldDescriptor* Descriptor::FindFieldByName(ConstStringParam key) const { + const FieldDescriptor* field = + file()->tables_->FindNestedSymbol(this, key).field_descriptor(); + return field != nullptr && !field->is_extension() ? field : nullptr; } -const OneofDescriptor* -Descriptor::FindOneofByName(const string& key) const { - Symbol result = - file()->tables_->FindNestedSymbolOfType(this, key, Symbol::ONEOF); - if (!result.IsNull()) { - return result.oneof_descriptor; - } else { - return NULL; - } +const OneofDescriptor* Descriptor::FindOneofByName(ConstStringParam key) const { + return file()->tables_->FindNestedSymbol(this, key).oneof_descriptor(); } -const FieldDescriptor* -Descriptor::FindExtensionByName(const string& key) const { - Symbol result = - file()->tables_->FindNestedSymbolOfType(this, key, Symbol::FIELD); - if (!result.IsNull() && result.field_descriptor->is_extension()) { - return result.field_descriptor; - } else { - return NULL; - } +const FieldDescriptor* Descriptor::FindExtensionByName( + ConstStringParam key) const { + const FieldDescriptor* field = + file()->tables_->FindNestedSymbol(this, key).field_descriptor(); + return field != nullptr && field->is_extension() ? field : nullptr; } -const FieldDescriptor* -Descriptor::FindExtensionByLowercaseName(const string& key) const { +const FieldDescriptor* Descriptor::FindExtensionByLowercaseName( + ConstStringParam key) const { const FieldDescriptor* result = - file()->tables_->FindFieldByLowercaseName(this, key); - if (result == NULL || !result->is_extension()) { - return NULL; + file()->tables_->FindFieldByLowercaseName(this, key); + if (result == nullptr || !result->is_extension()) { + return nullptr; } else { return result; } } -const FieldDescriptor* -Descriptor::FindExtensionByCamelcaseName(const string& key) const { +const FieldDescriptor* Descriptor::FindExtensionByCamelcaseName( + ConstStringParam key) const { const FieldDescriptor* result = - file()->tables_->FindFieldByCamelcaseName(this, key); - if (result == NULL || !result->is_extension()) { - return NULL; + file()->tables_->FindFieldByCamelcaseName(this, key); + if (result == nullptr || !result->is_extension()) { + return nullptr; } else { return result; } } -const Descriptor* -Descriptor::FindNestedTypeByName(const string& key) const { - Symbol result = - file()->tables_->FindNestedSymbolOfType(this, key, Symbol::MESSAGE); - if (!result.IsNull()) { - return result.descriptor; - } else { - return NULL; - } +const Descriptor* Descriptor::FindNestedTypeByName(ConstStringParam key) const { + return file()->tables_->FindNestedSymbol(this, key).descriptor(); } -const EnumDescriptor* -Descriptor::FindEnumTypeByName(const string& key) const { - Symbol result = - file()->tables_->FindNestedSymbolOfType(this, key, Symbol::ENUM); - if (!result.IsNull()) { - return result.enum_descriptor; - } else { - return NULL; - } +const EnumDescriptor* Descriptor::FindEnumTypeByName( + ConstStringParam key) const { + return file()->tables_->FindNestedSymbol(this, key).enum_descriptor(); } -const EnumValueDescriptor* -Descriptor::FindEnumValueByName(const string& key) const { - Symbol result = - file()->tables_->FindNestedSymbolOfType(this, key, Symbol::ENUM_VALUE); - if (!result.IsNull()) { - return result.enum_value_descriptor; - } else { - return NULL; - } +const EnumValueDescriptor* Descriptor::FindEnumValueByName( + ConstStringParam key) const { + return file()->tables_->FindNestedSymbol(this, key).enum_value_descriptor(); } -const EnumValueDescriptor* -EnumDescriptor::FindValueByName(const string& key) const { - Symbol result = - file()->tables_->FindNestedSymbolOfType(this, key, Symbol::ENUM_VALUE); - if (!result.IsNull()) { - return result.enum_value_descriptor; - } else { - return NULL; - } +const FieldDescriptor* Descriptor::map_key() const { + if (!options().map_entry()) return nullptr; + GOOGLE_DCHECK_EQ(field_count(), 2); + return field(0); } -const EnumValueDescriptor* -EnumDescriptor::FindValueByNumber(int key) const { +const FieldDescriptor* Descriptor::map_value() const { + if (!options().map_entry()) return nullptr; + GOOGLE_DCHECK_EQ(field_count(), 2); + return field(1); +} + +const EnumValueDescriptor* EnumDescriptor::FindValueByName( + ConstStringParam key) const { + return file()->tables_->FindNestedSymbol(this, key).enum_value_descriptor(); +} + +const EnumValueDescriptor* EnumDescriptor::FindValueByNumber(int key) const { return file()->tables_->FindEnumValueByNumber(this, key); } -const EnumValueDescriptor* -EnumDescriptor::FindValueByNumberCreatingIfUnknown(int key) const { +const EnumValueDescriptor* EnumDescriptor::FindValueByNumberCreatingIfUnknown( + int key) const { return file()->tables_->FindEnumValueByNumberCreatingIfUnknown(this, key); } -const MethodDescriptor* -ServiceDescriptor::FindMethodByName(const string& key) const { - Symbol result = - file()->tables_->FindNestedSymbolOfType(this, key, Symbol::METHOD); - if (!result.IsNull()) { - return result.method_descriptor; - } else { - return NULL; - } +const MethodDescriptor* ServiceDescriptor::FindMethodByName( + ConstStringParam key) const { + return file()->tables_->FindNestedSymbol(this, key).method_descriptor(); } -const Descriptor* -FileDescriptor::FindMessageTypeByName(const string& key) const { - Symbol result = tables_->FindNestedSymbolOfType(this, key, Symbol::MESSAGE); - if (!result.IsNull()) { - return result.descriptor; - } else { - return NULL; - } +const Descriptor* FileDescriptor::FindMessageTypeByName( + ConstStringParam key) const { + return tables_->FindNestedSymbol(this, key).descriptor(); } -const EnumDescriptor* -FileDescriptor::FindEnumTypeByName(const string& key) const { - Symbol result = tables_->FindNestedSymbolOfType(this, key, Symbol::ENUM); - if (!result.IsNull()) { - return result.enum_descriptor; - } else { - return NULL; - } +const EnumDescriptor* FileDescriptor::FindEnumTypeByName( + ConstStringParam key) const { + return tables_->FindNestedSymbol(this, key).enum_descriptor(); } -const EnumValueDescriptor* -FileDescriptor::FindEnumValueByName(const string& key) const { - Symbol result = - tables_->FindNestedSymbolOfType(this, key, Symbol::ENUM_VALUE); - if (!result.IsNull()) { - return result.enum_value_descriptor; - } else { - return NULL; - } +const EnumValueDescriptor* FileDescriptor::FindEnumValueByName( + ConstStringParam key) const { + return tables_->FindNestedSymbol(this, key).enum_value_descriptor(); } -const ServiceDescriptor* -FileDescriptor::FindServiceByName(const string& key) const { - Symbol result = tables_->FindNestedSymbolOfType(this, key, Symbol::SERVICE); - if (!result.IsNull()) { - return result.service_descriptor; - } else { - return NULL; - } +const ServiceDescriptor* FileDescriptor::FindServiceByName( + ConstStringParam key) const { + return tables_->FindNestedSymbol(this, key).service_descriptor(); } -const FieldDescriptor* -FileDescriptor::FindExtensionByName(const string& key) const { - Symbol result = tables_->FindNestedSymbolOfType(this, key, Symbol::FIELD); - if (!result.IsNull() && result.field_descriptor->is_extension()) { - return result.field_descriptor; - } else { - return NULL; - } +const FieldDescriptor* FileDescriptor::FindExtensionByName( + ConstStringParam key) const { + const FieldDescriptor* field = + tables_->FindNestedSymbol(this, key).field_descriptor(); + return field != nullptr && field->is_extension() ? field : nullptr; } -const FieldDescriptor* -FileDescriptor::FindExtensionByLowercaseName(const string& key) const { +const FieldDescriptor* FileDescriptor::FindExtensionByLowercaseName( + ConstStringParam key) const { const FieldDescriptor* result = tables_->FindFieldByLowercaseName(this, key); - if (result == NULL || !result->is_extension()) { - return NULL; + if (result == nullptr || !result->is_extension()) { + return nullptr; } else { return result; } } -const FieldDescriptor* -FileDescriptor::FindExtensionByCamelcaseName(const string& key) const { +const FieldDescriptor* FileDescriptor::FindExtensionByCamelcaseName( + ConstStringParam key) const { const FieldDescriptor* result = tables_->FindFieldByCamelcaseName(this, key); - if (result == NULL || !result->is_extension()) { - return NULL; + if (result == nullptr || !result->is_extension()) { + return nullptr; } else { return result; } } +void Descriptor::ExtensionRange::CopyTo( + DescriptorProto_ExtensionRange* proto) const { + proto->set_start(this->start); + proto->set_end(this->end); + if (options_ != &ExtensionRangeOptions::default_instance()) { + *proto->mutable_options() = *options_; + } +} + const Descriptor::ExtensionRange* Descriptor::FindExtensionRangeContainingNumber(int number) const { // Linear search should be fine because we don't expect a message to have // more than a couple extension ranges. for (int i = 0; i < extension_range_count(); i++) { if (number >= extension_range(i)->start && - number < extension_range(i)->end) { + number < extension_range(i)->end) { return extension_range(i); } } - return NULL; + return nullptr; } -const Descriptor::ReservedRange* -Descriptor::FindReservedRangeContainingNumber(int number) const { +const Descriptor::ReservedRange* Descriptor::FindReservedRangeContainingNumber( + int number) const { // TODO(chrisn): Consider a non-linear search. for (int i = 0; i < reserved_range_count(); i++) { - if (number >= reserved_range(i)->start && - number < reserved_range(i)->end) { + if (number >= reserved_range(i)->start && number < reserved_range(i)->end) { return reserved_range(i); } } - return NULL; + return nullptr; } const EnumDescriptor::ReservedRange* @@ -1818,82 +2437,86 @@ EnumDescriptor::FindReservedRangeContainingNumber(int number) const { return reserved_range(i); } } - return NULL; + return nullptr; } // ------------------------------------------------------------------- -bool DescriptorPool::TryFindFileInFallbackDatabase(const string& name) const { - if (fallback_database_ == NULL) return false; +bool DescriptorPool::TryFindFileInFallbackDatabase( + StringPiece name) const { + if (fallback_database_ == nullptr) return false; - if (tables_->known_bad_files_.count(name) > 0) return false; + auto name_string = std::string(name); + if (tables_->known_bad_files_.count(name_string) > 0) return false; FileDescriptorProto file_proto; - if (!fallback_database_->FindFileByName(name, &file_proto) || - BuildFileFromDatabase(file_proto) == NULL) { - tables_->known_bad_files_.insert(name); + if (!fallback_database_->FindFileByName(name_string, &file_proto) || + BuildFileFromDatabase(file_proto) == nullptr) { + tables_->known_bad_files_.insert(std::move(name_string)); return false; } return true; } -bool DescriptorPool::IsSubSymbolOfBuiltType(const string& name) const { - string prefix = name; +bool DescriptorPool::IsSubSymbolOfBuiltType(StringPiece name) const { + auto prefix = std::string(name); for (;;) { - string::size_type dot_pos = prefix.find_last_of('.'); - if (dot_pos == string::npos) { + std::string::size_type dot_pos = prefix.find_last_of('.'); + if (dot_pos == std::string::npos) { break; } prefix = prefix.substr(0, dot_pos); Symbol symbol = tables_->FindSymbol(prefix); // If the symbol type is anything other than PACKAGE, then its complete // definition is already known. - if (!symbol.IsNull() && symbol.type != Symbol::PACKAGE) { + if (!symbol.IsNull() && symbol.type() != Symbol::PACKAGE) { return true; } } - if (underlay_ != NULL) { + if (underlay_ != nullptr) { // Check to see if any prefix of this symbol exists in the underlay. return underlay_->IsSubSymbolOfBuiltType(name); } return false; } -bool DescriptorPool::TryFindSymbolInFallbackDatabase(const string& name) const { - if (fallback_database_ == NULL) return false; +bool DescriptorPool::TryFindSymbolInFallbackDatabase( + StringPiece name) const { + if (fallback_database_ == nullptr) return false; - if (tables_->known_bad_symbols_.count(name) > 0) return false; + auto name_string = std::string(name); + if (tables_->known_bad_symbols_.count(name_string) > 0) return false; FileDescriptorProto file_proto; - if (// We skip looking in the fallback database if the name is a sub-symbol - // of any descriptor that already exists in the descriptor pool (except - // for package descriptors). This is valid because all symbols except - // for packages are defined in a single file, so if the symbol exists - // then we should already have its definition. - // - // The other reason to do this is to support "overriding" type - // definitions by merging two databases that define the same type. (Yes, - // people do this.) The main difficulty with making this work is that - // FindFileContainingSymbol() is allowed to return both false positives - // (e.g., SimpleDescriptorDatabase, UpgradedDescriptorDatabase) and false - // negatives (e.g. ProtoFileParser, SourceTreeDescriptorDatabase). - // When two such databases are merged, looking up a non-existent - // sub-symbol of a type that already exists in the descriptor pool can - // result in an attempt to load multiple definitions of the same type. - // The check below avoids this. + if ( // We skip looking in the fallback database if the name is a sub-symbol + // of any descriptor that already exists in the descriptor pool (except + // for package descriptors). This is valid because all symbols except + // for packages are defined in a single file, so if the symbol exists + // then we should already have its definition. + // + // The other reason to do this is to support "overriding" type + // definitions by merging two databases that define the same type. (Yes, + // people do this.) The main difficulty with making this work is that + // FindFileContainingSymbol() is allowed to return both false positives + // (e.g., SimpleDescriptorDatabase, UpgradedDescriptorDatabase) and + // false negatives (e.g. ProtoFileParser, SourceTreeDescriptorDatabase). + // When two such databases are merged, looking up a non-existent + // sub-symbol of a type that already exists in the descriptor pool can + // result in an attempt to load multiple definitions of the same type. + // The check below avoids this. IsSubSymbolOfBuiltType(name) // Look up file containing this symbol in fallback database. - || !fallback_database_->FindFileContainingSymbol(name, &file_proto) + || !fallback_database_->FindFileContainingSymbol(name_string, &file_proto) // Check if we've already built this file. If so, it apparently doesn't // contain the symbol we're looking for. Some DescriptorDatabases // return false positives. - || tables_->FindFile(file_proto.name()) != NULL + || tables_->FindFile(file_proto.name()) != nullptr // Build the file. - || BuildFileFromDatabase(file_proto) == NULL) { - tables_->known_bad_symbols_.insert(name); + || BuildFileFromDatabase(file_proto) == nullptr) { + tables_->known_bad_symbols_.insert(std::move(name_string)); return false; } @@ -1902,22 +2525,22 @@ bool DescriptorPool::TryFindSymbolInFallbackDatabase(const string& name) const { bool DescriptorPool::TryFindExtensionInFallbackDatabase( const Descriptor* containing_type, int field_number) const { - if (fallback_database_ == NULL) return false; + if (fallback_database_ == nullptr) return false; FileDescriptorProto file_proto; if (!fallback_database_->FindFileContainingExtension( - containing_type->full_name(), field_number, &file_proto)) { + containing_type->full_name(), field_number, &file_proto)) { return false; } - if (tables_->FindFile(file_proto.name()) != NULL) { + if (tables_->FindFile(file_proto.name()) != nullptr) { // We've already loaded this file, and it apparently doesn't contain the // extension we're looking for. Some DescriptorDatabases return false // positives. return false; } - if (BuildFileFromDatabase(file_proto) == NULL) { + if (BuildFileFromDatabase(file_proto) == nullptr) { return false; } @@ -1927,33 +2550,27 @@ bool DescriptorPool::TryFindExtensionInFallbackDatabase( // =================================================================== bool FieldDescriptor::is_map_message_type() const { - return message_type_->options().map_entry(); + return type_descriptor_.message_type->options().map_entry(); } -string FieldDescriptor::DefaultValueAsString(bool quote_string_type) const { +std::string FieldDescriptor::DefaultValueAsString( + bool quote_string_type) const { GOOGLE_CHECK(has_default_value()) << "No default value"; switch (cpp_type()) { case CPPTYPE_INT32: - return SimpleItoa(default_value_int32()); - break; + return StrCat(default_value_int32_t()); case CPPTYPE_INT64: - return SimpleItoa(default_value_int64()); - break; + return StrCat(default_value_int64_t()); case CPPTYPE_UINT32: - return SimpleItoa(default_value_uint32()); - break; + return StrCat(default_value_uint32_t()); case CPPTYPE_UINT64: - return SimpleItoa(default_value_uint64()); - break; + return StrCat(default_value_uint64_t()); case CPPTYPE_FLOAT: return SimpleFtoa(default_value_float()); - break; case CPPTYPE_DOUBLE: return SimpleDtoa(default_value_double()); - break; case CPPTYPE_BOOL: return default_value_bool() ? "true" : "false"; - break; case CPPTYPE_STRING: if (quote_string_type) { return "\"" + CEscape(default_value_string()) + "\""; @@ -1964,10 +2581,8 @@ string FieldDescriptor::DefaultValueAsString(bool quote_string_type) const { return default_value_string(); } } - break; case CPPTYPE_ENUM: return default_value_enum()->name(); - break; case CPPTYPE_MESSAGE: GOOGLE_LOG(DFATAL) << "Messages can't have default values!"; break; @@ -2051,13 +2666,7 @@ void Descriptor::CopyTo(DescriptorProto* proto) const { enum_type(i)->CopyTo(proto->add_enum_type()); } for (int i = 0; i < extension_range_count(); i++) { - DescriptorProto::ExtensionRange* range = proto->add_extension_range(); - range->set_start(extension_range(i)->start); - range->set_end(extension_range(i)->end); - const ExtensionRangeOptions* options = extension_range(i)->options_; - if (options != &ExtensionRangeOptions::default_instance()) { - range->mutable_options()->CopyFrom(*options); - } + extension_range(i)->CopyTo(proto->add_extension_range()); } for (int i = 0; i < extension_count(); i++) { extension(i)->CopyTo(proto->add_extension()); @@ -2100,13 +2709,15 @@ void FieldDescriptor::CopyTo(FieldDescriptorProto* proto) const { if (has_json_name_) { proto->set_json_name(json_name()); } - + if (proto3_optional_) { + proto->set_proto3_optional(true); + } // Some compilers do not allow static_cast directly between two enum types, // so we must cast to int first. proto->set_label(static_cast( - implicit_cast(label()))); + implicit_cast(label()))); proto->set_type(static_cast( - implicit_cast(type()))); + implicit_cast(type()))); if (is_extension()) { if (!containing_type()->is_unqualified_placeholder_) { @@ -2137,7 +2748,7 @@ void FieldDescriptor::CopyTo(FieldDescriptorProto* proto) const { proto->set_default_value(DefaultValueAsString(false)); } - if (containing_oneof() != NULL && !is_extension()) { + if (containing_oneof() != nullptr && !is_extension()) { proto->set_oneof_index(containing_oneof()->index()); } @@ -2227,40 +2838,42 @@ void MethodDescriptor::CopyTo(MethodDescriptorProto* proto) const { namespace { -bool RetrieveOptionsAssumingRightPool(int depth, const Message& options, - std::vector* option_entries) { +bool RetrieveOptionsAssumingRightPool( + int depth, const Message& options, + std::vector* option_entries) { option_entries->clear(); const Reflection* reflection = options.GetReflection(); std::vector fields; reflection->ListFields(options, &fields); - for (int i = 0; i < fields.size(); i++) { + for (const FieldDescriptor* field : fields) { int count = 1; bool repeated = false; - if (fields[i]->is_repeated()) { - count = reflection->FieldSize(options, fields[i]); + if (field->is_repeated()) { + count = reflection->FieldSize(options, field); repeated = true; } for (int j = 0; j < count; j++) { - string fieldval; - if (fields[i]->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { - string tmp; + std::string fieldval; + if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + std::string tmp; TextFormat::Printer printer; + printer.SetExpandAny(true); printer.SetInitialIndentLevel(depth + 1); - printer.PrintFieldValueToString(options, fields[i], - repeated ? j : -1, &tmp); + printer.PrintFieldValueToString(options, field, repeated ? j : -1, + &tmp); fieldval.append("{\n"); fieldval.append(tmp); fieldval.append(depth * 2, ' '); fieldval.append("}"); } else { - TextFormat::PrintFieldValueToString(options, fields[i], - repeated ? j : -1, &fieldval); + TextFormat::PrintFieldValueToString(options, field, repeated ? j : -1, + &fieldval); } - string name; - if (fields[i]->is_extension()) { - name = "(." + fields[i]->full_name() + ")"; + std::string name; + if (field->is_extension()) { + name = "(." + field->full_name() + ")"; } else { - name = fields[i]->name(); + name = field->name(); } option_entries->push_back(name + " = " + fieldval); } @@ -2271,7 +2884,7 @@ bool RetrieveOptionsAssumingRightPool(int depth, const Message& options, // Used by each of the option formatters. bool RetrieveOptions(int depth, const Message& options, const DescriptorPool* pool, - std::vector* option_entries) { + std::vector* option_entries) { // When printing custom options for a descriptor, we must use an options // message built on top of the same DescriptorPool where the descriptor // is coming from. This is to ensure we are interpreting custom options @@ -2281,16 +2894,20 @@ bool RetrieveOptions(int depth, const Message& options, } else { const Descriptor* option_descriptor = pool->FindMessageTypeByName(options.GetDescriptor()->full_name()); - if (option_descriptor == NULL) { - // google/protobuf/descriptor.proto is not in the pool. This means no - // custom options are used so we are safe to proceed with the compiled - // options message type. + if (option_descriptor == nullptr) { + // descriptor.proto is not in the pool. This means no custom options are + // used so we are safe to proceed with the compiled options message type. return RetrieveOptionsAssumingRightPool(depth, options, option_entries); } DynamicMessageFactory factory; - google::protobuf::scoped_ptr dynamic_options( + std::unique_ptr dynamic_options( factory.GetPrototype(option_descriptor)->New()); - if (dynamic_options->ParseFromString(options.SerializeAsString())) { + std::string serialized = options.SerializeAsString(); + io::CodedInputStream input( + reinterpret_cast(serialized.c_str()), + serialized.size()); + input.SetExtensionRegistry(pool, &factory); + if (dynamic_options->ParseFromCodedStream(&input)) { return RetrieveOptionsAssumingRightPool(depth, *dynamic_options, option_entries); } else { @@ -2304,8 +2921,8 @@ bool RetrieveOptions(int depth, const Message& options, // Formats options that all appear together in brackets. Does not include // brackets. bool FormatBracketedOptions(int depth, const Message& options, - const DescriptorPool* pool, string* output) { - std::vector all_options; + const DescriptorPool* pool, std::string* output) { + std::vector all_options; if (RetrieveOptions(depth, options, pool, &all_options)) { output->append(Join(all_options, ", ")); } @@ -2314,13 +2931,12 @@ bool FormatBracketedOptions(int depth, const Message& options, // Formats options one per line bool FormatLineOptions(int depth, const Message& options, - const DescriptorPool* pool, string* output) { - string prefix(depth * 2, ' '); - std::vector all_options; + const DescriptorPool* pool, std::string* output) { + std::string prefix(depth * 2, ' '); + std::vector all_options; if (RetrieveOptions(depth, options, pool, &all_options)) { - for (int i = 0; i < all_options.size(); i++) { - strings::SubstituteAndAppend(output, "$0option $1;\n", - prefix, all_options[i]); + for (const std::string& option : all_options) { + strings::SubstituteAndAppend(output, "$0option $1;\n", prefix, option); } } return !all_options.empty(); @@ -2328,31 +2944,31 @@ bool FormatLineOptions(int depth, const Message& options, class SourceLocationCommentPrinter { public: - template - SourceLocationCommentPrinter(const DescType* desc, - const string& prefix, + template + SourceLocationCommentPrinter(const DescType* desc, const std::string& prefix, const DebugStringOptions& options) : options_(options), prefix_(prefix) { // Perform the SourceLocation lookup only if we're including user comments, // because the lookup is fairly expensive. - have_source_loc_ = options.include_comments && - desc->GetSourceLocation(&source_loc_); + have_source_loc_ = + options.include_comments && desc->GetSourceLocation(&source_loc_); } SourceLocationCommentPrinter(const FileDescriptor* file, const std::vector& path, - const string& prefix, + const std::string& prefix, const DebugStringOptions& options) : options_(options), prefix_(prefix) { // Perform the SourceLocation lookup only if we're including user comments, // because the lookup is fairly expensive. - have_source_loc_ = options.include_comments && - file->GetSourceLocation(path, &source_loc_); + have_source_loc_ = + options.include_comments && file->GetSourceLocation(path, &source_loc_); } - void AddPreComment(string* output) { + void AddPreComment(std::string* output) { if (have_source_loc_) { // Detached leading comments. - for (int i = 0 ; i < source_loc_.leading_detached_comments.size(); ++i) { - *output += FormatComment(source_loc_.leading_detached_comments[i]); + for (const std::string& leading_detached_comment : + source_loc_.leading_detached_comments) { + *output += FormatComment(leading_detached_comment); *output += "\n"; } // Attached leading comments. @@ -2361,7 +2977,7 @@ class SourceLocationCommentPrinter { } } } - void AddPostComment(string* output) { + void AddPostComment(std::string* output) { if (have_source_loc_ && source_loc_.trailing_comments.size() > 0) { *output += FormatComment(source_loc_.trailing_comments); } @@ -2369,13 +2985,12 @@ class SourceLocationCommentPrinter { // Format comment such that each line becomes a full-line C++-style comment in // the DebugString() output. - string FormatComment(const string& comment_text) { - string stripped_comment = comment_text; + std::string FormatComment(const std::string& comment_text) { + std::string stripped_comment = comment_text; StripWhitespace(&stripped_comment); - std::vector lines = Split(stripped_comment, "\n"); - string output; - for (int i = 0; i < lines.size(); ++i) { - const string& line = lines[i]; + std::vector lines = Split(stripped_comment, "\n"); + std::string output; + for (const std::string& line : lines) { strings::SubstituteAndAppend(&output, "$0// $1\n", prefix_, line); } return output; @@ -2386,32 +3001,31 @@ class SourceLocationCommentPrinter { bool have_source_loc_; SourceLocation source_loc_; DebugStringOptions options_; - string prefix_; + std::string prefix_; }; } // anonymous namespace -string FileDescriptor::DebugString() const { +std::string FileDescriptor::DebugString() const { DebugStringOptions options; // default options return DebugStringWithOptions(options); } -string FileDescriptor::DebugStringWithOptions( +std::string FileDescriptor::DebugStringWithOptions( const DebugStringOptions& debug_string_options) const { - string contents; + std::string contents; { std::vector path; path.push_back(FileDescriptorProto::kSyntaxFieldNumber); - SourceLocationCommentPrinter syntax_comment( - this, path, "", debug_string_options); + SourceLocationCommentPrinter syntax_comment(this, path, "", + debug_string_options); syntax_comment.AddPreComment(&contents); strings::SubstituteAndAppend(&contents, "syntax = \"$0\";\n\n", - SyntaxName(syntax())); + SyntaxName(syntax())); syntax_comment.AddPostComment(&contents); } - SourceLocationCommentPrinter - comment_printer(this, "", debug_string_options); + SourceLocationCommentPrinter comment_printer(this, "", debug_string_options); comment_printer.AddPreComment(&contents); std::set public_dependencies; @@ -2424,21 +3038,21 @@ string FileDescriptor::DebugStringWithOptions( for (int i = 0; i < dependency_count(); i++) { if (public_dependencies.count(i) > 0) { strings::SubstituteAndAppend(&contents, "import public \"$0\";\n", - dependency(i)->name()); + dependency(i)->name()); } else if (weak_dependencies.count(i) > 0) { strings::SubstituteAndAppend(&contents, "import weak \"$0\";\n", - dependency(i)->name()); + dependency(i)->name()); } else { strings::SubstituteAndAppend(&contents, "import \"$0\";\n", - dependency(i)->name()); + dependency(i)->name()); } } if (!package().empty()) { std::vector path; path.push_back(FileDescriptorProto::kPackageFieldNumber); - SourceLocationCommentPrinter package_comment( - this, path, "", debug_string_options); + SourceLocationCommentPrinter package_comment(this, path, "", + debug_string_options); package_comment.AddPreComment(&contents); strings::SubstituteAndAppend(&contents, "package $0;\n\n", package()); package_comment.AddPostComment(&contents); @@ -2475,16 +3089,15 @@ string FileDescriptor::DebugStringWithOptions( contents.append("\n"); } - const Descriptor* containing_type = NULL; + const Descriptor* containing_type = nullptr; for (int i = 0; i < extension_count(); i++) { if (extension(i)->containing_type() != containing_type) { if (i > 0) contents.append("}\n\n"); containing_type = extension(i)->containing_type(); strings::SubstituteAndAppend(&contents, "extend .$0 {\n", - containing_type->full_name()); + containing_type->full_name()); } - extension(i)->DebugString(1, FieldDescriptor::PRINT_LABEL, &contents, - debug_string_options); + extension(i)->DebugString(1, &contents, debug_string_options); } if (extension_count() > 0) contents.append("}\n\n"); @@ -2493,31 +3106,30 @@ string FileDescriptor::DebugStringWithOptions( return contents; } -string Descriptor::DebugString() const { +std::string Descriptor::DebugString() const { DebugStringOptions options; // default options return DebugStringWithOptions(options); } -string Descriptor::DebugStringWithOptions( +std::string Descriptor::DebugStringWithOptions( const DebugStringOptions& options) const { - string contents; + std::string contents; DebugString(0, &contents, options, /* include_opening_clause */ true); return contents; } -void Descriptor::DebugString(int depth, string *contents, - const DebugStringOptions& - debug_string_options, +void Descriptor::DebugString(int depth, std::string* contents, + const DebugStringOptions& debug_string_options, bool include_opening_clause) const { if (options().map_entry()) { // Do not generate debug string for auto-generated map-entry type. return; } - string prefix(depth * 2, ' '); + std::string prefix(depth * 2, ' '); ++depth; - SourceLocationCommentPrinter - comment_printer(this, prefix, debug_string_options); + SourceLocationCommentPrinter comment_printer(this, prefix, + debug_string_options); comment_printer.AddPreComment(contents); if (include_opening_clause) { @@ -2552,9 +3164,8 @@ void Descriptor::DebugString(int depth, string *contents, enum_type(i)->DebugString(depth, contents, debug_string_options); } for (int i = 0; i < field_count(); i++) { - if (field(i)->containing_oneof() == NULL) { - field(i)->DebugString(depth, FieldDescriptor::PRINT_LABEL, contents, - debug_string_options); + if (field(i)->real_containing_oneof() == nullptr) { + field(i)->DebugString(depth, contents, debug_string_options); } else if (field(i)->containing_oneof()->field(0) == field(i)) { // This is the first field in this oneof, so print the whole oneof. field(i)->containing_oneof()->DebugString(depth, contents, @@ -2563,24 +3174,21 @@ void Descriptor::DebugString(int depth, string *contents, } for (int i = 0; i < extension_range_count(); i++) { - strings::SubstituteAndAppend(contents, "$0 extensions $1 to $2;\n", - prefix, - extension_range(i)->start, - extension_range(i)->end - 1); + strings::SubstituteAndAppend(contents, "$0 extensions $1 to $2;\n", prefix, + extension_range(i)->start, + extension_range(i)->end - 1); } // Group extensions by what they extend, so they can be printed out together. - const Descriptor* containing_type = NULL; + const Descriptor* containing_type = nullptr; for (int i = 0; i < extension_count(); i++) { if (extension(i)->containing_type() != containing_type) { if (i > 0) strings::SubstituteAndAppend(contents, "$0 }\n", prefix); containing_type = extension(i)->containing_type(); - strings::SubstituteAndAppend(contents, "$0 extend .$1 {\n", - prefix, containing_type->full_name()); + strings::SubstituteAndAppend(contents, "$0 extend .$1 {\n", prefix, + containing_type->full_name()); } - extension(i)->DebugString( - depth + 1, FieldDescriptor::PRINT_LABEL, contents, - debug_string_options); + extension(i)->DebugString(depth + 1, contents, debug_string_options); } if (extension_count() > 0) strings::SubstituteAndAppend(contents, "$0 }\n", prefix); @@ -2591,9 +3199,11 @@ void Descriptor::DebugString(int depth, string *contents, const Descriptor::ReservedRange* range = reserved_range(i); if (range->end == range->start + 1) { strings::SubstituteAndAppend(contents, "$0, ", range->start); + } else if (range->end > FieldDescriptor::kMaxNumber) { + strings::SubstituteAndAppend(contents, "$0 to max, ", range->start); } else { - strings::SubstituteAndAppend(contents, "$0 to $1, ", - range->start, range->end - 1); + strings::SubstituteAndAppend(contents, "$0 to $1, ", range->start, + range->end - 1); } } contents->replace(contents->size() - 2, 2, ";\n"); @@ -2603,7 +3213,7 @@ void Descriptor::DebugString(int depth, string *contents, strings::SubstituteAndAppend(contents, "$0 reserved ", prefix); for (int i = 0; i < reserved_name_count(); i++) { strings::SubstituteAndAppend(contents, "\"$0\", ", - CEscape(reserved_name(i))); + CEscape(reserved_name(i))); } contents->replace(contents->size() - 2, 2, ";\n"); } @@ -2612,21 +3222,21 @@ void Descriptor::DebugString(int depth, string *contents, comment_printer.AddPostComment(contents); } -string FieldDescriptor::DebugString() const { +std::string FieldDescriptor::DebugString() const { DebugStringOptions options; // default options return DebugStringWithOptions(options); } -string FieldDescriptor::DebugStringWithOptions( +std::string FieldDescriptor::DebugStringWithOptions( const DebugStringOptions& debug_string_options) const { - string contents; + std::string contents; int depth = 0; if (is_extension()) { strings::SubstituteAndAppend(&contents, "extend .$0 {\n", - containing_type()->full_name()); + containing_type()->full_name()); depth = 1; } - DebugString(depth, PRINT_LABEL, &contents, debug_string_options); + DebugString(depth, &contents, debug_string_options); if (is_extension()) { contents.append("}\n"); } @@ -2634,8 +3244,8 @@ string FieldDescriptor::DebugStringWithOptions( } // The field type string used in FieldDescriptor::DebugString() -string FieldDescriptor::FieldTypeNameDebugString() const { - switch(type()) { +std::string FieldDescriptor::FieldTypeNameDebugString() const { + switch (type()) { case TYPE_MESSAGE: return "." + message_type()->full_name(); case TYPE_ENUM: @@ -2645,13 +3255,11 @@ string FieldDescriptor::FieldTypeNameDebugString() const { } } -void FieldDescriptor::DebugString(int depth, - PrintLabelFlag print_label_flag, - string *contents, - const DebugStringOptions& - debug_string_options) const { - string prefix(depth * 2, ' '); - string field_type; +void FieldDescriptor::DebugString( + int depth, std::string* contents, + const DebugStringOptions& debug_string_options) const { + std::string prefix(depth * 2, ' '); + std::string field_type; // Special case map fields. if (is_map()) { @@ -2663,44 +3271,32 @@ void FieldDescriptor::DebugString(int depth, field_type = FieldTypeNameDebugString(); } - bool print_label = true; - // Determine whether to omit label: - // 1. For an optional field, omit label if it's in oneof or in proto3. - // 2. For a repeated field, omit label if it's a map. - if (is_optional() && (print_label_flag == OMIT_LABEL || - file()->syntax() == FileDescriptor::SYNTAX_PROTO3)) { - print_label = false; - } else if (is_map()) { - print_label = false; - } - string label; - if (print_label) { - label = kLabelToName[this->label()]; - label.push_back(' '); + std::string label = StrCat(kLabelToName[this->label()], " "); + + // Label is omitted for maps, oneof, and plain proto3 fields. + if (is_map() || real_containing_oneof() || + (is_optional() && !has_optional_keyword())) { + label.clear(); } - SourceLocationCommentPrinter - comment_printer(this, prefix, debug_string_options); + SourceLocationCommentPrinter comment_printer(this, prefix, + debug_string_options); comment_printer.AddPreComment(contents); - strings::SubstituteAndAppend(contents, "$0$1$2 $3 = $4", - prefix, - label, - field_type, - type() == TYPE_GROUP ? message_type()->name() : - name(), - number()); + strings::SubstituteAndAppend( + contents, "$0$1$2 $3 = $4", prefix, label, field_type, + type() == TYPE_GROUP ? message_type()->name() : name(), number()); bool bracketed = false; if (has_default_value()) { bracketed = true; strings::SubstituteAndAppend(contents, " [default = $0", - DefaultValueAsString(true)); + DefaultValueAsString(true)); } if (has_json_name_) { if (!bracketed) { bracketed = true; - contents->append("["); + contents->append(" ["); } else { contents->append(", "); } @@ -2709,7 +3305,7 @@ void FieldDescriptor::DebugString(int depth, contents->append("\""); } - string formatted_options; + std::string formatted_options; if (FormatBracketedOptions(depth, options(), file()->pool(), &formatted_options)) { contents->append(bracketed ? ", " : " ["); @@ -2735,25 +3331,25 @@ void FieldDescriptor::DebugString(int depth, comment_printer.AddPostComment(contents); } -string OneofDescriptor::DebugString() const { +std::string OneofDescriptor::DebugString() const { DebugStringOptions options; // default values return DebugStringWithOptions(options); } -string OneofDescriptor::DebugStringWithOptions( +std::string OneofDescriptor::DebugStringWithOptions( const DebugStringOptions& options) const { - string contents; + std::string contents; DebugString(0, &contents, options); return contents; } -void OneofDescriptor::DebugString(int depth, string* contents, - const DebugStringOptions& - debug_string_options) const { - string prefix(depth * 2, ' '); +void OneofDescriptor::DebugString( + int depth, std::string* contents, + const DebugStringOptions& debug_string_options) const { + std::string prefix(depth * 2, ' '); ++depth; - SourceLocationCommentPrinter - comment_printer(this, prefix, debug_string_options); + SourceLocationCommentPrinter comment_printer(this, prefix, + debug_string_options); comment_printer.AddPreComment(contents); strings::SubstituteAndAppend(contents, "$0oneof $1 {", prefix, name()); @@ -2765,38 +3361,36 @@ void OneofDescriptor::DebugString(int depth, string* contents, } else { contents->append("\n"); for (int i = 0; i < field_count(); i++) { - field(i)->DebugString(depth, FieldDescriptor::OMIT_LABEL, contents, - debug_string_options); + field(i)->DebugString(depth, contents, debug_string_options); } strings::SubstituteAndAppend(contents, "$0}\n", prefix); } comment_printer.AddPostComment(contents); } -string EnumDescriptor::DebugString() const { +std::string EnumDescriptor::DebugString() const { DebugStringOptions options; // default values return DebugStringWithOptions(options); } -string EnumDescriptor::DebugStringWithOptions( +std::string EnumDescriptor::DebugStringWithOptions( const DebugStringOptions& options) const { - string contents; + std::string contents; DebugString(0, &contents, options); return contents; } -void EnumDescriptor::DebugString(int depth, string *contents, - const DebugStringOptions& - debug_string_options) const { - string prefix(depth * 2, ' '); +void EnumDescriptor::DebugString( + int depth, std::string* contents, + const DebugStringOptions& debug_string_options) const { + std::string prefix(depth * 2, ' '); ++depth; - SourceLocationCommentPrinter - comment_printer(this, prefix, debug_string_options); + SourceLocationCommentPrinter comment_printer(this, prefix, + debug_string_options); comment_printer.AddPreComment(contents); - strings::SubstituteAndAppend(contents, "$0enum $1 {\n", - prefix, name()); + strings::SubstituteAndAppend(contents, "$0enum $1 {\n", prefix, name()); FormatLineOptions(depth, options(), file()->pool(), contents); @@ -2810,9 +3404,11 @@ void EnumDescriptor::DebugString(int depth, string *contents, const EnumDescriptor::ReservedRange* range = reserved_range(i); if (range->end == range->start) { strings::SubstituteAndAppend(contents, "$0, ", range->start); + } else if (range->end == INT_MAX) { + strings::SubstituteAndAppend(contents, "$0 to max, ", range->start); } else { - strings::SubstituteAndAppend(contents, "$0 to $1, ", - range->start, range->end); + strings::SubstituteAndAppend(contents, "$0 to $1, ", range->start, + range->end); } } contents->replace(contents->size() - 2, 2, ";\n"); @@ -2822,7 +3418,7 @@ void EnumDescriptor::DebugString(int depth, string *contents, strings::SubstituteAndAppend(contents, "$0 reserved ", prefix); for (int i = 0; i < reserved_name_count(); i++) { strings::SubstituteAndAppend(contents, "\"$0\", ", - CEscape(reserved_name(i))); + CEscape(reserved_name(i))); } contents->replace(contents->size() - 2, 2, ";\n"); } @@ -2832,31 +3428,30 @@ void EnumDescriptor::DebugString(int depth, string *contents, comment_printer.AddPostComment(contents); } -string EnumValueDescriptor::DebugString() const { +std::string EnumValueDescriptor::DebugString() const { DebugStringOptions options; // default values return DebugStringWithOptions(options); } -string EnumValueDescriptor::DebugStringWithOptions( +std::string EnumValueDescriptor::DebugStringWithOptions( const DebugStringOptions& options) const { - string contents; + std::string contents; DebugString(0, &contents, options); return contents; } -void EnumValueDescriptor::DebugString(int depth, string *contents, - const DebugStringOptions& - debug_string_options) const { - string prefix(depth * 2, ' '); +void EnumValueDescriptor::DebugString( + int depth, std::string* contents, + const DebugStringOptions& debug_string_options) const { + std::string prefix(depth * 2, ' '); - SourceLocationCommentPrinter - comment_printer(this, prefix, debug_string_options); + SourceLocationCommentPrinter comment_printer(this, prefix, + debug_string_options); comment_printer.AddPreComment(contents); - strings::SubstituteAndAppend(contents, "$0$1 = $2", - prefix, name(), number()); + strings::SubstituteAndAppend(contents, "$0$1 = $2", prefix, name(), number()); - string formatted_options; + std::string formatted_options; if (FormatBracketedOptions(depth, options(), type()->file()->pool(), &formatted_options)) { strings::SubstituteAndAppend(contents, " [$0]", formatted_options); @@ -2866,23 +3461,23 @@ void EnumValueDescriptor::DebugString(int depth, string *contents, comment_printer.AddPostComment(contents); } -string ServiceDescriptor::DebugString() const { +std::string ServiceDescriptor::DebugString() const { DebugStringOptions options; // default values return DebugStringWithOptions(options); } -string ServiceDescriptor::DebugStringWithOptions( +std::string ServiceDescriptor::DebugStringWithOptions( const DebugStringOptions& options) const { - string contents; + std::string contents; DebugString(&contents, options); return contents; } -void ServiceDescriptor::DebugString(string *contents, - const DebugStringOptions& - debug_string_options) const { - SourceLocationCommentPrinter - comment_printer(this, /* prefix */ "", debug_string_options); +void ServiceDescriptor::DebugString( + std::string* contents, + const DebugStringOptions& debug_string_options) const { + SourceLocationCommentPrinter comment_printer(this, /* prefix */ "", + debug_string_options); comment_printer.AddPreComment(contents); strings::SubstituteAndAppend(contents, "service $0 {\n", name()); @@ -2898,40 +3493,38 @@ void ServiceDescriptor::DebugString(string *contents, comment_printer.AddPostComment(contents); } -string MethodDescriptor::DebugString() const { +std::string MethodDescriptor::DebugString() const { DebugStringOptions options; // default values return DebugStringWithOptions(options); } -string MethodDescriptor::DebugStringWithOptions( +std::string MethodDescriptor::DebugStringWithOptions( const DebugStringOptions& options) const { - string contents; + std::string contents; DebugString(0, &contents, options); return contents; } -void MethodDescriptor::DebugString(int depth, string *contents, - const DebugStringOptions& - debug_string_options) const { - string prefix(depth * 2, ' '); +void MethodDescriptor::DebugString( + int depth, std::string* contents, + const DebugStringOptions& debug_string_options) const { + std::string prefix(depth * 2, ' '); ++depth; - SourceLocationCommentPrinter - comment_printer(this, prefix, debug_string_options); + SourceLocationCommentPrinter comment_printer(this, prefix, + debug_string_options); comment_printer.AddPreComment(contents); - strings::SubstituteAndAppend(contents, "$0rpc $1($4.$2) returns ($5.$3)", - prefix, name(), - input_type()->full_name(), - output_type()->full_name(), - client_streaming() ? "stream " : "", - server_streaming() ? "stream " : ""); + strings::SubstituteAndAppend( + contents, "$0rpc $1($4.$2) returns ($5.$3)", prefix, name(), + input_type()->full_name(), output_type()->full_name(), + client_streaming() ? "stream " : "", server_streaming() ? "stream " : ""); - string formatted_options; + std::string formatted_options; if (FormatLineOptions(depth, options(), service()->file()->pool(), &formatted_options)) { - strings::SubstituteAndAppend(contents, " {\n$0$1}\n", - formatted_options, prefix); + strings::SubstituteAndAppend(contents, " {\n$0$1}\n", formatted_options, + prefix); } else { contents->append(";\n"); } @@ -2944,16 +3537,16 @@ void MethodDescriptor::DebugString(int depth, string *contents, bool FileDescriptor::GetSourceLocation(const std::vector& path, SourceLocation* out_location) const { - GOOGLE_CHECK_NOTNULL(out_location); + GOOGLE_CHECK(out_location != nullptr); if (source_code_info_) { if (const SourceCodeInfo_Location* loc = - tables_->GetSourceLocation(path, source_code_info_)) { - const RepeatedField& span = loc->span(); + tables_->GetSourceLocation(path, source_code_info_)) { + const RepeatedField& span = loc->span(); if (span.size() == 3 || span.size() == 4) { - out_location->start_line = span.Get(0); + out_location->start_line = span.Get(0); out_location->start_column = span.Get(1); - out_location->end_line = span.Get(span.size() == 3 ? 0 : 2); - out_location->end_column = span.Get(span.size() - 1); + out_location->end_line = span.Get(span.size() == 3 ? 0 : 2); + out_location->end_column = span.Get(span.size() - 1); out_location->leading_comments = loc->leading_comments(); out_location->trailing_comments = loc->trailing_comments(); @@ -2975,9 +3568,9 @@ bool FileDescriptor::GetSourceLocation(SourceLocation* out_location) const { bool FieldDescriptor::is_packed() const { if (!is_packable()) return false; if (file_->syntax() == FileDescriptor::SYNTAX_PROTO2) { - return (options_ != NULL) && options_->packed(); + return (options_ != nullptr) && options_->packed(); } else { - return options_ == NULL || !options_->has_packed() || options_->packed(); + return options_ == nullptr || !options_->has_packed() || options_->packed(); } } @@ -3037,7 +3630,7 @@ void Descriptor::GetLocationPath(std::vector* output) const { void FieldDescriptor::GetLocationPath(std::vector* output) const { if (is_extension()) { - if (extension_scope() == NULL) { + if (extension_scope() == nullptr) { output->push_back(FileDescriptorProto::kExtensionFieldNumber); output->push_back(index()); } else { @@ -3096,17 +3689,17 @@ namespace { // pointers in the original options, not the mutable copy). The Message must be // one of the Options messages in descriptor.proto. struct OptionsToInterpret { - OptionsToInterpret(const string& ns, - const string& el, - const Message* orig_opt, + OptionsToInterpret(const std::string& ns, const std::string& el, + const std::vector& path, const Message* orig_opt, Message* opt) : name_scope(ns), element_name(el), + element_path(path), original_options(orig_opt), - options(opt) { - } - string name_scope; - string element_name; + options(opt) {} + std::string name_scope; + std::string element_name; + std::vector element_path; const Message* original_options; Message* options; }; @@ -3115,8 +3708,7 @@ struct OptionsToInterpret { class DescriptorBuilder { public: - DescriptorBuilder(const DescriptorPool* pool, - DescriptorPool::Tables* tables, + DescriptorBuilder(const DescriptorPool* pool, DescriptorPool::Tables* tables, DescriptorPool::ErrorCollector* error_collector); ~DescriptorBuilder(); @@ -3138,7 +3730,7 @@ class DescriptorBuilder { std::vector options_to_interpret_; bool had_errors_; - string filename_; + std::string filename_; FileDescriptor* file_; FileDescriptorTables* file_tables_; std::set dependencies_; @@ -3155,19 +3747,17 @@ class DescriptorBuilder { // actually found in possible_undeclared_dependency_, which may be a parent // of the symbol actually looked for. const FileDescriptor* possible_undeclared_dependency_; - string possible_undeclared_dependency_name_; + std::string possible_undeclared_dependency_name_; // If LookupSymbol() could resolve a symbol which is not defined, // record the resolved name. This is only used by AddNotDefinedError() // to report a more useful error message. - string undefine_resolved_name_; + std::string undefine_resolved_name_; - void AddError(const string& element_name, - const Message& descriptor, + void AddError(const std::string& element_name, const Message& descriptor, DescriptorPool::ErrorCollector::ErrorLocation location, - const string& error); - void AddError(const string& element_name, - const Message& descriptor, + const std::string& error); + void AddError(const std::string& element_name, const Message& descriptor, DescriptorPool::ErrorCollector::ErrorLocation location, const char* error); void AddRecursiveImportError(const FileDescriptorProto& proto, int from_here); @@ -3177,19 +3767,18 @@ class DescriptorBuilder { // Adds an error indicating that undefined_symbol was not defined. Must // only be called after LookupSymbol() fails. void AddNotDefinedError( - const string& element_name, - const Message& descriptor, - DescriptorPool::ErrorCollector::ErrorLocation location, - const string& undefined_symbol); + const std::string& element_name, const Message& descriptor, + DescriptorPool::ErrorCollector::ErrorLocation location, + const std::string& undefined_symbol); - void AddWarning(const string& element_name, const Message& descriptor, + void AddWarning(const std::string& element_name, const Message& descriptor, DescriptorPool::ErrorCollector::ErrorLocation location, - const string& error); + const std::string& error); // Silly helper which determines if the given file is in the given package. // I.e., either file->package() == package_name or file->package() is a // nested package within package_name. - bool IsInPackage(const FileDescriptor* file, const string& package_name); + bool IsInPackage(const FileDescriptor* file, const std::string& package_name); // Helper function which finds all public dependencies of the given file, and // stores the them in the dependencies_ set in the builder. @@ -3199,15 +3788,16 @@ class DescriptorBuilder { // - Search the pool's underlay if not found in tables_. // - Insure that the resulting Symbol is from one of the file's declared // dependencies. - Symbol FindSymbol(const string& name, bool build_it = true); + Symbol FindSymbol(const std::string& name, bool build_it = true); // Like FindSymbol() but does not require that the symbol is in one of the // file's declared dependencies. - Symbol FindSymbolNotEnforcingDeps(const string& name, bool build_it = true); + Symbol FindSymbolNotEnforcingDeps(const std::string& name, + bool build_it = true); // This implements the body of FindSymbolNotEnforcingDeps(). Symbol FindSymbolNotEnforcingDepsHelper(const DescriptorPool* pool, - const string& name, + const std::string& name, bool build_it = true); // Like FindSymbol(), but looks up the name relative to some other symbol @@ -3224,10 +3814,8 @@ class DescriptorBuilder { // that LookupSymbol may still return a non-type symbol in LOOKUP_TYPES mode, // if it believes that's all it could refer to. The caller should always // check that it receives the type of symbol it was expecting. - enum ResolveMode { - LOOKUP_ALL, LOOKUP_TYPES - }; - Symbol LookupSymbol(const string& name, const string& relative_to, + enum ResolveMode { LOOKUP_ALL, LOOKUP_TYPES }; + Symbol LookupSymbol(const std::string& name, const std::string& relative_to, DescriptorPool::PlaceholderType placeholder_type = DescriptorPool::PLACEHOLDER_MESSAGE, ResolveMode resolve_mode = LOOKUP_ALL, @@ -3235,29 +3823,28 @@ class DescriptorBuilder { // Like LookupSymbol() but will not return a placeholder even if // AllowUnknownDependencies() has been used. - Symbol LookupSymbolNoPlaceholder(const string& name, - const string& relative_to, + Symbol LookupSymbolNoPlaceholder(const std::string& name, + const std::string& relative_to, ResolveMode resolve_mode = LOOKUP_ALL, bool build_it = true); // Calls tables_->AddSymbol() and records an error if it fails. Returns // true if successful or false if failed, though most callers can ignore // the return value since an error has already been recorded. - bool AddSymbol(const string& full_name, - const void* parent, const string& name, - const Message& proto, Symbol symbol); + bool AddSymbol(const std::string& full_name, const void* parent, + const std::string& name, const Message& proto, Symbol symbol); // Like AddSymbol(), but succeeds if the symbol is already defined as long // as the existing definition is also a package (because it's OK to define // the same package in two different files). Also adds all parents of the - // packgae to the symbol table (e.g. AddPackage("foo.bar", ...) will add + // package to the symbol table (e.g. AddPackage("foo.bar", ...) will add // "foo.bar" and "foo" to the table). - void AddPackage(const string& name, const Message& proto, - const FileDescriptor* file); + void AddPackage(const std::string& name, const Message& proto, + FileDescriptor* file); // Checks that the symbol name contains only alphanumeric characters and // underscores. Records an error otherwise. - void ValidateSymbolName(const string& name, const string& full_name, + void ValidateSymbolName(const std::string& name, const std::string& full_name, const Message& proto); // Used by BUILD_ARRAY macro (below) to avoid having to have the type @@ -3271,36 +3858,41 @@ class DescriptorBuilder { // descriptor. Remembers its uninterpreted options, to be interpreted // later. DescriptorT must be one of the Descriptor messages from // descriptor.proto. - template void AllocateOptions( - const typename DescriptorT::OptionsType& orig_options, - DescriptorT* descriptor); + template + void AllocateOptions(const typename DescriptorT::OptionsType& orig_options, + DescriptorT* descriptor, int options_field_tag, + const std::string& option_name); // Specialization for FileOptions. void AllocateOptions(const FileOptions& orig_options, FileDescriptor* descriptor); // Implementation for AllocateOptions(). Don't call this directly. - template void AllocateOptionsImpl( - const string& name_scope, - const string& element_name, + template + void AllocateOptionsImpl( + const std::string& name_scope, const std::string& element_name, const typename DescriptorT::OptionsType& orig_options, - DescriptorT* descriptor); + DescriptorT* descriptor, const std::vector& options_path, + const std::string& option_name); + + // Allocates an array of two strings, the first one is a copy of `proto_name`, + // and the second one is the full name. + // Full proto name is "scope.proto_name" if scope is non-empty and + // "proto_name" otherwise. + const std::string* AllocateNameStrings(const std::string& scope, + const std::string& proto_name); // These methods all have the same signature for the sake of the BUILD_ARRAY // macro, below. - void BuildMessage(const DescriptorProto& proto, - const Descriptor* parent, + void BuildMessage(const DescriptorProto& proto, const Descriptor* parent, Descriptor* result); void BuildFieldOrExtension(const FieldDescriptorProto& proto, - const Descriptor* parent, - FieldDescriptor* result, + Descriptor* parent, FieldDescriptor* result, bool is_extension); - void BuildField(const FieldDescriptorProto& proto, - const Descriptor* parent, + void BuildField(const FieldDescriptorProto& proto, Descriptor* parent, FieldDescriptor* result) { BuildFieldOrExtension(proto, parent, result, false); } - void BuildExtension(const FieldDescriptorProto& proto, - const Descriptor* parent, + void BuildExtension(const FieldDescriptorProto& proto, Descriptor* parent, FieldDescriptor* result) { BuildFieldOrExtension(proto, parent, result, true); } @@ -3308,28 +3900,24 @@ class DescriptorBuilder { const Descriptor* parent, Descriptor::ExtensionRange* result); void BuildReservedRange(const DescriptorProto::ReservedRange& proto, - const Descriptor* parent, - Descriptor::ReservedRange* result); + const Descriptor* parent, + Descriptor::ReservedRange* result); void BuildReservedRange(const EnumDescriptorProto::EnumReservedRange& proto, const EnumDescriptor* parent, EnumDescriptor::ReservedRange* result); - void BuildOneof(const OneofDescriptorProto& proto, - Descriptor* parent, + void BuildOneof(const OneofDescriptorProto& proto, Descriptor* parent, OneofDescriptor* result); void CheckEnumValueUniqueness(const EnumDescriptorProto& proto, const EnumDescriptor* result); - void BuildEnum(const EnumDescriptorProto& proto, - const Descriptor* parent, + void BuildEnum(const EnumDescriptorProto& proto, const Descriptor* parent, EnumDescriptor* result); void BuildEnumValue(const EnumValueDescriptorProto& proto, const EnumDescriptor* parent, EnumValueDescriptor* result); - void BuildService(const ServiceDescriptorProto& proto, - const void* dummy, + void BuildService(const ServiceDescriptorProto& proto, const void* dummy, ServiceDescriptor* result); void BuildMethod(const MethodDescriptorProto& proto, - const ServiceDescriptor* parent, - MethodDescriptor* result); + const ServiceDescriptor* parent, MethodDescriptor* result); void LogUnusedDependency(const FileDescriptorProto& proto, const FileDescriptor* result); @@ -3361,8 +3949,8 @@ class DescriptorBuilder { class OptionInterpreter { public: // Creates an interpreter that operates in the context of the pool of the - // specified builder, which must not be NULL. We don't take ownership of the - // builder. + // specified builder, which must not be nullptr. We don't take ownership of + // the builder. explicit OptionInterpreter(DescriptorBuilder* builder); ~OptionInterpreter(); @@ -3372,13 +3960,22 @@ class DescriptorBuilder { // Otherwise returns true. bool InterpretOptions(OptionsToInterpret* options_to_interpret); + // Updates the given source code info by re-writing uninterpreted option + // locations to refer to the corresponding interpreted option. + void UpdateSourceCodeInfo(SourceCodeInfo* info); + class AggregateOptionFinder; private: // Interprets uninterpreted_option_ on the specified message, which // must be the mutable copy of the original options message to which - // uninterpreted_option_ belongs. - bool InterpretSingleOption(Message* options); + // uninterpreted_option_ belongs. The given src_path is the source + // location path to the uninterpreted option, and options_path is the + // source location path to the options message. The location paths are + // recorded and then used in UpdateSourceCodeInfo. + bool InterpretSingleOption(Message* options, + const std::vector& src_path, + const std::vector& options_path); // Adds the uninterpreted_option to the given options message verbatim. // Used when AllowUnknownDependencies() is in effect and we can't find @@ -3394,7 +3991,8 @@ class DescriptorBuilder { intermediate_fields_iter, std::vector::const_iterator intermediate_fields_end, - const FieldDescriptor* innermost_field, const string& debug_msg_name, + const FieldDescriptor* innermost_field, + const std::string& debug_msg_name, const UnknownFieldSet& unknown_fields); // Validates the value for the option field of the currently interpreted @@ -3409,19 +4007,19 @@ class DescriptorBuilder { // Convenience functions to set an int field the right way, depending on // its wire type (a single int CppType can represent multiple wire types). - void SetInt32(int number, int32 value, FieldDescriptor::Type type, + void SetInt32(int number, int32_t value, FieldDescriptor::Type type, UnknownFieldSet* unknown_fields); - void SetInt64(int number, int64 value, FieldDescriptor::Type type, + void SetInt64(int number, int64_t value, FieldDescriptor::Type type, UnknownFieldSet* unknown_fields); - void SetUInt32(int number, uint32 value, FieldDescriptor::Type type, + void SetUInt32(int number, uint32_t value, FieldDescriptor::Type type, UnknownFieldSet* unknown_fields); - void SetUInt64(int number, uint64 value, FieldDescriptor::Type type, + void SetUInt64(int number, uint64_t value, FieldDescriptor::Type type, UnknownFieldSet* unknown_fields); // A helper function that adds an error at the specified location of the // option we're currently interpreting, and returns false. bool AddOptionError(DescriptorPool::ErrorCollector::ErrorLocation location, - const string& msg) { + const std::string& msg) { builder_->AddError(options_to_interpret_->element_name, *uninterpreted_option_, location, msg); return false; @@ -3429,30 +4027,44 @@ class DescriptorBuilder { // A helper function that adds an error at the location of the option name // and returns false. - bool AddNameError(const string& msg) { + bool AddNameError(const std::string& msg) { +#ifdef PROTOBUF_INTERNAL_IGNORE_FIELD_NAME_ERRORS_ + return true; +#else // PROTOBUF_INTERNAL_IGNORE_FIELD_NAME_ERRORS_ return AddOptionError(DescriptorPool::ErrorCollector::OPTION_NAME, msg); +#endif // PROTOBUF_INTERNAL_IGNORE_FIELD_NAME_ERRORS_ } // A helper function that adds an error at the location of the option name // and returns false. - bool AddValueError(const string& msg) { + bool AddValueError(const std::string& msg) { return AddOptionError(DescriptorPool::ErrorCollector::OPTION_VALUE, msg); } - // We interpret against this builder's pool. Is never NULL. We don't own + // We interpret against this builder's pool. Is never nullptr. We don't own // this pointer. DescriptorBuilder* builder_; - // The options we're currently interpreting, or NULL if we're not in a call - // to InterpretOptions. + // The options we're currently interpreting, or nullptr if we're not in a + // call to InterpretOptions. const OptionsToInterpret* options_to_interpret_; // The option we're currently interpreting within options_to_interpret_, or - // NULL if we're not in a call to InterpretOptions(). This points to a + // nullptr if we're not in a call to InterpretOptions(). This points to a // submessage of the original option, not the mutable copy. Therefore we // can use it to find locations recorded by the parser. const UninterpretedOption* uninterpreted_option_; + // This maps the element path of uninterpreted options to the element path + // of the resulting interpreted option. This is used to modify a file's + // source code info to account for option interpretation. + std::map, std::vector> interpreted_paths_; + + // This maps the path to a repeated option field to the known number of + // elements the field contains. This is used to track the compute the + // index portion of the element path when interpreting a single option. + std::map, int> repeated_option_counts_; + // Factory used to create the dynamic messages we need to parse // any aggregate option values we encounter. DynamicMessageFactory dynamic_factory_; @@ -3477,10 +4089,10 @@ class DescriptorBuilder { return pool->enforce_weak_; } static inline bool get_is_placeholder(const Descriptor* descriptor) { - return descriptor->is_placeholder_; + return descriptor != nullptr && descriptor->is_placeholder_; } static inline void assert_mutex_held(const DescriptorPool* pool) { - if (pool->mutex_ != NULL) { + if (pool->mutex_ != nullptr) { pool->mutex_->AssertHeld(); } } @@ -3501,14 +4113,15 @@ class DescriptorBuilder { const EnumDescriptorProto& proto); void ValidateEnumValueOptions(EnumValueDescriptor* enum_value, const EnumValueDescriptorProto& proto); + void ValidateExtensionRangeOptions( + const std::string& full_name, Descriptor::ExtensionRange* extension_range, + const DescriptorProto_ExtensionRange& proto); void ValidateServiceOptions(ServiceDescriptor* service, const ServiceDescriptorProto& proto); void ValidateMethodOptions(MethodDescriptor* method, const MethodDescriptorProto& proto); - void ValidateProto3(FileDescriptor* file, - const FileDescriptorProto& proto); - void ValidateProto3Message(Descriptor* message, - const DescriptorProto& proto); + void ValidateProto3(FileDescriptor* file, const FileDescriptorProto& proto); + void ValidateProto3Message(Descriptor* message, const DescriptorProto& proto); void ValidateProto3Field(FieldDescriptor* field, const FieldDescriptorProto& proto); void ValidateProto3Enum(EnumDescriptor* enm, @@ -3530,119 +4143,116 @@ class DescriptorBuilder { const FileDescriptor* DescriptorPool::BuildFile( const FileDescriptorProto& proto) { - GOOGLE_CHECK(fallback_database_ == NULL) - << "Cannot call BuildFile on a DescriptorPool that uses a " - "DescriptorDatabase. You must instead find a way to get your file " - "into the underlying database."; - GOOGLE_CHECK(mutex_ == NULL); // Implied by the above GOOGLE_CHECK. + GOOGLE_CHECK(fallback_database_ == nullptr) + << "Cannot call BuildFile on a DescriptorPool that uses a " + "DescriptorDatabase. You must instead find a way to get your file " + "into the underlying database."; + GOOGLE_CHECK(mutex_ == nullptr); // Implied by the above GOOGLE_CHECK. tables_->known_bad_symbols_.clear(); tables_->known_bad_files_.clear(); - return DescriptorBuilder(this, tables_.get(), NULL).BuildFile(proto); + return DescriptorBuilder(this, tables_.get(), nullptr).BuildFile(proto); } const FileDescriptor* DescriptorPool::BuildFileCollectingErrors( - const FileDescriptorProto& proto, - ErrorCollector* error_collector) { - GOOGLE_CHECK(fallback_database_ == NULL) - << "Cannot call BuildFile on a DescriptorPool that uses a " - "DescriptorDatabase. You must instead find a way to get your file " - "into the underlying database."; - GOOGLE_CHECK(mutex_ == NULL); // Implied by the above GOOGLE_CHECK. + const FileDescriptorProto& proto, ErrorCollector* error_collector) { + GOOGLE_CHECK(fallback_database_ == nullptr) + << "Cannot call BuildFile on a DescriptorPool that uses a " + "DescriptorDatabase. You must instead find a way to get your file " + "into the underlying database."; + GOOGLE_CHECK(mutex_ == nullptr); // Implied by the above GOOGLE_CHECK. tables_->known_bad_symbols_.clear(); tables_->known_bad_files_.clear(); - return DescriptorBuilder(this, tables_.get(), - error_collector).BuildFile(proto); + return DescriptorBuilder(this, tables_.get(), error_collector) + .BuildFile(proto); } const FileDescriptor* DescriptorPool::BuildFileFromDatabase( const FileDescriptorProto& proto) const { mutex_->AssertHeld(); if (tables_->known_bad_files_.count(proto.name()) > 0) { - return NULL; + return nullptr; } const FileDescriptor* result = - DescriptorBuilder(this, tables_.get(), - default_error_collector_).BuildFile(proto); - if (result == NULL) { + DescriptorBuilder(this, tables_.get(), default_error_collector_) + .BuildFile(proto); + if (result == nullptr) { tables_->known_bad_files_.insert(proto.name()); } return result; } DescriptorBuilder::DescriptorBuilder( - const DescriptorPool* pool, - DescriptorPool::Tables* tables, + const DescriptorPool* pool, DescriptorPool::Tables* tables, DescriptorPool::ErrorCollector* error_collector) - : pool_(pool), - tables_(tables), - error_collector_(error_collector), - had_errors_(false), - possible_undeclared_dependency_(NULL), - undefine_resolved_name_("") {} + : pool_(pool), + tables_(tables), + error_collector_(error_collector), + had_errors_(false), + possible_undeclared_dependency_(nullptr), + undefine_resolved_name_("") {} DescriptorBuilder::~DescriptorBuilder() {} void DescriptorBuilder::AddError( - const string& element_name, - const Message& descriptor, + const std::string& element_name, const Message& descriptor, DescriptorPool::ErrorCollector::ErrorLocation location, - const string& error) { - if (error_collector_ == NULL) { + const std::string& error) { + if (error_collector_ == nullptr) { if (!had_errors_) { GOOGLE_LOG(ERROR) << "Invalid proto descriptor for file \"" << filename_ << "\":"; } GOOGLE_LOG(ERROR) << " " << element_name << ": " << error; } else { - error_collector_->AddError(filename_, element_name, - &descriptor, location, error); + error_collector_->AddError(filename_, element_name, &descriptor, location, + error); } had_errors_ = true; } void DescriptorBuilder::AddError( - const string& element_name, - const Message& descriptor, - DescriptorPool::ErrorCollector::ErrorLocation location, - const char* error) { - AddError(element_name, descriptor, location, string(error)); + const std::string& element_name, const Message& descriptor, + DescriptorPool::ErrorCollector::ErrorLocation location, const char* error) { + AddError(element_name, descriptor, location, std::string(error)); } void DescriptorBuilder::AddNotDefinedError( - const string& element_name, - const Message& descriptor, + const std::string& element_name, const Message& descriptor, DescriptorPool::ErrorCollector::ErrorLocation location, - const string& undefined_symbol) { - if (possible_undeclared_dependency_ == NULL && + const std::string& undefined_symbol) { + if (possible_undeclared_dependency_ == nullptr && undefine_resolved_name_.empty()) { AddError(element_name, descriptor, location, "\"" + undefined_symbol + "\" is not defined."); } else { - if (possible_undeclared_dependency_ != NULL) { + if (possible_undeclared_dependency_ != nullptr) { AddError(element_name, descriptor, location, "\"" + possible_undeclared_dependency_name_ + - "\" seems to be defined in \"" + - possible_undeclared_dependency_->name() + "\", which is not " - "imported by \"" + filename_ + "\". To use it here, please " - "add the necessary import."); + "\" seems to be defined in \"" + + possible_undeclared_dependency_->name() + + "\", which is not " + "imported by \"" + + filename_ + + "\". To use it here, please " + "add the necessary import."); } if (!undefine_resolved_name_.empty()) { AddError(element_name, descriptor, location, "\"" + undefined_symbol + "\" is resolved to \"" + - undefine_resolved_name_ + "\", which is not defined. " - "The innermost scope is searched first in name resolution. " - "Consider using a leading '.'(i.e., \"." - + undefined_symbol + - "\") to start from the outermost scope."); + undefine_resolved_name_ + + "\", which is not defined. " + "The innermost scope is searched first in name resolution. " + "Consider using a leading '.'(i.e., \"." + + undefined_symbol + "\") to start from the outermost scope."); } } } void DescriptorBuilder::AddWarning( - const string& element_name, const Message& descriptor, + const std::string& element_name, const Message& descriptor, DescriptorPool::ErrorCollector::ErrorLocation location, - const string& error) { - if (error_collector_ == NULL) { + const std::string& error) { + if (error_collector_ == nullptr) { GOOGLE_LOG(WARNING) << filename_ << " " << element_name << ": " << error; } else { error_collector_->AddWarning(filename_, element_name, &descriptor, location, @@ -3651,27 +4261,27 @@ void DescriptorBuilder::AddWarning( } bool DescriptorBuilder::IsInPackage(const FileDescriptor* file, - const string& package_name) { + const std::string& package_name) { return HasPrefixString(file->package(), package_name) && (file->package().size() == package_name.size() || file->package()[package_name.size()] == '.'); } void DescriptorBuilder::RecordPublicDependencies(const FileDescriptor* file) { - if (file == NULL || !dependencies_.insert(file).second) return; - for (int i = 0; file != NULL && i < file->public_dependency_count(); i++) { + if (file == nullptr || !dependencies_.insert(file).second) return; + for (int i = 0; file != nullptr && i < file->public_dependency_count(); i++) { RecordPublicDependencies(file->public_dependency(i)); } } Symbol DescriptorBuilder::FindSymbolNotEnforcingDepsHelper( - const DescriptorPool* pool, const string& name, bool build_it) { + const DescriptorPool* pool, const std::string& name, bool build_it) { // If we are looking at an underlay, we must lock its mutex_, since we are // accessing the underlay's tables_ directly. - MutexLockMaybe lock((pool == pool_) ? NULL : pool->mutex_); + MutexLockMaybe lock((pool == pool_) ? nullptr : pool->mutex_); Symbol result = pool->tables_->FindSymbol(name); - if (result.IsNull() && pool->underlay_ != NULL) { + if (result.IsNull() && pool->underlay_ != nullptr) { // Symbol not found; check the underlay. result = FindSymbolNotEnforcingDepsHelper(pool->underlay_, name); } @@ -3692,12 +4302,19 @@ Symbol DescriptorBuilder::FindSymbolNotEnforcingDepsHelper( return result; } -Symbol DescriptorBuilder::FindSymbolNotEnforcingDeps(const string& name, +Symbol DescriptorBuilder::FindSymbolNotEnforcingDeps(const std::string& name, bool build_it) { - return FindSymbolNotEnforcingDepsHelper(pool_, name, build_it); + Symbol result = FindSymbolNotEnforcingDepsHelper(pool_, name, build_it); + // Only find symbols which were defined in this file or one of its + // dependencies. + const FileDescriptor* file = result.GetFile(); + if (file == file_ || dependencies_.count(file) > 0) { + unused_dependency_.erase(file); + } + return result; } -Symbol DescriptorBuilder::FindSymbol(const string& name, bool build_it) { +Symbol DescriptorBuilder::FindSymbol(const std::string& name, bool build_it) { Symbol result = FindSymbolNotEnforcingDeps(name, build_it); if (result.IsNull()) return result; @@ -3711,11 +4328,10 @@ Symbol DescriptorBuilder::FindSymbol(const string& name, bool build_it) { // dependencies. const FileDescriptor* file = result.GetFile(); if (file == file_ || dependencies_.count(file) > 0) { - unused_dependency_.erase(file); return result; } - if (result.type == Symbol::PACKAGE) { + if (result.type() == Symbol::PACKAGE) { // Arg, this is overcomplicated. The symbol is a package name. It could // be that the package was defined in multiple files. result.GetFile() // returns the first file we saw that used this package. We've determined @@ -3727,8 +4343,8 @@ Symbol DescriptorBuilder::FindSymbol(const string& name, bool build_it) { for (std::set::const_iterator it = dependencies_.begin(); it != dependencies_.end(); ++it) { - // Note: A dependency may be NULL if it was not found or had errors. - if (*it != NULL && IsInPackage(*it, name)) return result; + // Note: A dependency may be nullptr if it was not found or had errors. + if (*it != nullptr && IsInPackage(*it, name)) return result; } } @@ -3737,11 +4353,10 @@ Symbol DescriptorBuilder::FindSymbol(const string& name, bool build_it) { return kNullSymbol; } -Symbol DescriptorBuilder::LookupSymbolNoPlaceholder(const string& name, - const string& relative_to, - ResolveMode resolve_mode, - bool build_it) { - possible_undeclared_dependency_ = NULL; +Symbol DescriptorBuilder::LookupSymbolNoPlaceholder( + const std::string& name, const std::string& relative_to, + ResolveMode resolve_mode, bool build_it) { + possible_undeclared_dependency_ = nullptr; undefine_resolved_name_.clear(); if (!name.empty() && name[0] == '.') { @@ -3760,27 +4375,27 @@ Symbol DescriptorBuilder::LookupSymbolNoPlaceholder(const string& name, // } // So, we look for just "Foo" first, then look for "Bar.baz" within it if // found. - string::size_type name_dot_pos = name.find_first_of('.'); - string first_part_of_name; - if (name_dot_pos == string::npos) { + std::string::size_type name_dot_pos = name.find_first_of('.'); + std::string first_part_of_name; + if (name_dot_pos == std::string::npos) { first_part_of_name = name; } else { first_part_of_name = name.substr(0, name_dot_pos); } - string scope_to_try(relative_to); + std::string scope_to_try(relative_to); while (true) { // Chop off the last component of the scope. - string::size_type dot_pos = scope_to_try.find_last_of('.'); - if (dot_pos == string::npos) { + std::string::size_type dot_pos = scope_to_try.find_last_of('.'); + if (dot_pos == std::string::npos) { return FindSymbol(name, build_it); } else { scope_to_try.erase(dot_pos); } // Append ".first_part_of_name" and try to find. - string::size_type old_size = scope_to_try.size(); + std::string::size_type old_size = scope_to_try.size(); scope_to_try.append(1, '.'); scope_to_try.append(first_part_of_name); Symbol result = FindSymbol(scope_to_try, build_it); @@ -3814,7 +4429,7 @@ Symbol DescriptorBuilder::LookupSymbolNoPlaceholder(const string& name, } Symbol DescriptorBuilder::LookupSymbol( - const string& name, const string& relative_to, + const std::string& name, const std::string& relative_to, DescriptorPool::PlaceholderType placeholder_type, ResolveMode resolve_mode, bool build_it) { Symbol result = @@ -3827,16 +4442,16 @@ Symbol DescriptorBuilder::LookupSymbol( return result; } -static bool ValidateQualifiedName(const string& name) { +static bool ValidateQualifiedName(StringPiece name) { bool last_was_period = false; - for (int i = 0; i < name.size(); i++) { + for (char character : name) { // I don't trust isalnum() due to locales. :( - if (('a' <= name[i] && name[i] <= 'z') || - ('A' <= name[i] && name[i] <= 'Z') || - ('0' <= name[i] && name[i] <= '9') || (name[i] == '_')) { + if (('a' <= character && character <= 'z') || + ('A' <= character && character <= 'Z') || + ('0' <= character && character <= '9') || (character == '_')) { last_was_period = false; - } else if (name[i] == '.') { + } else if (character == '.') { if (last_was_period) return false; last_was_period = true; } else { @@ -3847,36 +4462,35 @@ static bool ValidateQualifiedName(const string& name) { return !name.empty() && !last_was_period; } -Symbol DescriptorPool::NewPlaceholder(const string& name, +Symbol DescriptorPool::NewPlaceholder(StringPiece name, PlaceholderType placeholder_type) const { MutexLockMaybe lock(mutex_); return NewPlaceholderWithMutexHeld(name, placeholder_type); } Symbol DescriptorPool::NewPlaceholderWithMutexHeld( - const string& name, PlaceholderType placeholder_type) const { + StringPiece name, PlaceholderType placeholder_type) const { if (mutex_) { mutex_->AssertHeld(); } // Compute names. - const string* placeholder_full_name; - const string* placeholder_name; - const string* placeholder_package; + StringPiece placeholder_full_name; + StringPiece placeholder_name; + const std::string* placeholder_package; if (!ValidateQualifiedName(name)) return kNullSymbol; if (name[0] == '.') { // Fully-qualified. - placeholder_full_name = tables_->AllocateString(name.substr(1)); + placeholder_full_name = name.substr(1); } else { - placeholder_full_name = tables_->AllocateString(name); + placeholder_full_name = name; } - string::size_type dotpos = placeholder_full_name->find_last_of('.'); - if (dotpos != string::npos) { - placeholder_package = tables_->AllocateString( - placeholder_full_name->substr(0, dotpos)); - placeholder_name = tables_->AllocateString( - placeholder_full_name->substr(dotpos + 1)); + std::string::size_type dotpos = placeholder_full_name.find_last_of('.'); + if (dotpos != std::string::npos) { + placeholder_package = + tables_->AllocateString(placeholder_full_name.substr(0, dotpos)); + placeholder_name = placeholder_full_name.substr(dotpos + 1); } else { placeholder_package = &internal::GetEmptyString(); placeholder_name = placeholder_full_name; @@ -3884,19 +4498,18 @@ Symbol DescriptorPool::NewPlaceholderWithMutexHeld( // Create the placeholders. FileDescriptor* placeholder_file = NewPlaceholderFileWithMutexHeld( - *placeholder_full_name + ".placeholder.proto"); + StrCat(placeholder_full_name, ".placeholder.proto")); placeholder_file->package_ = placeholder_package; if (placeholder_type == PLACEHOLDER_ENUM) { placeholder_file->enum_type_count_ = 1; - placeholder_file->enum_types_ = - tables_->AllocateArray(1); + placeholder_file->enum_types_ = tables_->AllocateArray(1); EnumDescriptor* placeholder_enum = &placeholder_file->enum_types_[0]; - memset(placeholder_enum, 0, sizeof(*placeholder_enum)); + memset(static_cast(placeholder_enum), 0, sizeof(*placeholder_enum)); - placeholder_enum->full_name_ = placeholder_full_name; - placeholder_enum->name_ = placeholder_name; + placeholder_enum->all_names_ = + tables_->AllocateStringArray(placeholder_name, placeholder_full_name); placeholder_enum->file_ = placeholder_file; placeholder_enum->options_ = &EnumOptions::default_instance(); placeholder_enum->is_placeholder_ = true; @@ -3905,15 +4518,18 @@ Symbol DescriptorPool::NewPlaceholderWithMutexHeld( // Enums must have at least one value. placeholder_enum->value_count_ = 1; placeholder_enum->values_ = tables_->AllocateArray(1); + // Disable fast-path lookup for this enum. + placeholder_enum->sequential_value_limit_ = -1; EnumValueDescriptor* placeholder_value = &placeholder_enum->values_[0]; - memset(placeholder_value, 0, sizeof(*placeholder_value)); + memset(static_cast(placeholder_value), 0, + sizeof(*placeholder_value)); - placeholder_value->name_ = tables_->AllocateString("PLACEHOLDER_VALUE"); // Note that enum value names are siblings of their type, not children. - placeholder_value->full_name_ = - placeholder_package->empty() ? placeholder_value->name_ : - tables_->AllocateString(*placeholder_package + ".PLACEHOLDER_VALUE"); + placeholder_value->all_names_ = tables_->AllocateStringArray( + "PLACEHOLDER_VALUE", placeholder_package->empty() + ? "PLACEHOLDER_VALUE" + : *placeholder_package + ".PLACEHOLDER_VALUE"); placeholder_value->number_ = 0; placeholder_value->type_ = placeholder_enum; @@ -3922,14 +4538,14 @@ Symbol DescriptorPool::NewPlaceholderWithMutexHeld( return Symbol(placeholder_enum); } else { placeholder_file->message_type_count_ = 1; - placeholder_file->message_types_ = - tables_->AllocateArray(1); + placeholder_file->message_types_ = tables_->AllocateArray(1); Descriptor* placeholder_message = &placeholder_file->message_types_[0]; - memset(placeholder_message, 0, sizeof(*placeholder_message)); + memset(static_cast(placeholder_message), 0, + sizeof(*placeholder_message)); - placeholder_message->full_name_ = placeholder_full_name; - placeholder_message->name_ = placeholder_name; + placeholder_message->all_names_ = + tables_->AllocateStringArray(placeholder_name, placeholder_full_name); placeholder_message->file_ = placeholder_file; placeholder_message->options_ = &MessageOptions::default_instance(); placeholder_message->is_placeholder_ = true; @@ -3938,29 +4554,31 @@ Symbol DescriptorPool::NewPlaceholderWithMutexHeld( if (placeholder_type == PLACEHOLDER_EXTENDABLE_MESSAGE) { placeholder_message->extension_range_count_ = 1; placeholder_message->extension_ranges_ = - tables_->AllocateArray(1); + tables_->AllocateArray(1); placeholder_message->extension_ranges_->start = 1; // kMaxNumber + 1 because ExtensionRange::end is exclusive. placeholder_message->extension_ranges_->end = - FieldDescriptor::kMaxNumber + 1; + FieldDescriptor::kMaxNumber + 1; + placeholder_message->extension_ranges_->options_ = nullptr; } return Symbol(placeholder_message); } } -FileDescriptor* DescriptorPool::NewPlaceholderFile(const string& name) const { +FileDescriptor* DescriptorPool::NewPlaceholderFile( + StringPiece name) const { MutexLockMaybe lock(mutex_); return NewPlaceholderFileWithMutexHeld(name); } FileDescriptor* DescriptorPool::NewPlaceholderFileWithMutexHeld( - const string& name) const { + StringPiece name) const { if (mutex_) { mutex_->AssertHeld(); } FileDescriptor* placeholder = tables_->Allocate(); - memset(placeholder, 0, sizeof(*placeholder)); + memset(static_cast(placeholder), 0, sizeof(*placeholder)); placeholder->name_ = tables_->AllocateString(name); placeholder->package_ = &internal::GetEmptyString(); @@ -3969,26 +4587,32 @@ FileDescriptor* DescriptorPool::NewPlaceholderFileWithMutexHeld( placeholder->tables_ = &FileDescriptorTables::GetEmptyInstance(); placeholder->source_code_info_ = &SourceCodeInfo::default_instance(); placeholder->is_placeholder_ = true; - placeholder->syntax_ = FileDescriptor::SYNTAX_PROTO2; + placeholder->syntax_ = FileDescriptor::SYNTAX_UNKNOWN; placeholder->finished_building_ = true; - // All other fields are zero or NULL. + // All other fields are zero or nullptr. return placeholder; } -bool DescriptorBuilder::AddSymbol( - const string& full_name, const void* parent, const string& name, - const Message& proto, Symbol symbol) { - // If the caller passed NULL for the parent, the symbol is at file scope. +bool DescriptorBuilder::AddSymbol(const std::string& full_name, + const void* parent, const std::string& name, + const Message& proto, Symbol symbol) { + // If the caller passed nullptr for the parent, the symbol is at file scope. // Use its file as the parent instead. - if (parent == NULL) parent = file_; + if (parent == nullptr) parent = file_; + if (full_name.find('\0') != std::string::npos) { + AddError(full_name, proto, DescriptorPool::ErrorCollector::NAME, + "\"" + full_name + "\" contains null character."); + return false; + } if (tables_->AddSymbol(full_name, symbol)) { if (!file_tables_->AddAliasUnderParent(parent, name, symbol)) { // This is only possible if there was already an error adding something of // the same name. if (!had_errors_) { - GOOGLE_LOG(DFATAL) << "\"" << full_name << "\" not previously defined in " + GOOGLE_LOG(DFATAL) << "\"" << full_name + << "\" not previously defined in " "symbols_by_name_, but was defined in " "symbols_by_parent_; this shouldn't be possible."; } @@ -3998,65 +4622,77 @@ bool DescriptorBuilder::AddSymbol( } else { const FileDescriptor* other_file = tables_->FindSymbol(full_name).GetFile(); if (other_file == file_) { - string::size_type dot_pos = full_name.find_last_of('.'); - if (dot_pos == string::npos) { + std::string::size_type dot_pos = full_name.find_last_of('.'); + if (dot_pos == std::string::npos) { AddError(full_name, proto, DescriptorPool::ErrorCollector::NAME, "\"" + full_name + "\" is already defined."); } else { AddError(full_name, proto, DescriptorPool::ErrorCollector::NAME, "\"" + full_name.substr(dot_pos + 1) + - "\" is already defined in \"" + - full_name.substr(0, dot_pos) + "\"."); + "\" is already defined in \"" + + full_name.substr(0, dot_pos) + "\"."); } } else { // Symbol seems to have been defined in a different file. AddError(full_name, proto, DescriptorPool::ErrorCollector::NAME, "\"" + full_name + "\" is already defined in file \"" + - other_file->name() + "\"."); + (other_file == nullptr ? "null" : other_file->name()) + + "\"."); } return false; } } -void DescriptorBuilder::AddPackage( - const string& name, const Message& proto, const FileDescriptor* file) { - if (tables_->AddSymbol(name, Symbol(file))) { - // Success. Also add parent package, if any. - string::size_type dot_pos = name.find_last_of('.'); - if (dot_pos == string::npos) { +void DescriptorBuilder::AddPackage(const std::string& name, + const Message& proto, FileDescriptor* file) { + if (name.find('\0') != std::string::npos) { + AddError(name, proto, DescriptorPool::ErrorCollector::NAME, + "\"" + name + "\" contains null character."); + return; + } + + Symbol existing_symbol = tables_->FindSymbol(name); + // It's OK to redefine a package. + if (existing_symbol.IsNull()) { + auto* package = tables_->AllocateArray(1); + // If the name is the package name, then it is already in the arena. + // If not, copy it there. It came from the call to AddPackage below. + package->name = + &name == &file->package() ? &name : tables_->AllocateString(name); + package->file = file; + tables_->AddSymbol(*package->name, Symbol(package)); + // Also add parent package, if any. + std::string::size_type dot_pos = name.find_last_of('.'); + if (dot_pos == std::string::npos) { // No parents. ValidateSymbolName(name, name, proto); } else { // Has parent. - string* parent_name = tables_->AllocateString(name.substr(0, dot_pos)); - AddPackage(*parent_name, proto, file); + AddPackage(name.substr(0, dot_pos), proto, file); ValidateSymbolName(name.substr(dot_pos + 1), name, proto); } - } else { - Symbol existing_symbol = tables_->FindSymbol(name); - // It's OK to redefine a package. - if (existing_symbol.type != Symbol::PACKAGE) { - // Symbol seems to have been defined in a different file. - AddError(name, proto, DescriptorPool::ErrorCollector::NAME, - "\"" + name + "\" is already defined (as something other than " - "a package) in file \"" + existing_symbol.GetFile()->name() + - "\"."); - } + } else if (existing_symbol.type() != Symbol::PACKAGE) { + // Symbol seems to have been defined in a different file. + AddError(name, proto, DescriptorPool::ErrorCollector::NAME, + "\"" + name + + "\" is already defined (as something other than " + "a package) in file \"" + + existing_symbol.GetFile()->name() + "\"."); } } -void DescriptorBuilder::ValidateSymbolName( - const string& name, const string& full_name, const Message& proto) { +void DescriptorBuilder::ValidateSymbolName(const std::string& name, + const std::string& full_name, + const Message& proto) { if (name.empty()) { AddError(full_name, proto, DescriptorPool::ErrorCollector::NAME, "Missing name."); } else { - for (int i = 0; i < name.size(); i++) { + for (char character : name) { // I don't trust isalnum() due to locales. :( - if ((name[i] < 'a' || 'z' < name[i]) && - (name[i] < 'A' || 'Z' < name[i]) && - (name[i] < '0' || '9' < name[i]) && - (name[i] != '_')) { + if ((character < 'a' || 'z' < character) && + (character < 'A' || 'Z' < character) && + (character < '0' || '9' < character) && (character != '_')) { AddError(full_name, proto, DescriptorPool::ErrorCollector::NAME, "\"" + name + "\" is not a valid identifier."); } @@ -4068,32 +4704,49 @@ void DescriptorBuilder::ValidateSymbolName( // This generic implementation is good for all descriptors except // FileDescriptor. -template void DescriptorBuilder::AllocateOptions( +template +void DescriptorBuilder::AllocateOptions( const typename DescriptorT::OptionsType& orig_options, - DescriptorT* descriptor) { + DescriptorT* descriptor, int options_field_tag, + const std::string& option_name) { + std::vector options_path; + descriptor->GetLocationPath(&options_path); + options_path.push_back(options_field_tag); AllocateOptionsImpl(descriptor->full_name(), descriptor->full_name(), - orig_options, descriptor); + orig_options, descriptor, options_path, option_name); } // We specialize for FileDescriptor. void DescriptorBuilder::AllocateOptions(const FileOptions& orig_options, FileDescriptor* descriptor) { + std::vector options_path; + options_path.push_back(FileDescriptorProto::kOptionsFieldNumber); // We add the dummy token so that LookupSymbol does the right thing. AllocateOptionsImpl(descriptor->package() + ".dummy", descriptor->name(), - orig_options, descriptor); + orig_options, descriptor, options_path, + "google.protobuf.FileOptions"); } -template void DescriptorBuilder::AllocateOptionsImpl( - const string& name_scope, - const string& element_name, +template +void DescriptorBuilder::AllocateOptionsImpl( + const std::string& name_scope, const std::string& element_name, const typename DescriptorT::OptionsType& orig_options, - DescriptorT* descriptor) { + DescriptorT* descriptor, const std::vector& options_path, + const std::string& option_name) { // We need to use a dummy pointer to work around a bug in older versions of // GCC. Otherwise, the following two lines could be replaced with: // typename DescriptorT::OptionsType* options = // tables_->AllocateMessage(); - typename DescriptorT::OptionsType* const dummy = NULL; + typename DescriptorT::OptionsType* const dummy = nullptr; typename DescriptorT::OptionsType* options = tables_->AllocateMessage(dummy); + + if (!orig_options.IsInitialized()) { + AddError(name_scope + "." + element_name, orig_options, + DescriptorPool::ErrorCollector::OPTION_NAME, + "Uninterpreted option is missing name or value."); + return; + } + // Avoid using MergeFrom()/CopyFrom() in this class to make it -fno-rtti // friendly. Without RTTI, MergeFrom() and CopyFrom() will fallback to the // reflection based method, which requires the Descriptor. However, we are in @@ -4109,51 +4762,75 @@ template void DescriptorBuilder::AllocateOptionsImpl( // OptionsType::GetDescriptor() to be called which may then deadlock since // we're still trying to build it. if (options->uninterpreted_option_size() > 0) { - options_to_interpret_.push_back( - OptionsToInterpret(name_scope, element_name, &orig_options, options)); + options_to_interpret_.push_back(OptionsToInterpret( + name_scope, element_name, options_path, &orig_options, options)); + } + + // If the custom option is in unknown fields, no need to interpret it. + // Remove the dependency file from unused_dependency. + const UnknownFieldSet& unknown_fields = orig_options.unknown_fields(); + if (!unknown_fields.empty()) { + // Can not use options->GetDescriptor() which may case deadlock. + Symbol msg_symbol = tables_->FindSymbol(option_name); + if (msg_symbol.type() == Symbol::MESSAGE) { + for (int i = 0; i < unknown_fields.field_count(); ++i) { + assert_mutex_held(pool_); + const FieldDescriptor* field = + pool_->InternalFindExtensionByNumberNoLock( + msg_symbol.descriptor(), unknown_fields.field(i).number()); + if (field) { + unused_dependency_.erase(field->file()); + } + } + } } } - // A common pattern: We want to convert a repeated field in the descriptor // to an array of values, calling some method to build each value. -#define BUILD_ARRAY(INPUT, OUTPUT, NAME, METHOD, PARENT) \ - OUTPUT->NAME##_count_ = INPUT.NAME##_size(); \ - AllocateArray(INPUT.NAME##_size(), &OUTPUT->NAME##s_); \ - for (int i = 0; i < INPUT.NAME##_size(); i++) { \ - METHOD(INPUT.NAME(i), PARENT, OUTPUT->NAME##s_ + i); \ +#define BUILD_ARRAY(INPUT, OUTPUT, NAME, METHOD, PARENT) \ + OUTPUT->NAME##_count_ = INPUT.NAME##_size(); \ + AllocateArray(INPUT.NAME##_size(), &OUTPUT->NAME##s_); \ + for (int i = 0; i < INPUT.NAME##_size(); i++) { \ + METHOD(INPUT.NAME(i), PARENT, OUTPUT->NAME##s_ + i); \ } void DescriptorBuilder::AddRecursiveImportError( const FileDescriptorProto& proto, int from_here) { - string error_message("File recursively imports itself: "); - for (int i = from_here; i < tables_->pending_files_.size(); i++) { + std::string error_message("File recursively imports itself: "); + for (size_t i = from_here; i < tables_->pending_files_.size(); i++) { error_message.append(tables_->pending_files_[i]); error_message.append(" -> "); } error_message.append(proto.name()); - AddError(proto.name(), proto, DescriptorPool::ErrorCollector::OTHER, - error_message); + if (static_cast(from_here) < tables_->pending_files_.size() - 1) { + AddError(tables_->pending_files_[from_here + 1], proto, + DescriptorPool::ErrorCollector::IMPORT, error_message); + } else { + AddError(proto.name(), proto, DescriptorPool::ErrorCollector::IMPORT, + error_message); + } } void DescriptorBuilder::AddTwiceListedError(const FileDescriptorProto& proto, int index) { - AddError(proto.name(), proto, DescriptorPool::ErrorCollector::OTHER, + AddError(proto.dependency(index), proto, + DescriptorPool::ErrorCollector::IMPORT, "Import \"" + proto.dependency(index) + "\" was listed twice."); } void DescriptorBuilder::AddImportError(const FileDescriptorProto& proto, int index) { - string message; - if (pool_->fallback_database_ == NULL) { - message = "Import \"" + proto.dependency(index) + - "\" has not been loaded."; + std::string message; + if (pool_->fallback_database_ == nullptr) { + message = "Import \"" + proto.dependency(index) + "\" has not been loaded."; } else { message = "Import \"" + proto.dependency(index) + "\" was not found or had errors."; } - AddError(proto.name(), proto, DescriptorPool::ErrorCollector::OTHER, message); + AddError(proto.dependency(index), proto, + DescriptorPool::ErrorCollector::IMPORT, message); } static bool ExistingFileMatchesProto(const FileDescriptor* existing_file, @@ -4181,7 +4858,7 @@ const FileDescriptor* DescriptorBuilder::BuildFile( // This is fine, because this idempotency "feature" really only exists to // accommodate one hack in the proto1->proto2 migration layer. const FileDescriptor* existing_file = tables_->FindFile(filename_); - if (existing_file != NULL) { + if (existing_file != nullptr) { // File already in pool. Compare the existing one to the input. if (ExistingFileMatchesProto(existing_file, proto)) { // They're identical. Return the existing descriptor. @@ -4200,10 +4877,10 @@ const FileDescriptor* DescriptorBuilder::BuildFile( // mid-file, but that's pretty ugly, and I'm pretty sure there are // some languages out there that do not allow recursive dependencies // at all. - for (int i = 0; i < tables_->pending_files_.size(); i++) { + for (size_t i = 0; i < tables_->pending_files_.size(); i++) { if (tables_->pending_files_[i] == proto.name()) { AddRecursiveImportError(proto, i); - return NULL; + return nullptr; } } @@ -4211,12 +4888,13 @@ const FileDescriptor* DescriptorBuilder::BuildFile( // attempt to load all dependencies now, before checkpointing tables_. This // avoids confusion with recursive checkpoints. if (!pool_->lazily_build_dependencies_) { - if (pool_->fallback_database_ != NULL) { + if (pool_->fallback_database_ != nullptr) { tables_->pending_files_.push_back(proto.name()); for (int i = 0; i < proto.dependency_size(); i++) { - if (tables_->FindFile(proto.dependency(i)) == NULL && - (pool_->underlay_ == NULL || - pool_->underlay_->FindFileByName(proto.dependency(i)) == NULL)) { + if (tables_->FindFile(proto.dependency(i)) == nullptr && + (pool_->underlay_ == nullptr || + pool_->underlay_->FindFileByName(proto.dependency(i)) == + nullptr)) { // We don't care what this returns since we'll find out below anyway. pool_->TryFindFileInFallbackDatabase(proto.dependency(i)); } @@ -4248,8 +4926,9 @@ FileDescriptor* DescriptorBuilder::BuildFileImpl( result->is_placeholder_ = false; result->finished_building_ = false; + SourceCodeInfo* info = nullptr; if (proto.has_source_code_info()) { - SourceCodeInfo *info = tables_->AllocateMessage(); + info = tables_->AllocateMessage(); info->CopyFrom(proto.source_code_info()); result->source_code_info_ = info; } else { @@ -4288,35 +4967,30 @@ FileDescriptor* DescriptorBuilder::BuildFileImpl( } result->pool_ = pool_; + if (result->name().find('\0') != std::string::npos) { + AddError(result->name(), proto, DescriptorPool::ErrorCollector::NAME, + "\"" + result->name() + "\" contains null character."); + return nullptr; + } + // Add to tables. if (!tables_->AddFile(result)) { AddError(proto.name(), proto, DescriptorPool::ErrorCollector::OTHER, "A file with this name is already in the pool."); // Bail out early so that if this is actually the exact same file, we // don't end up reporting that every single symbol is already defined. - return NULL; + return nullptr; } if (!result->package().empty()) { AddPackage(result->package(), proto, result); } // Make sure all dependencies are loaded. - std::set seen_dependencies; + std::set seen_dependencies; result->dependency_count_ = proto.dependency_size(); result->dependencies_ = tables_->AllocateArray(proto.dependency_size()); - if (pool_->lazily_build_dependencies_) { - result->dependencies_once_ = tables_->AllocateOnceDynamic(); - result->dependencies_names_ = - tables_->AllocateArray(proto.dependency_size()); - if (proto.dependency_size() > 0) { - memset(result->dependencies_names_, 0, - sizeof(*result->dependencies_names_) * proto.dependency_size()); - } - } else { - result->dependencies_once_ = NULL; - result->dependencies_names_ = NULL; - } + result->dependencies_once_ = nullptr; unused_dependency_.clear(); std::set weak_deps; for (int i = 0; i < proto.weak_dependency_size(); ++i) { @@ -4328,7 +5002,7 @@ FileDescriptor* DescriptorBuilder::BuildFileImpl( } const FileDescriptor* dependency = tables_->FindFile(proto.dependency(i)); - if (dependency == NULL && pool_->underlay_ != NULL) { + if (dependency == nullptr && pool_->underlay_ != nullptr) { dependency = pool_->underlay_->FindFileByName(proto.dependency(i)); } @@ -4336,10 +5010,10 @@ FileDescriptor* DescriptorBuilder::BuildFileImpl( // Recursive import. dependency/result is not fully initialized, and it's // dangerous to try to do anything with it. The recursive import error // will be detected and reported in DescriptorBuilder::BuildFile(). - return NULL; + return nullptr; } - if (dependency == NULL) { + if (dependency == nullptr) { if (!pool_->lazily_build_dependencies_) { if (pool_->allow_unknown_ || (!pool_->enforce_weak_ && weak_deps.find(i) != weak_deps.end())) { @@ -4362,15 +5036,26 @@ FileDescriptor* DescriptorBuilder::BuildFileImpl( result->dependencies_[i] = dependency; if (pool_->lazily_build_dependencies_ && !dependency) { - result->dependencies_names_[i] = - tables_->AllocateString(proto.dependency(i)); + if (result->dependencies_once_ == nullptr) { + result->dependencies_once_ = + tables_->Create(); + result->dependencies_once_->dependencies_names = + tables_->AllocateArray(proto.dependency_size()); + if (proto.dependency_size() > 0) { + std::fill_n(result->dependencies_once_->dependencies_names, + proto.dependency_size(), nullptr); + } + } + + result->dependencies_once_->dependencies_names[i] = + tables_->Strdup(proto.dependency(i)); } } // Check public dependencies. int public_dependency_count = 0; - result->public_dependencies_ = tables_->AllocateArray( - proto.public_dependency_size()); + result->public_dependencies_ = + tables_->AllocateArray(proto.public_dependency_size()); for (int i = 0; i < proto.public_dependency_size(); i++) { // Only put valid public dependency indexes. int index = proto.public_dependency(i); @@ -4384,8 +5069,7 @@ FileDescriptor* DescriptorBuilder::BuildFileImpl( unused_dependency_.erase(result->dependency(index)); } } else { - AddError(proto.name(), proto, - DescriptorPool::ErrorCollector::OTHER, + AddError(proto.name(), proto, DescriptorPool::ErrorCollector::OTHER, "Invalid public dependency index."); } } @@ -4404,30 +5088,28 @@ FileDescriptor* DescriptorBuilder::BuildFileImpl( // Check weak dependencies. int weak_dependency_count = 0; - result->weak_dependencies_ = tables_->AllocateArray( - proto.weak_dependency_size()); + result->weak_dependencies_ = + tables_->AllocateArray(proto.weak_dependency_size()); for (int i = 0; i < proto.weak_dependency_size(); i++) { int index = proto.weak_dependency(i); if (index >= 0 && index < proto.dependency_size()) { result->weak_dependencies_[weak_dependency_count++] = index; } else { - AddError(proto.name(), proto, - DescriptorPool::ErrorCollector::OTHER, + AddError(proto.name(), proto, DescriptorPool::ErrorCollector::OTHER, "Invalid weak dependency index."); } } result->weak_dependency_count_ = weak_dependency_count; // Convert children. - BUILD_ARRAY(proto, result, message_type, BuildMessage , NULL); - BUILD_ARRAY(proto, result, enum_type , BuildEnum , NULL); - BUILD_ARRAY(proto, result, service , BuildService , NULL); - BUILD_ARRAY(proto, result, extension , BuildExtension, NULL); + BUILD_ARRAY(proto, result, message_type, BuildMessage, nullptr); + BUILD_ARRAY(proto, result, enum_type, BuildEnum, nullptr); + BUILD_ARRAY(proto, result, service, BuildService, nullptr); + BUILD_ARRAY(proto, result, extension, BuildExtension, nullptr); // Copy options. - if (!proto.has_options()) { - result->options_ = NULL; // Will set to default_instance later. - } else { + result->options_ = nullptr; // Set to default_instance later if necessary. + if (proto.has_options()) { AllocateOptions(proto.options(), result); } @@ -4447,6 +5129,9 @@ FileDescriptor* DescriptorBuilder::BuildFileImpl( option_interpreter.InterpretOptions(&(*iter)); } options_to_interpret_.clear(); + if (info != nullptr) { + option_interpreter.UpdateSourceCodeInfo(info); + } } // Validate options. See comments at InternalSetLazilyBuildDependencies about @@ -4465,64 +5150,91 @@ FileDescriptor* DescriptorBuilder::BuildFileImpl( // Again, see comments at InternalSetLazilyBuildDependencies about error - // checking. - if (!unused_dependency_.empty() && !pool_->lazily_build_dependencies_) { + // checking. Also, don't log unused dependencies if there were previous + // errors, since the results might be inaccurate. + if (!had_errors_ && !unused_dependency_.empty() && + !pool_->lazily_build_dependencies_) { LogUnusedDependency(proto, result); } if (had_errors_) { - return NULL; + return nullptr; } else { return result; } } + +const std::string* DescriptorBuilder::AllocateNameStrings( + const std::string& scope, const std::string& proto_name) { + if (scope.empty()) { + return tables_->AllocateStringArray(proto_name, proto_name); + } else { + return tables_->AllocateStringArray(proto_name, + StrCat(scope, ".", proto_name)); + } +} + void DescriptorBuilder::BuildMessage(const DescriptorProto& proto, const Descriptor* parent, Descriptor* result) { - const string& scope = (parent == NULL) ? - file_->package() : parent->full_name(); - string* full_name = tables_->AllocateString(scope); - if (!full_name->empty()) full_name->append(1, '.'); - full_name->append(proto.name()); + const std::string& scope = + (parent == nullptr) ? file_->package() : parent->full_name(); + result->all_names_ = AllocateNameStrings(scope, proto.name()); + ValidateSymbolName(proto.name(), result->full_name(), proto); - ValidateSymbolName(proto.name(), *full_name, proto); - - result->name_ = tables_->AllocateString(proto.name()); - result->full_name_ = full_name; - result->file_ = file_; + result->file_ = file_; result->containing_type_ = parent; - result->is_placeholder_ = false; + result->is_placeholder_ = false; result->is_unqualified_placeholder_ = false; + result->well_known_type_ = Descriptor::WELLKNOWNTYPE_UNSPECIFIED; + + auto it = pool_->tables_->well_known_types_.find(result->full_name()); + if (it != pool_->tables_->well_known_types_.end()) { + result->well_known_type_ = it->second; + } + + // Calculate the continuous sequence of fields. + // These can be fast-path'd during lookup and don't need to be added to the + // tables. + // We use uint16_t to save space for sequential_field_limit_, so stop before + // overflowing it. Worst case, we are not taking full advantage on huge + // messages, but it is unlikely. + result->sequential_field_limit_ = 0; + for (int i = 0; i < std::numeric_limits::max() && + i < proto.field_size() && proto.field(i).number() == i + 1; + ++i) { + result->sequential_field_limit_ = i + 1; + } // Build oneofs first so that fields and extension ranges can refer to them. - BUILD_ARRAY(proto, result, oneof_decl , BuildOneof , result); - BUILD_ARRAY(proto, result, field , BuildField , result); - BUILD_ARRAY(proto, result, nested_type , BuildMessage , result); - BUILD_ARRAY(proto, result, enum_type , BuildEnum , result); + BUILD_ARRAY(proto, result, oneof_decl, BuildOneof, result); + BUILD_ARRAY(proto, result, field, BuildField, result); + BUILD_ARRAY(proto, result, nested_type, BuildMessage, result); + BUILD_ARRAY(proto, result, enum_type, BuildEnum, result); BUILD_ARRAY(proto, result, extension_range, BuildExtensionRange, result); - BUILD_ARRAY(proto, result, extension , BuildExtension , result); - BUILD_ARRAY(proto, result, reserved_range , BuildReservedRange , result); + BUILD_ARRAY(proto, result, extension, BuildExtension, result); + BUILD_ARRAY(proto, result, reserved_range, BuildReservedRange, result); // Copy reserved names. int reserved_name_count = proto.reserved_name_size(); result->reserved_name_count_ = reserved_name_count; result->reserved_names_ = - tables_->AllocateArray(reserved_name_count); + tables_->AllocateArray(reserved_name_count); for (int i = 0; i < reserved_name_count; ++i) { result->reserved_names_[i] = tables_->AllocateString(proto.reserved_name(i)); } // Copy options. - if (!proto.has_options()) { - result->options_ = NULL; // Will set to default_instance later. - } else { - AllocateOptions(proto.options(), result); + result->options_ = nullptr; // Set to default_instance later if necessary. + if (proto.has_options()) { + AllocateOptions(proto.options(), result, + DescriptorProto::kOptionsFieldNumber, + "google.protobuf.MessageOptions"); } - AddSymbol(result->full_name(), parent, result->name(), - proto, Symbol(result)); + AddSymbol(result->full_name(), parent, result->name(), proto, Symbol(result)); for (int i = 0; i < proto.reserved_range_size(); i++) { const DescriptorProto_ReservedRange& range1 = proto.reserved_range(i); @@ -4532,37 +5244,37 @@ void DescriptorBuilder::BuildMessage(const DescriptorProto& proto, AddError(result->full_name(), proto.reserved_range(i), DescriptorPool::ErrorCollector::NUMBER, strings::Substitute("Reserved range $0 to $1 overlaps with " - "already-defined range $2 to $3.", - range2.start(), range2.end() - 1, - range1.start(), range1.end() - 1)); + "already-defined range $2 to $3.", + range2.start(), range2.end() - 1, + range1.start(), range1.end() - 1)); } } } - hash_set reserved_name_set; + HASH_SET reserved_name_set; for (int i = 0; i < proto.reserved_name_size(); i++) { - const string& name = proto.reserved_name(i); + const std::string& name = proto.reserved_name(i); if (reserved_name_set.find(name) == reserved_name_set.end()) { reserved_name_set.insert(name); } else { AddError(name, proto, DescriptorPool::ErrorCollector::NAME, - strings::Substitute( - "Field name \"$0\" is reserved multiple times.", - name)); + strings::Substitute("Field name \"$0\" is reserved multiple times.", + name)); } } + for (int i = 0; i < result->field_count(); i++) { const FieldDescriptor* field = result->field(i); for (int j = 0; j < result->extension_range_count(); j++) { const Descriptor::ExtensionRange* range = result->extension_range(j); if (range->start <= field->number() && field->number() < range->end) { - AddError(field->full_name(), proto.extension_range(j), - DescriptorPool::ErrorCollector::NUMBER, - strings::Substitute( - "Extension range $0 to $1 includes field \"$2\" ($3).", - range->start, range->end - 1, - field->name(), field->number())); + AddError( + field->full_name(), proto.extension_range(j), + DescriptorPool::ErrorCollector::NUMBER, + strings::Substitute( + "Extension range $0 to $1 includes field \"$2\" ($3).", + range->start, range->end - 1, field->name(), field->number())); } } for (int j = 0; j < result->reserved_range_count(); j++) { @@ -4570,21 +5282,21 @@ void DescriptorBuilder::BuildMessage(const DescriptorProto& proto, if (range->start <= field->number() && field->number() < range->end) { AddError(field->full_name(), proto.reserved_range(j), DescriptorPool::ErrorCollector::NUMBER, - strings::Substitute( - "Field \"$0\" uses reserved number $1.", - field->name(), field->number())); + strings::Substitute("Field \"$0\" uses reserved number $1.", + field->name(), field->number())); } } if (reserved_name_set.find(field->name()) != reserved_name_set.end()) { - AddError(field->full_name(), proto.field(i), - DescriptorPool::ErrorCollector::NAME, - strings::Substitute( - "Field name \"$0\" is reserved.", field->name())); + AddError( + field->full_name(), proto.field(i), + DescriptorPool::ErrorCollector::NAME, + strings::Substitute("Field name \"$0\" is reserved.", field->name())); } + } // Check that extension ranges don't overlap and don't include - // reserved field numbers. + // reserved field numbers or names. for (int i = 0; i < result->extension_range_count(); i++) { const Descriptor::ExtensionRange* range1 = result->extension_range(i); for (int j = 0; j < result->reserved_range_count(); j++) { @@ -4593,9 +5305,9 @@ void DescriptorBuilder::BuildMessage(const DescriptorProto& proto, AddError(result->full_name(), proto.extension_range(i), DescriptorPool::ErrorCollector::NUMBER, strings::Substitute("Extension range $0 to $1 overlaps with " - "reserved range $2 to $3.", - range1->start, range1->end - 1, - range2->start, range2->end - 1)); + "reserved range $2 to $3.", + range1->start, range1->end - 1, range2->start, + range2->end - 1)); } } for (int j = i + 1; j < result->extension_range_count(); j++) { @@ -4604,88 +5316,74 @@ void DescriptorBuilder::BuildMessage(const DescriptorProto& proto, AddError(result->full_name(), proto.extension_range(i), DescriptorPool::ErrorCollector::NUMBER, strings::Substitute("Extension range $0 to $1 overlaps with " - "already-defined range $2 to $3.", - range2->start, range2->end - 1, - range1->start, range1->end - 1)); + "already-defined range $2 to $3.", + range2->start, range2->end - 1, range1->start, + range1->end - 1)); } } } } - void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, - const Descriptor* parent, + Descriptor* parent, FieldDescriptor* result, bool is_extension) { - const string& scope = (parent == NULL) ? - file_->package() : parent->full_name(); - string* full_name = tables_->AllocateString(scope); - if (!full_name->empty()) full_name->append(1, '.'); - full_name->append(proto.name()); + const std::string& scope = + (parent == nullptr) ? file_->package() : parent->full_name(); - ValidateSymbolName(proto.name(), *full_name, proto); + // We allocate all names in a single array, and dedup them. + // We remember the indices for the potentially deduped values. + auto all_names = tables_->AllocateFieldNames( + proto.name(), scope, + proto.has_json_name() ? &proto.json_name() : nullptr); + result->all_names_ = all_names.array; + result->lowercase_name_index_ = all_names.lowercase_index; + result->camelcase_name_index_ = all_names.camelcase_index; + result->json_name_index_ = all_names.json_index; - result->name_ = tables_->AllocateString(proto.name()); - result->full_name_ = full_name; - result->file_ = file_; - result->number_ = proto.number(); + ValidateSymbolName(proto.name(), result->full_name(), proto); + + result->file_ = file_; + result->number_ = proto.number(); result->is_extension_ = is_extension; + result->is_oneof_ = false; + result->proto3_optional_ = proto.proto3_optional(); - // If .proto files follow the style guide then the name should already be - // lower-cased. If that's the case we can just reuse the string we already - // allocated rather than allocate a new one. - string lowercase_name(proto.name()); - LowerString(&lowercase_name); - if (lowercase_name == proto.name()) { - result->lowercase_name_ = result->name_; - } else { - result->lowercase_name_ = tables_->AllocateString(lowercase_name); + if (proto.proto3_optional() && + file_->syntax() != FileDescriptor::SYNTAX_PROTO3) { + AddError(result->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, + "The [proto3_optional=true] option may only be set on proto3" + "fields, not " + + result->full_name()); } - // Don't bother with the above optimization for camel-case names since - // .proto files that follow the guide shouldn't be using names in this - // format, so the optimization wouldn't help much. - result->camelcase_name_ = - tables_->AllocateString(ToCamelCase(proto.name(), - /* lower_first = */ true)); - - if (proto.has_json_name()) { - result->has_json_name_ = true; - result->json_name_ = tables_->AllocateString(proto.json_name()); - } else { - result->has_json_name_ = false; - result->json_name_ = tables_->AllocateString(ToJsonName(proto.name())); - } + result->has_json_name_ = proto.has_json_name(); // Some compilers do not allow static_cast directly between two enum types, // so we must cast to int first. - result->type_ = static_cast( - implicit_cast(proto.type())); + result->type_ = static_cast( + implicit_cast(proto.type())); result->label_ = static_cast( - implicit_cast(proto.label())); + implicit_cast(proto.label())); - // An extension cannot have a required field (b/13365836). - if (result->is_extension_ && - result->label_ == FieldDescriptor::LABEL_REQUIRED) { - AddError(result->full_name(), proto, - // Error location `TYPE`: we would really like to indicate - // `LABEL`, but the `ErrorLocation` enum has no entry for this, and - // we don't necessarily know about all implementations of the - // `ErrorCollector` interface to extend them to handle the new - // error location type properly. - DescriptorPool::ErrorCollector::TYPE, - "Message extensions cannot have required fields."); + if (result->label_ == FieldDescriptor::LABEL_REQUIRED) { + // An extension cannot have a required field (b/13365836). + if (result->is_extension_) { + AddError(result->full_name(), proto, + // Error location `TYPE`: we would really like to indicate + // `LABEL`, but the `ErrorLocation` enum has no entry for this, + // and we don't necessarily know about all implementations of the + // `ErrorCollector` interface to extend them to handle the new + // error location type properly. + DescriptorPool::ErrorCollector::TYPE, + "The extension " + result->full_name() + " cannot be required."); + } } // Some of these may be filled in when cross-linking. - result->containing_type_ = NULL; - result->extension_scope_ = NULL; - result->message_type_ = NULL; - result->enum_type_ = NULL; - result->type_name_ = NULL; - result->type_once_ = NULL; - result->default_value_enum_ = NULL; - result->default_value_enum_name_ = NULL; + result->containing_type_ = nullptr; + result->type_once_ = nullptr; + result->default_value_enum_ = nullptr; result->has_default_value_ = proto.has_default_value(); if (proto.has_default_value() && result->is_repeated()) { @@ -4696,23 +5394,23 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, if (proto.has_type()) { if (proto.has_default_value()) { - char* end_pos = NULL; + char* end_pos = nullptr; switch (result->cpp_type()) { case FieldDescriptor::CPPTYPE_INT32: - result->default_value_int32_ = - strtol(proto.default_value().c_str(), &end_pos, 0); + result->default_value_int32_t_ = + strtol(proto.default_value().c_str(), &end_pos, 0); break; case FieldDescriptor::CPPTYPE_INT64: - result->default_value_int64_ = - strto64(proto.default_value().c_str(), &end_pos, 0); + result->default_value_int64_t_ = + strto64(proto.default_value().c_str(), &end_pos, 0); break; case FieldDescriptor::CPPTYPE_UINT32: - result->default_value_uint32_ = - strtoul(proto.default_value().c_str(), &end_pos, 0); + result->default_value_uint32_t_ = + strtoul(proto.default_value().c_str(), &end_pos, 0); break; case FieldDescriptor::CPPTYPE_UINT64: - result->default_value_uint64_ = - strtou64(proto.default_value().c_str(), &end_pos, 0); + result->default_value_uint64_t_ = + strtou64(proto.default_value().c_str(), &end_pos, 0); break; case FieldDescriptor::CPPTYPE_FLOAT: if (proto.default_value() == "inf") { @@ -4724,7 +5422,7 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, } else if (proto.default_value() == "nan") { result->default_value_float_ = std::numeric_limits::quiet_NaN(); - } else { + } else { result->default_value_float_ = io::SafeDoubleToFloat( io::NoLocaleStrtod(proto.default_value().c_str(), &end_pos)); } @@ -4739,7 +5437,7 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, } else if (proto.default_value() == "nan") { result->default_value_double_ = std::numeric_limits::quiet_NaN(); - } else { + } else { result->default_value_double_ = io::NoLocaleStrtod(proto.default_value().c_str(), &end_pos); } @@ -4757,12 +5455,12 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, break; case FieldDescriptor::CPPTYPE_ENUM: // This will be filled in when cross-linking. - result->default_value_enum_ = NULL; + result->default_value_enum_ = nullptr; break; case FieldDescriptor::CPPTYPE_STRING: if (result->type() == FieldDescriptor::TYPE_BYTES) { result->default_value_string_ = tables_->AllocateString( - UnescapeCEscapeString(proto.default_value())); + UnescapeCEscapeString(proto.default_value())); } else { result->default_value_string_ = tables_->AllocateString(proto.default_value()); @@ -4773,34 +5471,35 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, DescriptorPool::ErrorCollector::DEFAULT_VALUE, "Messages can't have default values."); result->has_default_value_ = false; + result->default_generated_instance_ = nullptr; break; } - if (end_pos != NULL) { - // end_pos is only set non-NULL by the parsers for numeric types, above. - // This checks that the default was non-empty and had no extra junk - // after the end of the number. + if (end_pos != nullptr) { + // end_pos is only set non-null by the parsers for numeric types, + // above. This checks that the default was non-empty and had no extra + // junk after the end of the number. if (proto.default_value().empty() || *end_pos != '\0') { AddError(result->full_name(), proto, DescriptorPool::ErrorCollector::DEFAULT_VALUE, "Couldn't parse default value \"" + proto.default_value() + - "\"."); + "\"."); } } } else { // No explicit default value switch (result->cpp_type()) { case FieldDescriptor::CPPTYPE_INT32: - result->default_value_int32_ = 0; + result->default_value_int32_t_ = 0; break; case FieldDescriptor::CPPTYPE_INT64: - result->default_value_int64_ = 0; + result->default_value_int64_t_ = 0; break; case FieldDescriptor::CPPTYPE_UINT32: - result->default_value_uint32_ = 0; + result->default_value_uint32_t_ = 0; break; case FieldDescriptor::CPPTYPE_UINT64: - result->default_value_uint64_ = 0; + result->default_value_uint64_t_ = 0; break; case FieldDescriptor::CPPTYPE_FLOAT: result->default_value_float_ = 0.0f; @@ -4813,12 +5512,13 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, break; case FieldDescriptor::CPPTYPE_ENUM: // This will be filled in when cross-linking. - result->default_value_enum_ = NULL; + result->default_value_enum_ = nullptr; break; case FieldDescriptor::CPPTYPE_STRING: result->default_value_string_ = &internal::GetEmptyString(); break; case FieldDescriptor::CPPTYPE_MESSAGE: + result->default_generated_instance_ = nullptr; break; } } @@ -4838,15 +5538,15 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, // on extension numbers. AddError(result->full_name(), proto, DescriptorPool::ErrorCollector::NUMBER, strings::Substitute("Field numbers cannot be greater than $0.", - FieldDescriptor::kMaxNumber)); + FieldDescriptor::kMaxNumber)); } else if (result->number() >= FieldDescriptor::kFirstReservedNumber && result->number() <= FieldDescriptor::kLastReservedNumber) { AddError(result->full_name(), proto, DescriptorPool::ErrorCollector::NUMBER, strings::Substitute( - "Field numbers $0 through $1 are reserved for the protocol " - "buffer library implementation.", - FieldDescriptor::kFirstReservedNumber, - FieldDescriptor::kLastReservedNumber)); + "Field numbers $0 through $1 are reserved for the protocol " + "buffer library implementation.", + FieldDescriptor::kFirstReservedNumber, + FieldDescriptor::kLastReservedNumber)); } if (is_extension) { @@ -4856,17 +5556,13 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, "FieldDescriptorProto.extendee not set for extension field."); } - result->extension_scope_ = parent; + result->scope_.extension_scope = parent; if (proto.has_oneof_index()) { - AddError(result->full_name(), proto, - DescriptorPool::ErrorCollector::OTHER, + AddError(result->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, "FieldDescriptorProto.oneof_index should not be set for " "extensions."); } - - // Fill in later (maybe). - result->containing_oneof_ = NULL; } else { if (proto.has_extendee()) { AddError(result->full_name(), proto, @@ -4880,41 +5576,37 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, if (proto.oneof_index() < 0 || proto.oneof_index() >= parent->oneof_decl_count()) { AddError(result->full_name(), proto, - DescriptorPool::ErrorCollector::OTHER, + DescriptorPool::ErrorCollector::TYPE, strings::Substitute("FieldDescriptorProto.oneof_index $0 is " - "out of range for type \"$1\".", - proto.oneof_index(), - parent->name())); - result->containing_oneof_ = NULL; + "out of range for type \"$1\".", + proto.oneof_index(), parent->name())); } else { - result->containing_oneof_ = parent->oneof_decl(proto.oneof_index()); + result->is_oneof_ = true; + result->scope_.containing_oneof = + parent->oneof_decl(proto.oneof_index()); } - } else { - result->containing_oneof_ = NULL; } } // Copy options. - if (!proto.has_options()) { - result->options_ = NULL; // Will set to default_instance later. - } else { - AllocateOptions(proto.options(), result); + result->options_ = nullptr; // Set to default_instance later if necessary. + if (proto.has_options()) { + AllocateOptions(proto.options(), result, + FieldDescriptorProto::kOptionsFieldNumber, + "google.protobuf.FieldOptions"); } - AddSymbol(result->full_name(), parent, result->name(), - proto, Symbol(result)); + AddSymbol(result->full_name(), parent, result->name(), proto, Symbol(result)); } void DescriptorBuilder::BuildExtensionRange( - const DescriptorProto::ExtensionRange& proto, - const Descriptor* parent, + const DescriptorProto::ExtensionRange& proto, const Descriptor* parent, Descriptor::ExtensionRange* result) { result->start = proto.start(); result->end = proto.end(); if (result->start <= 0) { - AddError(parent->full_name(), proto, - DescriptorPool::ErrorCollector::NUMBER, + AddError(parent->full_name(), proto, DescriptorPool::ErrorCollector::NUMBER, "Extension numbers must be positive integers."); } @@ -4924,28 +5616,34 @@ void DescriptorBuilder::BuildExtensionRange( // numbers are actually used as int32s in the message_set_wire_format. if (result->start >= result->end) { - AddError(parent->full_name(), proto, - DescriptorPool::ErrorCollector::NUMBER, + AddError(parent->full_name(), proto, DescriptorPool::ErrorCollector::NUMBER, "Extension range end number must be greater than start number."); } - if (!proto.has_options()) { - result->options_ = NULL; // Will set to default_instance later. - } else { + result->options_ = nullptr; // Set to default_instance later if necessary. + if (proto.has_options()) { + std::vector options_path; + parent->GetLocationPath(&options_path); + options_path.push_back(DescriptorProto::kExtensionRangeFieldNumber); + // find index of this extension range in order to compute path + int index; + for (index = 0; parent->extension_ranges_ + index != result; index++) { + } + options_path.push_back(index); + options_path.push_back(DescriptorProto_ExtensionRange::kOptionsFieldNumber); AllocateOptionsImpl(parent->full_name(), parent->full_name(), - proto.options(), result); + proto.options(), result, options_path, + "google.protobuf.ExtensionRangeOptions"); } } void DescriptorBuilder::BuildReservedRange( - const DescriptorProto::ReservedRange& proto, - const Descriptor* parent, + const DescriptorProto::ReservedRange& proto, const Descriptor* parent, Descriptor::ReservedRange* result) { result->start = proto.start(); result->end = proto.end(); if (result->start <= 0) { - AddError(parent->full_name(), proto, - DescriptorPool::ErrorCollector::NUMBER, + AddError(parent->full_name(), proto, DescriptorPool::ErrorCollector::NUMBER, "Reserved numbers must be positive integers."); } } @@ -4957,8 +5655,7 @@ void DescriptorBuilder::BuildReservedRange( result->end = proto.end(); if (result->start > result->end) { - AddError(parent->full_name(), proto, - DescriptorPool::ErrorCollector::NUMBER, + AddError(parent->full_name(), proto, DescriptorPool::ErrorCollector::NUMBER, "Reserved range end number must be greater than start number."); } } @@ -4966,30 +5663,24 @@ void DescriptorBuilder::BuildReservedRange( void DescriptorBuilder::BuildOneof(const OneofDescriptorProto& proto, Descriptor* parent, OneofDescriptor* result) { - string* full_name = tables_->AllocateString(parent->full_name()); - full_name->append(1, '.'); - full_name->append(proto.name()); - - ValidateSymbolName(proto.name(), *full_name, proto); - - result->name_ = tables_->AllocateString(proto.name()); - result->full_name_ = full_name; + result->all_names_ = AllocateNameStrings(parent->full_name(), proto.name()); + ValidateSymbolName(proto.name(), result->full_name(), proto); result->containing_type_ = parent; // We need to fill these in later. result->field_count_ = 0; - result->fields_ = NULL; + result->fields_ = nullptr; + result->options_ = nullptr; // Copy options. - if (!proto.has_options()) { - result->options_ = NULL; // Will set to default_instance later. - } else { - AllocateOptions(proto.options(), result); + if (proto.has_options()) { + AllocateOptions(proto.options(), result, + OneofDescriptorProto::kOptionsFieldNumber, + "google.protobuf.OneofOptions"); } - AddSymbol(result->full_name(), parent, result->name(), - proto, Symbol(result)); + AddSymbol(result->full_name(), parent, result->name(), proto, Symbol(result)); } void DescriptorBuilder::CheckEnumValueUniqueness( @@ -5021,13 +5712,12 @@ void DescriptorBuilder::CheckEnumValueUniqueness( // NAME_TYPE_LAST_NAME = 2, // } PrefixRemover remover(result->name()); - std::map values; + std::map values; for (int i = 0; i < result->value_count(); i++) { - const google::protobuf::EnumValueDescriptor* value = result->value(i); - string stripped = + const EnumValueDescriptor* value = result->value(i); + std::string stripped = EnumValueToPascalCase(remover.MaybeRemove(value->name())); - std::pair::iterator, - bool> + std::pair::iterator, bool> insert_result = values.insert(std::make_pair(stripped, value)); bool inserted = insert_result.second; @@ -5039,11 +5729,13 @@ void DescriptorBuilder::CheckEnumValueUniqueness( // stripping should de-dup the labels in this case). if (!inserted && insert_result.first->second->name() != value->name() && insert_result.first->second->number() != value->number()) { - string error_message = - "When enum name is stripped and label is PascalCased (" + stripped + - "), this value label conflicts with " + values[stripped]->name() + - ". This will make the proto fail to compile for some languages, such " - "as C#."; + std::string error_message = + "Enum name " + value->name() + " has the same name as " + + values[stripped]->name() + + " if you ignore case and strip out the enum name prefix (if any). " + "This is error-prone and can lead to undefined behavior. " + "Please avoid doing this. If you are using allow_alias, please " + "assign the same numeric value to both enums."; // There are proto2 enums out there with conflicting names, so to preserve // compatibility we issue only a warning for proto2. if (result->file()->syntax() == FileDescriptor::SYNTAX_PROTO2) { @@ -5060,29 +5752,38 @@ void DescriptorBuilder::CheckEnumValueUniqueness( void DescriptorBuilder::BuildEnum(const EnumDescriptorProto& proto, const Descriptor* parent, EnumDescriptor* result) { - const string& scope = (parent == NULL) ? - file_->package() : parent->full_name(); - string* full_name = tables_->AllocateString(scope); - if (!full_name->empty()) full_name->append(1, '.'); - full_name->append(proto.name()); + const std::string& scope = + (parent == nullptr) ? file_->package() : parent->full_name(); - ValidateSymbolName(proto.name(), *full_name, proto); - - result->name_ = tables_->AllocateString(proto.name()); - result->full_name_ = full_name; - result->file_ = file_; + result->all_names_ = AllocateNameStrings(scope, proto.name()); + ValidateSymbolName(proto.name(), result->full_name(), proto); + result->file_ = file_; result->containing_type_ = parent; - result->is_placeholder_ = false; + result->is_placeholder_ = false; result->is_unqualified_placeholder_ = false; if (proto.value_size() == 0) { // We cannot allow enums with no values because this would mean there // would be no valid default value for fields of this type. - AddError(result->full_name(), proto, - DescriptorPool::ErrorCollector::NAME, + AddError(result->full_name(), proto, DescriptorPool::ErrorCollector::NAME, "Enums must contain at least one value."); } + // Calculate the continuous sequence of the labels. + // These can be fast-path'd during lookup and don't need to be added to the + // tables. + // We use uint16_t to save space for sequential_value_limit_, so stop before + // overflowing it. Worst case, we are not taking full advantage on huge + // enums, but it is unlikely. + for (int i = 0; + i < std::numeric_limits::max() && i < proto.value_size() && + // We do the math in int64_t to avoid overflows. + proto.value(i).number() == + static_cast(i) + proto.value(0).number(); + ++i) { + result->sequential_value_limit_ = i; + } + BUILD_ARRAY(proto, result, value, BuildEnumValue, result); BUILD_ARRAY(proto, result, reserved_range, BuildReservedRange, result); @@ -5090,7 +5791,7 @@ void DescriptorBuilder::BuildEnum(const EnumDescriptorProto& proto, int reserved_name_count = proto.reserved_name_size(); result->reserved_name_count_ = reserved_name_count; result->reserved_names_ = - tables_->AllocateArray(reserved_name_count); + tables_->AllocateArray(reserved_name_count); for (int i = 0; i < reserved_name_count; ++i) { result->reserved_names_[i] = tables_->AllocateString(proto.reserved_name(i)); @@ -5099,14 +5800,14 @@ void DescriptorBuilder::BuildEnum(const EnumDescriptorProto& proto, CheckEnumValueUniqueness(proto, result); // Copy options. - if (!proto.has_options()) { - result->options_ = NULL; // Will set to default_instance later. - } else { - AllocateOptions(proto.options(), result); + result->options_ = nullptr; // Set to default_instance later if necessary. + if (proto.has_options()) { + AllocateOptions(proto.options(), result, + EnumDescriptorProto::kOptionsFieldNumber, + "google.protobuf.EnumOptions"); } - AddSymbol(result->full_name(), parent, result->name(), - proto, Symbol(result)); + AddSymbol(result->full_name(), parent, result->name(), proto, Symbol(result)); for (int i = 0; i < proto.reserved_range_size(); i++) { const EnumDescriptorProto_EnumReservedRange& range1 = @@ -5114,27 +5815,26 @@ void DescriptorBuilder::BuildEnum(const EnumDescriptorProto& proto, for (int j = i + 1; j < proto.reserved_range_size(); j++) { const EnumDescriptorProto_EnumReservedRange& range2 = proto.reserved_range(j); - if (range1.end() > range2.start() && range2.end() > range1.start()) { + if (range1.end() >= range2.start() && range2.end() >= range1.start()) { AddError(result->full_name(), proto.reserved_range(i), DescriptorPool::ErrorCollector::NUMBER, strings::Substitute("Reserved range $0 to $1 overlaps with " - "already-defined range $2 to $3.", - range2.start(), range2.end() - 1, - range1.start(), range1.end() - 1)); + "already-defined range $2 to $3.", + range2.start(), range2.end(), range1.start(), + range1.end())); } } } - hash_set reserved_name_set; + HASH_SET reserved_name_set; for (int i = 0; i < proto.reserved_name_size(); i++) { - const string& name = proto.reserved_name(i); + const std::string& name = proto.reserved_name(i); if (reserved_name_set.find(name) == reserved_name_set.end()) { reserved_name_set.insert(name); } else { AddError(name, proto, DescriptorPool::ErrorCollector::NAME, - strings::Substitute( - "Enum value \"$0\" is reserved multiple times.", - name)); + strings::Substitute("Enum value \"$0\" is reserved multiple times.", + name)); } } @@ -5145,16 +5845,15 @@ void DescriptorBuilder::BuildEnum(const EnumDescriptorProto& proto, if (range->start <= value->number() && value->number() <= range->end) { AddError(value->full_name(), proto.reserved_range(j), DescriptorPool::ErrorCollector::NUMBER, - strings::Substitute( - "Enum value \"$0\" uses reserved number $1.", - value->name(), value->number())); + strings::Substitute("Enum value \"$0\" uses reserved number $1.", + value->name(), value->number())); } } if (reserved_name_set.find(value->name()) != reserved_name_set.end()) { - AddError(value->full_name(), proto.value(i), - DescriptorPool::ErrorCollector::NAME, - strings::Substitute( - "Enum value \"$0\" is reserved.", value->name())); + AddError( + value->full_name(), proto.value(i), + DescriptorPool::ErrorCollector::NAME, + strings::Substitute("Enum value \"$0\" is reserved.", value->name())); } } } @@ -5162,46 +5861,49 @@ void DescriptorBuilder::BuildEnum(const EnumDescriptorProto& proto, void DescriptorBuilder::BuildEnumValue(const EnumValueDescriptorProto& proto, const EnumDescriptor* parent, EnumValueDescriptor* result) { - result->name_ = tables_->AllocateString(proto.name()); - result->number_ = proto.number(); - result->type_ = parent; - // Note: full_name for enum values is a sibling to the parent's name, not a // child of it. - string* full_name = tables_->AllocateString(*parent->full_name_); - full_name->resize(full_name->size() - parent->name_->size()); - full_name->append(*result->name_); - result->full_name_ = full_name; + std::string full_name; + size_t scope_len = parent->full_name().size() - parent->name().size(); + full_name.reserve(scope_len + proto.name().size()); + full_name.append(parent->full_name().data(), scope_len); + full_name.append(proto.name()); - ValidateSymbolName(proto.name(), *full_name, proto); + result->all_names_ = + tables_->AllocateStringArray(proto.name(), std::move(full_name)); + result->number_ = proto.number(); + result->type_ = parent; + + ValidateSymbolName(proto.name(), result->full_name(), proto); // Copy options. - if (!proto.has_options()) { - result->options_ = NULL; // Will set to default_instance later. - } else { - AllocateOptions(proto.options(), result); + result->options_ = nullptr; // Set to default_instance later if necessary. + if (proto.has_options()) { + AllocateOptions(proto.options(), result, + EnumValueDescriptorProto::kOptionsFieldNumber, + "google.protobuf.EnumValueOptions"); } // Again, enum values are weird because we makes them appear as siblings // of the enum type instead of children of it. So, we use // parent->containing_type() as the value's parent. bool added_to_outer_scope = - AddSymbol(result->full_name(), parent->containing_type(), result->name(), - proto, Symbol(result)); + AddSymbol(result->full_name(), parent->containing_type(), result->name(), + proto, Symbol::EnumValue(result, 0)); // However, we also want to be able to search for values within a single // enum type, so we add it as a child of the enum type itself, too. // Note: This could fail, but if it does, the error has already been // reported by the above AddSymbol() call, so we ignore the return code. - bool added_to_inner_scope = - file_tables_->AddAliasUnderParent(parent, result->name(), Symbol(result)); + bool added_to_inner_scope = file_tables_->AddAliasUnderParent( + parent, result->name(), Symbol::EnumValue(result, 1)); if (added_to_inner_scope && !added_to_outer_scope) { // This value did not conflict with any values defined in the same enum, // but it did conflict with some other symbol defined in the enum type's // scope. Let's print an additional error to explain this. - string outer_scope; - if (parent->containing_type() == NULL) { + std::string outer_scope; + if (parent->containing_type() == nullptr) { outer_scope = file_->package(); } else { outer_scope = parent->containing_type()->full_name(); @@ -5213,12 +5915,12 @@ void DescriptorBuilder::BuildEnumValue(const EnumValueDescriptorProto& proto, outer_scope = "\"" + outer_scope + "\""; } - AddError(result->full_name(), proto, - DescriptorPool::ErrorCollector::NAME, + AddError(result->full_name(), proto, DescriptorPool::ErrorCollector::NAME, "Note that enum values use C++ scoping rules, meaning that " "enum values are siblings of their type, not children of it. " - "Therefore, \"" + result->name() + "\" must be unique within " - + outer_scope + ", not just within \"" + parent->name() + "\"."); + "Therefore, \"" + + result->name() + "\" must be unique within " + outer_scope + + ", not just within \"" + parent->name() + "\"."); } // An enum is allowed to define two numbers that refer to the same value. @@ -5230,67 +5932,57 @@ void DescriptorBuilder::BuildEnumValue(const EnumValueDescriptorProto& proto, void DescriptorBuilder::BuildService(const ServiceDescriptorProto& proto, const void* /* dummy */, ServiceDescriptor* result) { - string* full_name = tables_->AllocateString(file_->package()); - if (!full_name->empty()) full_name->append(1, '.'); - full_name->append(proto.name()); - - ValidateSymbolName(proto.name(), *full_name, proto); - - result->name_ = tables_->AllocateString(proto.name()); - result->full_name_ = full_name; - result->file_ = file_; + result->all_names_ = AllocateNameStrings(file_->package(), proto.name()); + result->file_ = file_; + ValidateSymbolName(proto.name(), result->full_name(), proto); BUILD_ARRAY(proto, result, method, BuildMethod, result); // Copy options. - if (!proto.has_options()) { - result->options_ = NULL; // Will set to default_instance later. - } else { - AllocateOptions(proto.options(), result); + result->options_ = nullptr; // Set to default_instance later if necessary. + if (proto.has_options()) { + AllocateOptions(proto.options(), result, + ServiceDescriptorProto::kOptionsFieldNumber, + "google.protobuf.ServiceOptions"); } - AddSymbol(result->full_name(), NULL, result->name(), - proto, Symbol(result)); + AddSymbol(result->full_name(), nullptr, result->name(), proto, + Symbol(result)); } void DescriptorBuilder::BuildMethod(const MethodDescriptorProto& proto, const ServiceDescriptor* parent, MethodDescriptor* result) { - result->name_ = tables_->AllocateString(proto.name()); result->service_ = parent; + result->all_names_ = AllocateNameStrings(parent->full_name(), proto.name()); - string* full_name = tables_->AllocateString(parent->full_name()); - full_name->append(1, '.'); - full_name->append(*result->name_); - result->full_name_ = full_name; - - ValidateSymbolName(proto.name(), *full_name, proto); + ValidateSymbolName(proto.name(), result->full_name(), proto); // These will be filled in when cross-linking. result->input_type_.Init(); result->output_type_.Init(); // Copy options. - if (!proto.has_options()) { - result->options_ = NULL; // Will set to default_instance later. - } else { - AllocateOptions(proto.options(), result); + result->options_ = nullptr; // Set to default_instance later if necessary. + if (proto.has_options()) { + AllocateOptions(proto.options(), result, + MethodDescriptorProto::kOptionsFieldNumber, + "google.protobuf.MethodOptions"); } result->client_streaming_ = proto.client_streaming(); result->server_streaming_ = proto.server_streaming(); - AddSymbol(result->full_name(), parent, result->name(), - proto, Symbol(result)); + AddSymbol(result->full_name(), parent, result->name(), proto, Symbol(result)); } #undef BUILD_ARRAY // ------------------------------------------------------------------- -void DescriptorBuilder::CrossLinkFile( - FileDescriptor* file, const FileDescriptorProto& proto) { - if (file->options_ == NULL) { +void DescriptorBuilder::CrossLinkFile(FileDescriptor* file, + const FileDescriptorProto& proto) { + if (file->options_ == nullptr) { file->options_ = &FileOptions::default_instance(); } @@ -5311,9 +6003,9 @@ void DescriptorBuilder::CrossLinkFile( } } -void DescriptorBuilder::CrossLinkMessage( - Descriptor* message, const DescriptorProto& proto) { - if (message->options_ == NULL) { +void DescriptorBuilder::CrossLinkMessage(Descriptor* message, + const DescriptorProto& proto) { + if (message->options_ == nullptr) { message->options_ = &MessageOptions::default_instance(); } @@ -5343,7 +6035,7 @@ void DescriptorBuilder::CrossLinkMessage( // First count the number of fields per oneof. for (int i = 0; i < message->field_count(); i++) { const OneofDescriptor* oneof_decl = message->field(i)->containing_oneof(); - if (oneof_decl != NULL) { + if (oneof_decl != nullptr) { // Make sure fields belonging to the same oneof are defined consecutively. // This enables optimizations in codegens and reflection libraries to // skip fields in the oneof group, as only one of the field can be set. @@ -5352,65 +6044,95 @@ void DescriptorBuilder::CrossLinkMessage( // safe. if (oneof_decl->field_count() > 0 && message->field(i - 1)->containing_oneof() != oneof_decl) { - AddError( - message->full_name() + "." + message->field(i - 1)->name(), - proto.field(i - 1), DescriptorPool::ErrorCollector::OTHER, - strings::Substitute( - "Fields in the same oneof must be defined consecutively. " - "\"$0\" cannot be defined before the completion of the " - "\"$1\" oneof definition.", - message->field(i - 1)->name(), oneof_decl->name())); + AddError(message->full_name() + "." + message->field(i - 1)->name(), + proto.field(i - 1), DescriptorPool::ErrorCollector::TYPE, + strings::Substitute( + "Fields in the same oneof must be defined consecutively. " + "\"$0\" cannot be defined before the completion of the " + "\"$1\" oneof definition.", + message->field(i - 1)->name(), oneof_decl->name())); } // Must go through oneof_decls_ array to get a non-const version of the // OneofDescriptor. - ++message->oneof_decls_[oneof_decl->index()].field_count_; + auto& out_oneof_decl = message->oneof_decls_[oneof_decl->index()]; + if (out_oneof_decl.field_count_ == 0) { + out_oneof_decl.fields_ = message->field(i); + } + + if (!had_errors_) { + // Verify that they are contiguous. + // This is assumed by OneofDescriptor::field(i). + // But only if there are no errors. + GOOGLE_CHECK_EQ(out_oneof_decl.fields_ + out_oneof_decl.field_count_, + message->field(i)); + } + ++out_oneof_decl.field_count_; } } - // Then allocate the arrays. + // Then verify the sizes. for (int i = 0; i < message->oneof_decl_count(); i++) { OneofDescriptor* oneof_decl = &message->oneof_decls_[i]; if (oneof_decl->field_count() == 0) { AddError(message->full_name() + "." + oneof_decl->name(), - proto.oneof_decl(i), - DescriptorPool::ErrorCollector::NAME, + proto.oneof_decl(i), DescriptorPool::ErrorCollector::NAME, "Oneof must have at least one field."); } - oneof_decl->fields_ = - tables_->AllocateArray(oneof_decl->field_count_); - oneof_decl->field_count_ = 0; - - if (oneof_decl->options_ == NULL) { + if (oneof_decl->options_ == nullptr) { oneof_decl->options_ = &OneofOptions::default_instance(); } } - // Then fill them in. for (int i = 0; i < message->field_count(); i++) { - const OneofDescriptor* oneof_decl = message->field(i)->containing_oneof(); - if (oneof_decl != NULL) { - OneofDescriptor* mutable_oneof_decl = - &message->oneof_decls_[oneof_decl->index()]; - message->fields_[i].index_in_oneof_ = mutable_oneof_decl->field_count_; - mutable_oneof_decl->fields_[mutable_oneof_decl->field_count_++] = - message->field(i); + const FieldDescriptor* field = message->field(i); + if (field->proto3_optional_) { + if (!field->containing_oneof() || + !field->containing_oneof()->is_synthetic()) { + AddError(message->full_name(), proto.field(i), + DescriptorPool::ErrorCollector::OTHER, + "Fields with proto3_optional set must be " + "a member of a one-field oneof"); + } } } + + // Synthetic oneofs must be last. + int first_synthetic = -1; + for (int i = 0; i < message->oneof_decl_count(); i++) { + const OneofDescriptor* oneof = message->oneof_decl(i); + if (oneof->is_synthetic()) { + if (first_synthetic == -1) { + first_synthetic = i; + } + } else { + if (first_synthetic != -1) { + AddError(message->full_name(), proto.oneof_decl(i), + DescriptorPool::ErrorCollector::OTHER, + "Synthetic oneofs must be after all other oneofs"); + } + } + } + + if (first_synthetic == -1) { + message->real_oneof_decl_count_ = message->oneof_decl_count_; + } else { + message->real_oneof_decl_count_ = first_synthetic; + } } void DescriptorBuilder::CrossLinkExtensionRange( Descriptor::ExtensionRange* range, - const DescriptorProto::ExtensionRange& proto) { - if (range->options_ == NULL) { + const DescriptorProto::ExtensionRange& /*proto*/) { + if (range->options_ == nullptr) { range->options_ = &ExtensionRangeOptions::default_instance(); } } -void DescriptorBuilder::CrossLinkField( - FieldDescriptor* field, const FieldDescriptorProto& proto) { - if (field->options_ == NULL) { +void DescriptorBuilder::CrossLinkField(FieldDescriptor* field, + const FieldDescriptorProto& proto) { + if (field->options_ == nullptr) { field->options_ = &FieldOptions::default_instance(); } @@ -5426,34 +6148,41 @@ void DescriptorBuilder::CrossLinkField( DescriptorPool::ErrorCollector::EXTENDEE, proto.extendee()); return; - } else if (extendee.type != Symbol::MESSAGE) { + } else if (extendee.type() != Symbol::MESSAGE) { AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::EXTENDEE, "\"" + proto.extendee() + "\" is not a message type."); return; } - field->containing_type_ = extendee.descriptor; + field->containing_type_ = extendee.descriptor(); - const Descriptor::ExtensionRange* extension_range = field->containing_type() - ->FindExtensionRangeContainingNumber(field->number()); + const Descriptor::ExtensionRange* extension_range = + field->containing_type()->FindExtensionRangeContainingNumber( + field->number()); - if (extension_range == NULL) { - AddError(field->full_name(), proto, - DescriptorPool::ErrorCollector::NUMBER, - strings::Substitute("\"$0\" does not declare $1 as an " - "extension number.", - field->containing_type()->full_name(), - field->number())); + if (extension_range == nullptr) { + // Set of valid extension numbers for MessageSet is different (< 2^32) + // from other extendees (< 2^29). If unknown deps are allowed, we may not + // have that information, and wrongly deem the extension as invalid. + auto skip_check = get_allow_unknown(pool_) && + proto.extendee() == "google.protobuf.bridge.MessageSet"; + if (!skip_check) { + AddError(field->full_name(), proto, + DescriptorPool::ErrorCollector::NUMBER, + strings::Substitute("\"$0\" does not declare $1 as an " + "extension number.", + field->containing_type()->full_name(), + field->number())); + } } } - if (field->containing_oneof() != NULL) { + if (field->containing_oneof() != nullptr) { if (field->label() != FieldDescriptor::LABEL_OPTIONAL) { // Note that this error will never happen when parsing .proto files. // It can only happen if you manually construct a FileDescriptorProto // that is incorrect. - AddError(field->full_name(), proto, - DescriptorPool::ErrorCollector::NAME, + AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::NAME, "Fields of oneofs must themselves have label LABEL_OPTIONAL."); } } @@ -5466,7 +6195,7 @@ void DescriptorBuilder::CrossLinkField( proto.has_default_value(); // In case of weak fields we force building the dependency. We need to know - // if the type exist or not. If it doesnt exist we substitute Empty which + // if the type exist or not. If it doesn't exist we substitute Empty which // should only be done if the type can't be found in the generated pool. // TODO(gerbens) Ideally we should query the database directly to check // if weak fields exist or not so that we don't need to force building @@ -5485,13 +6214,13 @@ void DescriptorBuilder::CrossLinkField( if (is_lazy) { // Save the symbol names for later for lookup, and allocate the once // object needed for the accessors. - string name = proto.type_name(); - field->type_once_ = tables_->AllocateOnceDynamic(); - field->type_name_ = tables_->AllocateString(name); - if (proto.has_default_value()) { - field->default_value_enum_name_ = - tables_->AllocateString(proto.default_value()); - } + std::string name = proto.type_name(); + field->type_once_ = tables_->Create(); + field->type_descriptor_.lazy_type_name = tables_->Strdup(name); + field->lazy_default_value_enum_name_ = + proto.has_default_value() ? tables_->Strdup(proto.default_value()) + : nullptr; + // AddFieldByNumber and AddExtension are done later in this function, // and can/must be done if the field type was not found. The related // error checking is not necessary when in lazily_build_dependencies_ @@ -5519,9 +6248,9 @@ void DescriptorBuilder::CrossLinkField( if (!proto.has_type()) { // Choose field type based on symbol. - if (type.type == Symbol::MESSAGE) { + if (type.type() == Symbol::MESSAGE) { field->type_ = FieldDescriptor::TYPE_MESSAGE; - } else if (type.type == Symbol::ENUM) { + } else if (type.type() == Symbol::ENUM) { field->type_ = FieldDescriptor::TYPE_ENUM; } else { AddError(field->full_name(), proto, @@ -5532,13 +6261,13 @@ void DescriptorBuilder::CrossLinkField( } if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { - if (type.type != Symbol::MESSAGE) { + field->type_descriptor_.message_type = type.descriptor(); + if (field->type_descriptor_.message_type == nullptr) { AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, "\"" + proto.type_name() + "\" is not a message type."); return; } - field->message_type_ = type.descriptor; if (field->has_default_value()) { AddError(field->full_name(), proto, @@ -5546,13 +6275,13 @@ void DescriptorBuilder::CrossLinkField( "Messages can't have default values."); } } else if (field->cpp_type() == FieldDescriptor::CPPTYPE_ENUM) { - if (type.type != Symbol::ENUM) { + field->type_descriptor_.enum_type = type.enum_descriptor(); + if (field->type_descriptor_.enum_type == nullptr) { AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, "\"" + proto.type_name() + "\" is not an enum type."); return; } - field->enum_type_ = type.enum_descriptor; if (field->enum_type()->is_placeholder_) { // We can't look up default values for placeholder types. We'll have @@ -5574,20 +6303,20 @@ void DescriptorBuilder::CrossLinkField( // We can't just use field->enum_type()->FindValueByName() here // because that locks the pool's mutex, which we have already locked // at this point. - Symbol default_value = - LookupSymbolNoPlaceholder(proto.default_value(), - field->enum_type()->full_name()); + const EnumValueDescriptor* default_value = + LookupSymbolNoPlaceholder(proto.default_value(), + field->enum_type()->full_name()) + .enum_value_descriptor(); - if (default_value.type == Symbol::ENUM_VALUE && - default_value.enum_value_descriptor->type() == - field->enum_type()) { - field->default_value_enum_ = default_value.enum_value_descriptor; + if (default_value != nullptr && + default_value->type() == field->enum_type()) { + field->default_value_enum_ = default_value; } else { AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::DEFAULT_VALUE, "Enum type \"" + field->enum_type()->full_name() + - "\" has no value named \"" + proto.default_value() + - "\"."); + "\" has no value named \"" + proto.default_value() + + "\"."); } } } else if (field->enum_type()->value_count() > 0) { @@ -5615,41 +6344,41 @@ void DescriptorBuilder::CrossLinkField( // risk to calling containing_type() or other accessors that will build // dependencies. if (!file_tables_->AddFieldByNumber(field)) { - const FieldDescriptor* conflicting_field = - file_tables_->FindFieldByNumber(field->containing_type(), - field->number()); - string containing_type_name = field->containing_type() == NULL - ? "unknown" - : field->containing_type()->full_name(); + const FieldDescriptor* conflicting_field = file_tables_->FindFieldByNumber( + field->containing_type(), field->number()); + std::string containing_type_name = + field->containing_type() == nullptr + ? "unknown" + : field->containing_type()->full_name(); if (field->is_extension()) { AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::NUMBER, strings::Substitute("Extension number $0 has already been used " - "in \"$1\" by extension \"$2\".", - field->number(), - containing_type_name, - conflicting_field->full_name())); + "in \"$1\" by extension \"$2\".", + field->number(), containing_type_name, + conflicting_field->full_name())); } else { AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::NUMBER, strings::Substitute("Field number $0 has already been used in " - "\"$1\" by field \"$2\".", - field->number(), - containing_type_name, - conflicting_field->name())); + "\"$1\" by field \"$2\".", + field->number(), containing_type_name, + conflicting_field->name())); } } else { if (field->is_extension()) { if (!tables_->AddExtension(field)) { const FieldDescriptor* conflicting_field = tables_->FindExtension(field->containing_type(), field->number()); - string error_msg = strings::Substitute( + std::string containing_type_name = + field->containing_type() == nullptr + ? "unknown" + : field->containing_type()->full_name(); + std::string error_msg = strings::Substitute( "Extension number $0 has already been used in \"$1\" by extension " "\"$2\" defined in $3.", - field->number(), - field->containing_type()->full_name(), - conflicting_field->full_name(), - conflicting_field->file()->name()); + field->number(), containing_type_name, + conflicting_field->full_name(), conflicting_field->file()->name()); // Conflicting extension numbers should be an error. However, before // turning this into an error we need to fix all existing broken // protos first. @@ -5661,9 +6390,9 @@ void DescriptorBuilder::CrossLinkField( } } -void DescriptorBuilder::CrossLinkEnum( - EnumDescriptor* enum_type, const EnumDescriptorProto& proto) { - if (enum_type->options_ == NULL) { +void DescriptorBuilder::CrossLinkEnum(EnumDescriptor* enum_type, + const EnumDescriptorProto& proto) { + if (enum_type->options_ == nullptr) { enum_type->options_ = &EnumOptions::default_instance(); } @@ -5675,14 +6404,14 @@ void DescriptorBuilder::CrossLinkEnum( void DescriptorBuilder::CrossLinkEnumValue( EnumValueDescriptor* enum_value, const EnumValueDescriptorProto& /* proto */) { - if (enum_value->options_ == NULL) { + if (enum_value->options_ == nullptr) { enum_value->options_ = &EnumValueOptions::default_instance(); } } -void DescriptorBuilder::CrossLinkService( - ServiceDescriptor* service, const ServiceDescriptorProto& proto) { - if (service->options_ == NULL) { +void DescriptorBuilder::CrossLinkService(ServiceDescriptor* service, + const ServiceDescriptorProto& proto) { + if (service->options_ == nullptr) { service->options_ = &ServiceOptions::default_instance(); } @@ -5691,9 +6420,9 @@ void DescriptorBuilder::CrossLinkService( } } -void DescriptorBuilder::CrossLinkMethod( - MethodDescriptor* method, const MethodDescriptorProto& proto) { - if (method->options_ == NULL) { +void DescriptorBuilder::CrossLinkMethod(MethodDescriptor* method, + const MethodDescriptorProto& proto) { + if (method->options_ == nullptr) { method->options_ = &MethodOptions::default_instance(); } @@ -5709,12 +6438,12 @@ void DescriptorBuilder::CrossLinkMethod( } else { method->input_type_.SetLazy(proto.input_type(), file_); } - } else if (input_type.type != Symbol::MESSAGE) { + } else if (input_type.type() != Symbol::MESSAGE) { AddError(method->full_name(), proto, DescriptorPool::ErrorCollector::INPUT_TYPE, "\"" + proto.input_type() + "\" is not a message type."); } else { - method->input_type_.Set(input_type.descriptor); + method->input_type_.Set(input_type.descriptor()); } Symbol output_type = @@ -5729,21 +6458,21 @@ void DescriptorBuilder::CrossLinkMethod( } else { method->output_type_.SetLazy(proto.output_type(), file_); } - } else if (output_type.type != Symbol::MESSAGE) { + } else if (output_type.type() != Symbol::MESSAGE) { AddError(method->full_name(), proto, DescriptorPool::ErrorCollector::OUTPUT_TYPE, "\"" + proto.output_type() + "\" is not a message type."); } else { - method->output_type_.Set(output_type.descriptor); + method->output_type_.Set(output_type.descriptor()); } } // ------------------------------------------------------------------- -#define VALIDATE_OPTIONS_FROM_ARRAY(descriptor, array_name, type) \ - for (int i = 0; i < descriptor->array_name##_count(); ++i) { \ - Validate##type##Options(descriptor->array_name##s_ + i, \ - proto.array_name(i)); \ +#define VALIDATE_OPTIONS_FROM_ARRAY(descriptor, array_name, type) \ + for (int i = 0; i < descriptor->array_name##_count(); ++i) { \ + Validate##type##Options(descriptor->array_name##s_ + i, \ + proto.array_name(i)); \ } // Determine if the file uses optimize_for = LITE_RUNTIME, being careful to @@ -5751,7 +6480,7 @@ void DescriptorBuilder::CrossLinkMethod( static bool IsLite(const FileDescriptor* file) { // TODO(kenton): I don't even remember how many of these conditions are // actually possible. I'm just being super-safe. - return file != NULL && + return file != nullptr && &file->options() != &FileOptions::default_instance() && file->options().optimize_for() == FileOptions::LITE_RUNTIME; } @@ -5768,11 +6497,12 @@ void DescriptorBuilder::ValidateFileOptions(FileDescriptor* file, for (int i = 0; i < file->dependency_count(); i++) { if (IsLite(file->dependency(i))) { AddError( - file->name(), proto, - DescriptorPool::ErrorCollector::OTHER, - "Files that do not use optimize_for = LITE_RUNTIME cannot import " - "files which do use this option. This file is not lite, but it " - "imports \"" + file->dependency(i)->name() + "\" which is."); + file->dependency(i)->name(), proto, + DescriptorPool::ErrorCollector::IMPORT, + "Files that do not use optimize_for = LITE_RUNTIME cannot import " + "files which do use this option. This file is not lite, but it " + "imports \"" + + file->dependency(i)->name() + "\" which is."); break; } } @@ -5782,8 +6512,8 @@ void DescriptorBuilder::ValidateFileOptions(FileDescriptor* file, } } -void DescriptorBuilder::ValidateProto3( - FileDescriptor* file, const FileDescriptorProto& proto) { +void DescriptorBuilder::ValidateProto3(FileDescriptor* file, + const FileDescriptorProto& proto) { for (int i = 0; i < file->extension_count(); ++i) { ValidateProto3Field(file->extensions_ + i, proto.extension(i)); } @@ -5795,111 +6525,106 @@ void DescriptorBuilder::ValidateProto3( } } -static string ToLowercaseWithoutUnderscores(const string& name) { - string result; - for (int i = 0; i < name.size(); ++i) { - if (name[i] != '_') { - if (name[i] >= 'A' && name[i] <= 'Z') { - result.push_back(name[i] - 'A' + 'a'); +static std::string ToLowercaseWithoutUnderscores(const std::string& name) { + std::string result; + for (char character : name) { + if (character != '_') { + if (character >= 'A' && character <= 'Z') { + result.push_back(character - 'A' + 'a'); } else { - result.push_back(name[i]); + result.push_back(character); } } } return result; } -void DescriptorBuilder::ValidateProto3Message( - Descriptor* message, const DescriptorProto& proto) { +void DescriptorBuilder::ValidateProto3Message(Descriptor* message, + const DescriptorProto& proto) { for (int i = 0; i < message->nested_type_count(); ++i) { - ValidateProto3Message(message->nested_types_ + i, - proto.nested_type(i)); + ValidateProto3Message(message->nested_types_ + i, proto.nested_type(i)); } for (int i = 0; i < message->enum_type_count(); ++i) { - ValidateProto3Enum(message->enum_types_ + i, - proto.enum_type(i)); + ValidateProto3Enum(message->enum_types_ + i, proto.enum_type(i)); } for (int i = 0; i < message->field_count(); ++i) { ValidateProto3Field(message->fields_ + i, proto.field(i)); } for (int i = 0; i < message->extension_count(); ++i) { - ValidateProto3Field(message->extensions_ +i, proto.extension(i)); + ValidateProto3Field(message->extensions_ + i, proto.extension(i)); } if (message->extension_range_count() > 0) { - AddError(message->full_name(), proto, - DescriptorPool::ErrorCollector::OTHER, + AddError(message->full_name(), proto.extension_range(0), + DescriptorPool::ErrorCollector::NUMBER, "Extension ranges are not allowed in proto3."); } if (message->options().message_set_wire_format()) { // Using MessageSet doesn't make sense since we disallow extensions. - AddError(message->full_name(), proto, - DescriptorPool::ErrorCollector::OTHER, + AddError(message->full_name(), proto, DescriptorPool::ErrorCollector::NAME, "MessageSet is not supported in proto3."); } // In proto3, we reject field names if they conflict in camelCase. // Note that we currently enforce a stricter rule: Field names must be // unique after being converted to lowercase with underscores removed. - std::map name_to_field; + std::map name_to_field; for (int i = 0; i < message->field_count(); ++i) { - string lowercase_name = ToLowercaseWithoutUnderscores( - message->field(i)->name()); + std::string lowercase_name = + ToLowercaseWithoutUnderscores(message->field(i)->name()); if (name_to_field.find(lowercase_name) != name_to_field.end()) { - AddError(message->full_name(), proto, - DescriptorPool::ErrorCollector::OTHER, + AddError(message->full_name(), proto.field(i), + DescriptorPool::ErrorCollector::NAME, "The JSON camel-case name of field \"" + - message->field(i)->name() + "\" conflicts with field \"" + - name_to_field[lowercase_name]->name() + "\". This is not " + - "allowed in proto3."); + message->field(i)->name() + "\" conflicts with field \"" + + name_to_field[lowercase_name]->name() + "\". This is not " + + "allowed in proto3."); } else { name_to_field[lowercase_name] = message->field(i); } } } -void DescriptorBuilder::ValidateProto3Field( - FieldDescriptor* field, const FieldDescriptorProto& proto) { +void DescriptorBuilder::ValidateProto3Field(FieldDescriptor* field, + const FieldDescriptorProto& proto) { if (field->is_extension() && !AllowedExtendeeInProto3(field->containing_type()->full_name())) { AddError(field->full_name(), proto, - DescriptorPool::ErrorCollector::OTHER, + DescriptorPool::ErrorCollector::EXTENDEE, "Extensions in proto3 are only allowed for defining options."); } if (field->is_required()) { - AddError(field->full_name(), proto, - DescriptorPool::ErrorCollector::OTHER, + AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, "Required fields are not allowed in proto3."); } if (field->has_default_value()) { - AddError( - field->full_name(), proto, DescriptorPool::ErrorCollector::OTHER, - "Explicit default values are not allowed in proto3."); + AddError(field->full_name(), proto, + DescriptorPool::ErrorCollector::DEFAULT_VALUE, + "Explicit default values are not allowed in proto3."); } if (field->cpp_type() == FieldDescriptor::CPPTYPE_ENUM && field->enum_type() && - field->enum_type()->file()->syntax() != FileDescriptor::SYNTAX_PROTO3) { + field->enum_type()->file()->syntax() != FileDescriptor::SYNTAX_PROTO3 && + field->enum_type()->file()->syntax() != FileDescriptor::SYNTAX_UNKNOWN) { // Proto3 messages can only use Proto3 enum types; otherwise we can't // guarantee that the default value is zero. - AddError(field->full_name(), proto, - DescriptorPool::ErrorCollector::TYPE, + AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, "Enum type \"" + field->enum_type()->full_name() + - "\" is not a proto3 enum, but is used in \"" + - field->containing_type()->full_name() + - "\" which is a proto3 message type."); + "\" is not a proto3 enum, but is used in \"" + + field->containing_type()->full_name() + + "\" which is a proto3 message type."); } if (field->type() == FieldDescriptor::TYPE_GROUP) { - AddError(field->full_name(), proto, - DescriptorPool::ErrorCollector::TYPE, + AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, "Groups are not supported in proto3 syntax."); } } -void DescriptorBuilder::ValidateProto3Enum( - EnumDescriptor* enm, const EnumDescriptorProto& proto) { +void DescriptorBuilder::ValidateProto3Enum(EnumDescriptor* enm, + const EnumDescriptorProto& proto) { if (enm->value_count() > 0 && enm->value(0)->number() != 0) { - AddError( - enm->full_name(), proto, DescriptorPool::ErrorCollector::OTHER, - "The first enum value must be zero in proto3."); + AddError(enm->full_name(), proto.value(0), + DescriptorPool::ErrorCollector::NUMBER, + "The first enum value must be zero in proto3."); } } @@ -5910,32 +6635,34 @@ void DescriptorBuilder::ValidateMessageOptions(Descriptor* message, VALIDATE_OPTIONS_FROM_ARRAY(message, enum_type, Enum); VALIDATE_OPTIONS_FROM_ARRAY(message, extension, Field); - const int64 max_extension_range = - static_cast(message->options().message_set_wire_format() ? - kint32max : - FieldDescriptor::kMaxNumber); + const int64_t max_extension_range = + static_cast(message->options().message_set_wire_format() + ? std::numeric_limits::max() + : FieldDescriptor::kMaxNumber); for (int i = 0; i < message->extension_range_count(); ++i) { if (message->extension_range(i)->end > max_extension_range + 1) { - AddError( - message->full_name(), proto.extension_range(i), - DescriptorPool::ErrorCollector::NUMBER, - strings::Substitute("Extension numbers cannot be greater than $0.", - max_extension_range)); + AddError(message->full_name(), proto.extension_range(i), + DescriptorPool::ErrorCollector::NUMBER, + strings::Substitute("Extension numbers cannot be greater than $0.", + max_extension_range)); } + + ValidateExtensionRangeOptions(message->full_name(), + message->extension_ranges_ + i, + proto.extension_range(i)); } } -void DescriptorBuilder::ValidateFieldOptions(FieldDescriptor* field, - const FieldDescriptorProto& proto) { +void DescriptorBuilder::ValidateFieldOptions( + FieldDescriptor* field, const FieldDescriptorProto& proto) { if (pool_->lazily_build_dependencies_ && (!field || !field->message_type())) { return; } // Only message type fields may be lazy. if (field->options().lazy()) { if (field->type() != FieldDescriptor::TYPE_MESSAGE) { - AddError(field->full_name(), proto, - DescriptorPool::ErrorCollector::TYPE, + AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, "[lazy = true] can only be specified for submessage fields."); } } @@ -5943,16 +6670,15 @@ void DescriptorBuilder::ValidateFieldOptions(FieldDescriptor* field, // Only repeated primitive fields may be packed. if (field->options().packed() && !field->is_packable()) { AddError( - field->full_name(), proto, - DescriptorPool::ErrorCollector::TYPE, - "[packed = true] can only be specified for repeated primitive fields."); + field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, + "[packed = true] can only be specified for repeated primitive fields."); } // Note: Default instance may not yet be initialized here, so we have to // avoid reading from it. - if (field->containing_type_ != NULL && + if (field->containing_type_ != nullptr && &field->containing_type()->options() != - &MessageOptions::default_instance() && + &MessageOptions::default_instance() && field->containing_type()->options().message_set_wire_format()) { if (field->is_extension()) { if (!field->is_optional() || @@ -5962,15 +6688,13 @@ void DescriptorBuilder::ValidateFieldOptions(FieldDescriptor* field, "Extensions of MessageSets must be optional messages."); } } else { - AddError(field->full_name(), proto, - DescriptorPool::ErrorCollector::NAME, + AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::NAME, "MessageSets cannot have fields, only extensions."); } } // Lite extensions can only be of Lite types. - if (IsLite(field->file()) && - field->containing_type_ != NULL && + if (IsLite(field->file()) && field->containing_type_ != nullptr && !IsLite(field->containing_type()->file())) { AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::EXTENDEE, @@ -5982,8 +6706,7 @@ void DescriptorBuilder::ValidateFieldOptions(FieldDescriptor* field, // Validate map types. if (field->is_map()) { if (!ValidateMapEntry(field, proto)) { - AddError(field->full_name(), proto, - DescriptorPool::ErrorCollector::OTHER, + AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, "map_entry should not be set explicitly. Use map instead."); } @@ -5991,30 +6714,43 @@ void DescriptorBuilder::ValidateFieldOptions(FieldDescriptor* field, ValidateJSType(field, proto); + // json_name option is not allowed on extension fields. Note that the + // json_name field in FieldDescriptorProto is always populated by protoc + // when it sends descriptor data to plugins (calculated from field name if + // the option is not explicitly set) so we can't rely on its presence to + // determine whether the json_name option is set on the field. Here we + // compare it against the default calculated json_name value and consider + // the option set if they are different. This won't catch the case when + // an user explicitly sets json_name to the default value, but should be + // good enough to catch common misuses. + if (field->is_extension() && + (field->has_json_name() && + field->json_name() != ToJsonName(field->name()))) { + AddError(field->full_name(), proto, + DescriptorPool::ErrorCollector::OPTION_NAME, + "option json_name is not allowed on extension fields."); + } + } void DescriptorBuilder::ValidateEnumOptions(EnumDescriptor* enm, const EnumDescriptorProto& proto) { VALIDATE_OPTIONS_FROM_ARRAY(enm, value, EnumValue); if (!enm->options().has_allow_alias() || !enm->options().allow_alias()) { - std::map used_values; + std::map used_values; for (int i = 0; i < enm->value_count(); ++i) { const EnumValueDescriptor* enum_value = enm->value(i); if (used_values.find(enum_value->number()) != used_values.end()) { - string error = + std::string error = "\"" + enum_value->full_name() + "\" uses the same enum value as \"" + - used_values[enum_value->number()] + "\". If this is intended, set " + used_values[enum_value->number()] + + "\". If this is intended, set " "'option allow_alias = true;' to the enum definition."; if (!enm->options().allow_alias()) { // Generate error if duplicated enum values are explicitly disallowed. - AddError(enm->full_name(), proto, - DescriptorPool::ErrorCollector::NUMBER, - error); - } else { - // Generate warning if duplicated values are found but the option - // isn't set. - GOOGLE_LOG(ERROR) << error; + AddError(enm->full_name(), proto.value(i), + DescriptorPool::ErrorCollector::NUMBER, error); } } else { used_values[enum_value->number()] = enum_value->full_name(); @@ -6028,31 +6764,38 @@ void DescriptorBuilder::ValidateEnumValueOptions( const EnumValueDescriptorProto& /* proto */) { // Nothing to do so far. } -void DescriptorBuilder::ValidateServiceOptions(ServiceDescriptor* service, - const ServiceDescriptorProto& proto) { + +void DescriptorBuilder::ValidateExtensionRangeOptions( + const std::string& full_name, Descriptor::ExtensionRange* extension_range, + const DescriptorProto_ExtensionRange& proto) { + (void)full_name; // Parameter is used by Google-internal code. + (void)extension_range; // Parameter is used by Google-internal code. +} + +void DescriptorBuilder::ValidateServiceOptions( + ServiceDescriptor* service, const ServiceDescriptorProto& proto) { if (IsLite(service->file()) && (service->file()->options().cc_generic_services() || service->file()->options().java_generic_services())) { - AddError(service->full_name(), proto, - DescriptorPool::ErrorCollector::NAME, + AddError(service->full_name(), proto, DescriptorPool::ErrorCollector::NAME, "Files with optimize_for = LITE_RUNTIME cannot define services " "unless you set both options cc_generic_services and " - "java_generic_sevices to false."); + "java_generic_services to false."); } VALIDATE_OPTIONS_FROM_ARRAY(service, method, Method); } -void DescriptorBuilder::ValidateMethodOptions(MethodDescriptor* /* method */, - const MethodDescriptorProto& /* proto */) { +void DescriptorBuilder::ValidateMethodOptions( + MethodDescriptor* /* method */, const MethodDescriptorProto& /* proto */) { // Nothing to do so far. } bool DescriptorBuilder::ValidateMapEntry(FieldDescriptor* field, const FieldDescriptorProto& proto) { const Descriptor* message = field->message_type(); - if (// Must not contain extensions, extension range or nested message or - // enums + if ( // Must not contain extensions, extension range or nested message or + // enums message->extension_count() != 0 || field->label() != FieldDescriptor::LABEL_REPEATED || message->extension_range_count() != 0 || @@ -6066,8 +6809,8 @@ bool DescriptorBuilder::ValidateMapEntry(FieldDescriptor* field, return false; } - const FieldDescriptor* key = message->field(0); - const FieldDescriptor* value = message->field(1); + const FieldDescriptor* key = message->map_key(); + const FieldDescriptor* value = message->map_value(); if (key->label() != FieldDescriptor::LABEL_OPTIONAL || key->number() != 1 || key->name() != "key") { return false; @@ -6080,9 +6823,8 @@ bool DescriptorBuilder::ValidateMapEntry(FieldDescriptor* field, // Check key types are legal. switch (key->type()) { case FieldDescriptor::TYPE_ENUM: - AddError( - field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, - "Key in map fields cannot be enum types."); + AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, + "Key in map fields cannot be enum types."); break; case FieldDescriptor::TYPE_FLOAT: case FieldDescriptor::TYPE_DOUBLE: @@ -6107,15 +6849,14 @@ bool DescriptorBuilder::ValidateMapEntry(FieldDescriptor* field, case FieldDescriptor::TYPE_SFIXED64: // Legal cases break; - // Do not add a default, so that the compiler will complain when new types - // are added. + // Do not add a default, so that the compiler will complain when new types + // are added. } if (value->type() == FieldDescriptor::TYPE_ENUM) { if (value->enum_type()->value(0)->number() != 0) { - AddError( - field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, - "Enum value in map must define 0 as the first value."); + AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, + "Enum value in map must define 0 as the first value."); } } @@ -6124,10 +6865,10 @@ bool DescriptorBuilder::ValidateMapEntry(FieldDescriptor* field, void DescriptorBuilder::DetectMapConflicts(const Descriptor* message, const DescriptorProto& proto) { - std::map seen_types; + std::map seen_types; for (int i = 0; i < message->nested_type_count(); ++i) { const Descriptor* nested = message->nested_type(i); - std::pair::iterator, bool> result = + std::pair::iterator, bool> result = seen_types.insert(std::make_pair(nested->name(), nested)); if (!result.second) { if (result.first->second->options().map_entry() || @@ -6135,7 +6876,7 @@ void DescriptorBuilder::DetectMapConflicts(const Descriptor* message, AddError(message->full_name(), proto, DescriptorPool::ErrorCollector::NAME, "Expanded map entry type " + nested->name() + - " conflicts with an existing nested message type."); + " conflicts with an existing nested message type."); } } // Recursively test on the nested types. @@ -6144,37 +6885,37 @@ void DescriptorBuilder::DetectMapConflicts(const Descriptor* message, // Check for conflicted field names. for (int i = 0; i < message->field_count(); ++i) { const FieldDescriptor* field = message->field(i); - std::map::iterator iter = + std::map::iterator iter = seen_types.find(field->name()); if (iter != seen_types.end() && iter->second->options().map_entry()) { AddError(message->full_name(), proto, DescriptorPool::ErrorCollector::NAME, "Expanded map entry type " + iter->second->name() + - " conflicts with an existing field."); + " conflicts with an existing field."); } } // Check for conflicted enum names. for (int i = 0; i < message->enum_type_count(); ++i) { const EnumDescriptor* enum_desc = message->enum_type(i); - std::map::iterator iter = + std::map::iterator iter = seen_types.find(enum_desc->name()); if (iter != seen_types.end() && iter->second->options().map_entry()) { AddError(message->full_name(), proto, DescriptorPool::ErrorCollector::NAME, "Expanded map entry type " + iter->second->name() + - " conflicts with an existing enum type."); + " conflicts with an existing enum type."); } } // Check for conflicted oneof names. for (int i = 0; i < message->oneof_decl_count(); ++i) { const OneofDescriptor* oneof_desc = message->oneof_decl(i); - std::map::iterator iter = + std::map::iterator iter = seen_types.find(oneof_desc->name()); if (iter != seen_types.end() && iter->second->options().map_entry()) { AddError(message->full_name(), proto, DescriptorPool::ErrorCollector::NAME, "Expanded map entry type " + iter->second->name() + - " conflicts with an existing oneof type."); + " conflicts with an existing oneof type."); } } } @@ -6202,7 +6943,7 @@ void DescriptorBuilder::ValidateJSType(FieldDescriptor* field, AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, "Illegal jstype for int64, uint64, sint64, fixed64 " "or sfixed64 field: " + - FieldOptions_JSType_descriptor()->value(jstype)->name()); + FieldOptions_JSType_descriptor()->value(jstype)->name()); break; // No other types permit a jstype option. @@ -6219,12 +6960,12 @@ void DescriptorBuilder::ValidateJSType(FieldDescriptor* field, // ------------------------------------------------------------------- DescriptorBuilder::OptionInterpreter::OptionInterpreter( - DescriptorBuilder* builder) : builder_(builder) { + DescriptorBuilder* builder) + : builder_(builder) { GOOGLE_CHECK(builder_); } -DescriptorBuilder::OptionInterpreter::~OptionInterpreter() { -} +DescriptorBuilder::OptionInterpreter::~OptionInterpreter() {} bool DescriptorBuilder::OptionInterpreter::InterpretOptions( OptionsToInterpret* options_to_interpret) { @@ -6240,32 +6981,39 @@ bool DescriptorBuilder::OptionInterpreter::InterpretOptions( // and clear them, since we're about to interpret them. const FieldDescriptor* uninterpreted_options_field = options->GetDescriptor()->FindFieldByName("uninterpreted_option"); - GOOGLE_CHECK(uninterpreted_options_field != NULL) + GOOGLE_CHECK(uninterpreted_options_field != nullptr) << "No field named \"uninterpreted_option\" in the Options proto."; options->GetReflection()->ClearField(options, uninterpreted_options_field); + std::vector src_path = options_to_interpret->element_path; + src_path.push_back(uninterpreted_options_field->number()); + // Find the uninterpreted_option field in the original options. const FieldDescriptor* original_uninterpreted_options_field = - original_options->GetDescriptor()-> - FindFieldByName("uninterpreted_option"); - GOOGLE_CHECK(original_uninterpreted_options_field != NULL) + original_options->GetDescriptor()->FindFieldByName( + "uninterpreted_option"); + GOOGLE_CHECK(original_uninterpreted_options_field != nullptr) << "No field named \"uninterpreted_option\" in the Options proto."; - const int num_uninterpreted_options = original_options->GetReflection()-> - FieldSize(*original_options, original_uninterpreted_options_field); + const int num_uninterpreted_options = + original_options->GetReflection()->FieldSize( + *original_options, original_uninterpreted_options_field); for (int i = 0; i < num_uninterpreted_options; ++i) { + src_path.push_back(i); uninterpreted_option_ = down_cast( &original_options->GetReflection()->GetRepeatedMessage( *original_options, original_uninterpreted_options_field, i)); - if (!InterpretSingleOption(options)) { + if (!InterpretSingleOption(options, src_path, + options_to_interpret->element_path)) { // Error already added by InterpretSingleOption(). failed = true; break; } + src_path.pop_back(); } // Reset these, so we don't have any dangling pointers. - uninterpreted_option_ = NULL; - options_to_interpret_ = NULL; + uninterpreted_option_ = nullptr; + options_to_interpret_ = nullptr; if (!failed) { // InterpretSingleOption() added the interpreted options in the @@ -6276,25 +7024,35 @@ bool DescriptorBuilder::OptionInterpreter::InterpretOptions( // If they are not known, that's OK too. They will get reparsed into the // UnknownFieldSet and wait there until the message is parsed by something // that does know about the options. - string buf; - GOOGLE_CHECK(options->AppendPartialToString(&buf)) - << "Protocol message could not be serialized."; - GOOGLE_CHECK(options->ParsePartialFromString(buf)) - << "Protocol message serialized itself in invalid fashion."; - if (!options->IsInitialized()) { - builder_->AddWarning( + + // Keep the unparsed options around in case the reparsing fails. + std::unique_ptr unparsed_options(options->New()); + options->GetReflection()->Swap(unparsed_options.get(), options); + + std::string buf; + if (!unparsed_options->AppendToString(&buf) || + !options->ParseFromString(buf)) { + builder_->AddError( options_to_interpret->element_name, *original_options, DescriptorPool::ErrorCollector::OTHER, - "Options could not be fully parsed using the proto descriptors " - "compiled into this binary. Missing required fields: " + - options->InitializationErrorString()); + "Some options could not be correctly parsed using the proto " + "descriptors compiled into this binary.\n" + "Unparsed options: " + + unparsed_options->ShortDebugString() + + "\n" + "Parsing attempt: " + + options->ShortDebugString()); + // Restore the unparsed options. + options->GetReflection()->Swap(unparsed_options.get(), options); } } + return !failed; } bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( - Message* options) { + Message* options, const std::vector& src_path, + const std::vector& options_path) { // First do some basic validation. if (uninterpreted_option_->name_size() == 0) { // This should never happen unless the parser has gone seriously awry or @@ -6302,15 +7060,16 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( return AddNameError("Option must have a name."); } if (uninterpreted_option_->name(0).name_part() == "uninterpreted_option") { - return AddNameError("Option must not use reserved name " - "\"uninterpreted_option\"."); + return AddNameError( + "Option must not use reserved name " + "\"uninterpreted_option\"."); } - const Descriptor* options_descriptor = NULL; + const Descriptor* options_descriptor = nullptr; // Get the options message's descriptor from the builder's pool, so that we - // get the version that knows about any extension options declared in the - // file we're currently building. The descriptor should be there as long as - // the file we're building imported "google/protobuf/descriptors.proto". + // get the version that knows about any extension options declared in the file + // we're currently building. The descriptor should be there as long as the + // file we're building imported descriptor.proto. // Note that we use DescriptorBuilder::FindSymbolNotEnforcingDeps(), not // DescriptorPool::FindMessageTypeByName() because we're already holding the @@ -6318,10 +7077,9 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( // FindSymbol() because files that use custom options only need to depend on // the file that defines the option, not descriptor.proto itself. Symbol symbol = builder_->FindSymbolNotEnforcingDeps( - options->GetDescriptor()->full_name()); - if (!symbol.IsNull() && symbol.type == Symbol::MESSAGE) { - options_descriptor = symbol.descriptor; - } else { + options->GetDescriptor()->full_name()); + options_descriptor = symbol.descriptor(); + if (options_descriptor == nullptr) { // The options message's descriptor was not in the builder's pool, so use // the standard version from the generated pool. We're not holding the // generated pool's mutex, so we can search it the straightforward way. @@ -6336,12 +7094,15 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( // through in |intermediate_fields|. As we go, we reconstruct the full option // name in |debug_msg_name|, for use in error messages. const Descriptor* descriptor = options_descriptor; - const FieldDescriptor* field = NULL; + const FieldDescriptor* field = nullptr; std::vector intermediate_fields; - string debug_msg_name = ""; + std::string debug_msg_name = ""; + + std::vector dest_path = options_path; for (int i = 0; i < uninterpreted_option_->name_size(); ++i) { - const string& name_part = uninterpreted_option_->name(i).name_part(); + builder_->undefine_resolved_name_.clear(); + const std::string& name_part = uninterpreted_option_->name(i).name_part(); if (debug_msg_name.size() > 0) { debug_msg_name += "."; } @@ -6352,11 +7113,9 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( // DescriptorPool::FindExtensionByName(), for two reasons: 1) It allows // relative lookups, and 2) because we're already holding the pool's // mutex, and the latter method locks it again. - symbol = builder_->LookupSymbol(name_part, - options_to_interpret_->name_scope); - if (!symbol.IsNull() && symbol.type == Symbol::FIELD) { - field = symbol.field_descriptor; - } + symbol = + builder_->LookupSymbol(name_part, options_to_interpret_->name_scope); + field = symbol.field_descriptor(); // If we don't find the field then the field's descriptor was not in the // builder's pool, but there's no point in looking in the generated // pool. We require that you import the file that defines any extensions @@ -6367,7 +7126,7 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( field = descriptor->FindFieldByName(name_part); } - if (field == NULL) { + if (field == nullptr) { if (get_allow_unknown(builder_->pool_)) { // We can't find the option, but AllowUnknownDependencies() is enabled, // so we will just leave it as uninterpreted. @@ -6383,7 +7142,10 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( debug_msg_name.substr(1) + "\") to start from the outermost scope."); } else { - return AddNameError("Option \"" + debug_msg_name + "\" unknown."); + return AddNameError( + "Option \"" + debug_msg_name + + "\" unknown. Ensure that your proto" + + " definition file imports the proto which defines the option."); } } else if (field->containing_type() != descriptor) { if (get_is_placeholder(field->containing_type())) { @@ -6402,19 +7164,24 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( "\" is not a field or extension of message \"" + descriptor->name() + "\"."); } - } else if (i < uninterpreted_option_->name_size() - 1) { - if (field->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) { - return AddNameError("Option \"" + debug_msg_name + - "\" is an atomic type, not a message."); - } else if (field->is_repeated()) { - return AddNameError("Option field \"" + debug_msg_name + - "\" is a repeated message. Repeated message " - "options must be initialized using an " - "aggregate value."); - } else { - // Drill down into the submessage. - intermediate_fields.push_back(field); - descriptor = field->message_type(); + } else { + // accumulate field numbers to form path to interpreted option + dest_path.push_back(field->number()); + + if (i < uninterpreted_option_->name_size() - 1) { + if (field->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) { + return AddNameError("Option \"" + debug_msg_name + + "\" is an atomic type, not a message."); + } else if (field->is_repeated()) { + return AddNameError("Option field \"" + debug_msg_name + + "\" is a repeated message. Repeated message " + "options must be initialized using an " + "aggregate value."); + } else { + // Drill down into the submessage. + intermediate_fields.push_back(field); + descriptor = field->message_type(); + } } } } @@ -6427,18 +7194,17 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( // known will populate them correctly. // First see if the option is already set. - if (!field->is_repeated() && !ExamineIfOptionIsSet( - intermediate_fields.begin(), - intermediate_fields.end(), - field, debug_msg_name, + if (!field->is_repeated() && + !ExamineIfOptionIsSet( + intermediate_fields.begin(), intermediate_fields.end(), field, + debug_msg_name, options->GetReflection()->GetUnknownFields(*options))) { return false; // ExamineIfOptionIsSet() already added the error. } - // First set the value on the UnknownFieldSet corresponding to the // innermost message. - google::protobuf::scoped_ptr unknown_fields(new UnknownFieldSet()); + std::unique_ptr unknown_fields(new UnknownFieldSet()); if (!SetOptionValue(field, unknown_fields.get())) { return false; // SetOptionValue() already added the error. } @@ -6448,7 +7214,7 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( for (std::vector::reverse_iterator iter = intermediate_fields.rbegin(); iter != intermediate_fields.rend(); ++iter) { - google::protobuf::scoped_ptr parent_unknown_fields( + std::unique_ptr parent_unknown_fields( new UnknownFieldSet()); switch ((*iter)->type()) { case FieldDescriptor::TYPE_MESSAGE: { @@ -6463,8 +7229,8 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( } case FieldDescriptor::TYPE_GROUP: { - parent_unknown_fields->AddGroup((*iter)->number()) - ->MergeFrom(*unknown_fields); + parent_unknown_fields->AddGroup((*iter)->number()) + ->MergeFrom(*unknown_fields); break; } @@ -6481,24 +7247,125 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( options->GetReflection()->MutableUnknownFields(options)->MergeFrom( *unknown_fields); + // record the element path of the interpreted option + if (field->is_repeated()) { + int index = repeated_option_counts_[dest_path]++; + dest_path.push_back(index); + } + interpreted_paths_[src_path] = dest_path; + return true; } +void DescriptorBuilder::OptionInterpreter::UpdateSourceCodeInfo( + SourceCodeInfo* info) { + if (interpreted_paths_.empty()) { + // nothing to do! + return; + } + + // We find locations that match keys in interpreted_paths_ and + // 1) replace the path with the corresponding value in interpreted_paths_ + // 2) remove any subsequent sub-locations (sub-location is one whose path + // has the parent path as a prefix) + // + // To avoid quadratic behavior of removing interior rows as we go, + // we keep a copy. But we don't actually copy anything until we've + // found the first match (so if the source code info has no locations + // that need to be changed, there is zero copy overhead). + + RepeatedPtrField* locs = info->mutable_location(); + RepeatedPtrField new_locs; + bool copying = false; + + std::vector pathv; + bool matched = false; + + for (RepeatedPtrField::iterator loc = locs->begin(); + loc != locs->end(); loc++) { + if (matched) { + // see if this location is in the range to remove + bool loc_matches = true; + if (loc->path_size() < static_cast(pathv.size())) { + loc_matches = false; + } else { + for (size_t j = 0; j < pathv.size(); j++) { + if (loc->path(j) != pathv[j]) { + loc_matches = false; + break; + } + } + } + + if (loc_matches) { + // don't copy this row since it is a sub-location that we're removing + continue; + } + + matched = false; + } + + pathv.clear(); + for (int j = 0; j < loc->path_size(); j++) { + pathv.push_back(loc->path(j)); + } + + std::map, std::vector>::iterator entry = + interpreted_paths_.find(pathv); + + if (entry == interpreted_paths_.end()) { + // not a match + if (copying) { + *new_locs.Add() = *loc; + } + continue; + } + + matched = true; + + if (!copying) { + // initialize the copy we are building + copying = true; + new_locs.Reserve(locs->size()); + for (RepeatedPtrField::iterator it = + locs->begin(); + it != loc; it++) { + *new_locs.Add() = *it; + } + } + + // add replacement and update its path + SourceCodeInfo_Location* replacement = new_locs.Add(); + *replacement = *loc; + replacement->clear_path(); + for (std::vector::iterator rit = entry->second.begin(); + rit != entry->second.end(); rit++) { + replacement->add_path(*rit); + } + } + + // if we made a changed copy, put it in place + if (copying) { + *locs = new_locs; + } +} + void DescriptorBuilder::OptionInterpreter::AddWithoutInterpreting( const UninterpretedOption& uninterpreted_option, Message* options) { const FieldDescriptor* field = - options->GetDescriptor()->FindFieldByName("uninterpreted_option"); - GOOGLE_CHECK(field != NULL); + options->GetDescriptor()->FindFieldByName("uninterpreted_option"); + GOOGLE_CHECK(field != nullptr); - options->GetReflection()->AddMessage(options, field) - ->CopyFrom(uninterpreted_option); + options->GetReflection() + ->AddMessage(options, field) + ->CopyFrom(uninterpreted_option); } bool DescriptorBuilder::OptionInterpreter::ExamineIfOptionIsSet( std::vector::const_iterator intermediate_fields_iter, std::vector::const_iterator intermediate_fields_end, - const FieldDescriptor* innermost_field, const string& debug_msg_name, + const FieldDescriptor* innermost_field, const std::string& debug_msg_name, const UnknownFieldSet& unknown_fields) { // We do linear searches of the UnknownFieldSet and its sub-groups. This // should be fine since it's unlikely that any one options structure will @@ -6528,8 +7395,8 @@ bool DescriptorBuilder::OptionInterpreter::ExamineIfOptionIsSet( if (intermediate_unknown_fields.ParseFromString( unknown_field->length_delimited()) && !ExamineIfOptionIsSet(intermediate_fields_iter + 1, - intermediate_fields_end, - innermost_field, debug_msg_name, + intermediate_fields_end, innermost_field, + debug_msg_name, intermediate_unknown_fields)) { return false; // Error already added. } @@ -6539,9 +7406,8 @@ bool DescriptorBuilder::OptionInterpreter::ExamineIfOptionIsSet( case FieldDescriptor::TYPE_GROUP: if (unknown_field->type() == UnknownField::TYPE_GROUP) { if (!ExamineIfOptionIsSet(intermediate_fields_iter + 1, - intermediate_fields_end, - innermost_field, debug_msg_name, - unknown_field->group())) { + intermediate_fields_end, innermost_field, + debug_msg_name, unknown_field->group())) { return false; // Error already added. } } @@ -6557,15 +7423,13 @@ bool DescriptorBuilder::OptionInterpreter::ExamineIfOptionIsSet( } bool DescriptorBuilder::OptionInterpreter::SetOptionValue( - const FieldDescriptor* option_field, - UnknownFieldSet* unknown_fields) { + const FieldDescriptor* option_field, UnknownFieldSet* unknown_fields) { // We switch on the CppType to validate. switch (option_field->cpp_type()) { - case FieldDescriptor::CPPTYPE_INT32: if (uninterpreted_option_->has_positive_int_value()) { if (uninterpreted_option_->positive_int_value() > - static_cast(kint32max)) { + static_cast(std::numeric_limits::max())) { return AddValueError("Value out of range for int32 option \"" + option_field->full_name() + "\"."); } else { @@ -6575,7 +7439,7 @@ bool DescriptorBuilder::OptionInterpreter::SetOptionValue( } } else if (uninterpreted_option_->has_negative_int_value()) { if (uninterpreted_option_->negative_int_value() < - static_cast(kint32min)) { + static_cast(std::numeric_limits::min())) { return AddValueError("Value out of range for int32 option \"" + option_field->full_name() + "\"."); } else { @@ -6592,7 +7456,7 @@ bool DescriptorBuilder::OptionInterpreter::SetOptionValue( case FieldDescriptor::CPPTYPE_INT64: if (uninterpreted_option_->has_positive_int_value()) { if (uninterpreted_option_->positive_int_value() > - static_cast(kint64max)) { + static_cast(std::numeric_limits::max())) { return AddValueError("Value out of range for int64 option \"" + option_field->full_name() + "\"."); } else { @@ -6612,7 +7476,8 @@ bool DescriptorBuilder::OptionInterpreter::SetOptionValue( case FieldDescriptor::CPPTYPE_UINT32: if (uninterpreted_option_->has_positive_int_value()) { - if (uninterpreted_option_->positive_int_value() > kuint32max) { + if (uninterpreted_option_->positive_int_value() > + std::numeric_limits::max()) { return AddValueError("Value out of range for uint32 option \"" + option_field->name() + "\"."); } else { @@ -6621,8 +7486,10 @@ bool DescriptorBuilder::OptionInterpreter::SetOptionValue( option_field->type(), unknown_fields); } } else { - return AddValueError("Value must be non-negative integer for uint32 " - "option \"" + option_field->full_name() + "\"."); + return AddValueError( + "Value must be non-negative integer for uint32 " + "option \"" + + option_field->full_name() + "\"."); } break; @@ -6632,8 +7499,10 @@ bool DescriptorBuilder::OptionInterpreter::SetOptionValue( uninterpreted_option_->positive_int_value(), option_field->type(), unknown_fields); } else { - return AddValueError("Value must be non-negative integer for uint64 " - "option \"" + option_field->full_name() + "\"."); + return AddValueError( + "Value must be non-negative integer for uint64 " + "option \"" + + option_field->full_name() + "\"."); } break; @@ -6650,7 +7519,7 @@ bool DescriptorBuilder::OptionInterpreter::SetOptionValue( option_field->full_name() + "\"."); } unknown_fields->AddFixed32(option_field->number(), - google::protobuf::internal::WireFormatLite::EncodeFloat(value)); + internal::WireFormatLite::EncodeFloat(value)); break; } @@ -6667,40 +7536,46 @@ bool DescriptorBuilder::OptionInterpreter::SetOptionValue( option_field->full_name() + "\"."); } unknown_fields->AddFixed64(option_field->number(), - google::protobuf::internal::WireFormatLite::EncodeDouble(value)); + internal::WireFormatLite::EncodeDouble(value)); break; } case FieldDescriptor::CPPTYPE_BOOL: - uint64 value; + uint64_t value; if (!uninterpreted_option_->has_identifier_value()) { - return AddValueError("Value must be identifier for boolean option " - "\"" + option_field->full_name() + "\"."); + return AddValueError( + "Value must be identifier for boolean option " + "\"" + + option_field->full_name() + "\"."); } if (uninterpreted_option_->identifier_value() == "true") { value = 1; } else if (uninterpreted_option_->identifier_value() == "false") { value = 0; } else { - return AddValueError("Value must be \"true\" or \"false\" for boolean " - "option \"" + option_field->full_name() + "\"."); + return AddValueError( + "Value must be \"true\" or \"false\" for boolean " + "option \"" + + option_field->full_name() + "\"."); } unknown_fields->AddVarint(option_field->number(), value); break; case FieldDescriptor::CPPTYPE_ENUM: { if (!uninterpreted_option_->has_identifier_value()) { - return AddValueError("Value must be identifier for enum-valued option " - "\"" + option_field->full_name() + "\"."); + return AddValueError( + "Value must be identifier for enum-valued option " + "\"" + + option_field->full_name() + "\"."); } const EnumDescriptor* enum_type = option_field->enum_type(); - const string& value_name = uninterpreted_option_->identifier_value(); - const EnumValueDescriptor* enum_value = NULL; + const std::string& value_name = uninterpreted_option_->identifier_value(); + const EnumValueDescriptor* enum_value = nullptr; if (enum_type->file()->pool() != DescriptorPool::generated_pool()) { // Note that the enum value's fully-qualified name is a sibling of the // enum's name, not a child of it. - string fully_qualified_name = enum_type->full_name(); + std::string fully_qualified_name = enum_type->full_name(); fully_qualified_name.resize(fully_qualified_name.size() - enum_type->name().size()); fully_qualified_name += value_name; @@ -6710,15 +7585,16 @@ bool DescriptorBuilder::OptionInterpreter::SetOptionValue( // DescriptorPool::FindEnumValueByName() because we're already holding // the pool's mutex, and the latter method locks it again. Symbol symbol = - builder_->FindSymbolNotEnforcingDeps(fully_qualified_name); - if (!symbol.IsNull() && symbol.type == Symbol::ENUM_VALUE) { - if (symbol.enum_value_descriptor->type() != enum_type) { - return AddValueError("Enum type \"" + enum_type->full_name() + + builder_->FindSymbolNotEnforcingDeps(fully_qualified_name); + if (auto* candicate_descriptor = symbol.enum_value_descriptor()) { + if (candicate_descriptor->type() != enum_type) { + return AddValueError( + "Enum type \"" + enum_type->full_name() + "\" has no value named \"" + value_name + "\" for option \"" + option_field->full_name() + "\". This appears to be a value from a sibling type."); } else { - enum_value = symbol.enum_value_descriptor; + enum_value = candicate_descriptor; } } } else { @@ -6727,28 +7603,33 @@ bool DescriptorBuilder::OptionInterpreter::SetOptionValue( enum_value = enum_type->FindValueByName(value_name); } - if (enum_value == NULL) { + if (enum_value == nullptr) { return AddValueError("Enum type \"" + option_field->enum_type()->full_name() + - "\" has no value named \"" + value_name + "\" for " - "option \"" + option_field->full_name() + "\"."); + "\" has no value named \"" + value_name + + "\" for " + "option \"" + + option_field->full_name() + "\"."); } else { - // Sign-extension is not a problem, since we cast directly from int32 to - // uint64, without first going through uint32. - unknown_fields->AddVarint(option_field->number(), - static_cast(static_cast(enum_value->number()))); + // Sign-extension is not a problem, since we cast directly from int32_t + // to uint64_t, without first going through uint32_t. + unknown_fields->AddVarint( + option_field->number(), + static_cast(static_cast(enum_value->number()))); } break; } case FieldDescriptor::CPPTYPE_STRING: if (!uninterpreted_option_->has_string_value()) { - return AddValueError("Value must be quoted string for string option " - "\"" + option_field->full_name() + "\"."); + return AddValueError( + "Value must be quoted string for string option " + "\"" + + option_field->full_name() + "\"."); } // The string has already been unquoted and unescaped by the parser. unknown_fields->AddLengthDelimited(option_field->number(), - uninterpreted_option_->string_value()); + uninterpreted_option_->string_value()); break; case FieldDescriptor::CPPTYPE_MESSAGE: @@ -6766,18 +7647,28 @@ class DescriptorBuilder::OptionInterpreter::AggregateOptionFinder public: DescriptorBuilder* builder_; - virtual const FieldDescriptor* FindExtension( - Message* message, const string& name) const { + const Descriptor* FindAnyType(const Message& /*message*/, + const std::string& prefix, + const std::string& name) const override { + if (prefix != internal::kTypeGoogleApisComPrefix && + prefix != internal::kTypeGoogleProdComPrefix) { + return nullptr; + } + assert_mutex_held(builder_->pool_); + return builder_->FindSymbol(name).descriptor(); + } + + const FieldDescriptor* FindExtension(Message* message, + const std::string& name) const override { assert_mutex_held(builder_->pool_); const Descriptor* descriptor = message->GetDescriptor(); - Symbol result = builder_->LookupSymbolNoPlaceholder( - name, descriptor->full_name()); - if (result.type == Symbol::FIELD && - result.field_descriptor->is_extension()) { - return result.field_descriptor; - } else if (result.type == Symbol::MESSAGE && + Symbol result = + builder_->LookupSymbolNoPlaceholder(name, descriptor->full_name()); + if (auto* field = result.field_descriptor()) { + return field; + } else if (result.type() == Symbol::MESSAGE && descriptor->options().message_set_wire_format()) { - const Descriptor* foreign_type = result.descriptor; + const Descriptor* foreign_type = result.descriptor(); // The text format allows MessageSet items to be specified using // the type name, rather than the extension identifier. If the symbol // lookup returned a Message, and the enclosing Message has @@ -6794,7 +7685,7 @@ class DescriptorBuilder::OptionInterpreter::AggregateOptionFinder } } } - return NULL; + return nullptr; } }; @@ -6802,42 +7693,42 @@ class DescriptorBuilder::OptionInterpreter::AggregateOptionFinder namespace { class AggregateErrorCollector : public io::ErrorCollector { public: - string error_; + std::string error_; - virtual void AddError(int /* line */, int /* column */, - const string& message) { + void AddError(int /* line */, int /* column */, + const std::string& message) override { if (!error_.empty()) { error_ += "; "; } error_ += message; } - virtual void AddWarning(int /* line */, int /* column */, - const string& /* message */) { + void AddWarning(int /* line */, int /* column */, + const std::string& /* message */) override { // Ignore warnings } }; -} +} // namespace // We construct a dynamic message of the type corresponding to // option_field, parse the supplied text-format string into this // message, and serialize the resulting message to produce the value. bool DescriptorBuilder::OptionInterpreter::SetAggregateOption( - const FieldDescriptor* option_field, - UnknownFieldSet* unknown_fields) { + const FieldDescriptor* option_field, UnknownFieldSet* unknown_fields) { if (!uninterpreted_option_->has_aggregate_value()) { return AddValueError("Option \"" + option_field->full_name() + "\" is a message. To set the entire message, use " - "syntax like \"" + option_field->name() + + "syntax like \"" + + option_field->name() + " = { }\". " "To set fields within it, use " - "syntax like \"" + option_field->name() + - ".foo = value\"."); + "syntax like \"" + + option_field->name() + ".foo = value\"."); } const Descriptor* type = option_field->message_type(); - google::protobuf::scoped_ptr dynamic(dynamic_factory_.GetPrototype(type)->New()); - GOOGLE_CHECK(dynamic.get() != NULL) + std::unique_ptr dynamic(dynamic_factory_.GetPrototype(type)->New()); + GOOGLE_CHECK(dynamic.get() != nullptr) << "Could not create an instance of " << option_field->DebugString(); AggregateErrorCollector collector; @@ -6852,12 +7743,12 @@ bool DescriptorBuilder::OptionInterpreter::SetAggregateOption( option_field->name() + "\": " + collector.error_); return false; } else { - string serial; + std::string serial; dynamic->SerializeToString(&serial); // Never fails if (option_field->type() == FieldDescriptor::TYPE_MESSAGE) { unknown_fields->AddLengthDelimited(option_field->number(), serial); } else { - GOOGLE_CHECK_EQ(option_field->type(), FieldDescriptor::TYPE_GROUP); + GOOGLE_CHECK_EQ(option_field->type(), FieldDescriptor::TYPE_GROUP); UnknownFieldSet* group = unknown_fields->AddGroup(option_field->number()); group->ParseFromString(serial); } @@ -6865,21 +7756,22 @@ bool DescriptorBuilder::OptionInterpreter::SetAggregateOption( } } -void DescriptorBuilder::OptionInterpreter::SetInt32(int number, int32 value, - FieldDescriptor::Type type, UnknownFieldSet* unknown_fields) { +void DescriptorBuilder::OptionInterpreter::SetInt32( + int number, int32_t value, FieldDescriptor::Type type, + UnknownFieldSet* unknown_fields) { switch (type) { case FieldDescriptor::TYPE_INT32: - unknown_fields->AddVarint(number, - static_cast(static_cast(value))); + unknown_fields->AddVarint( + number, static_cast(static_cast(value))); break; case FieldDescriptor::TYPE_SFIXED32: - unknown_fields->AddFixed32(number, static_cast(value)); + unknown_fields->AddFixed32(number, static_cast(value)); break; case FieldDescriptor::TYPE_SINT32: - unknown_fields->AddVarint(number, - google::protobuf::internal::WireFormatLite::ZigZagEncode32(value)); + unknown_fields->AddVarint( + number, internal::WireFormatLite::ZigZagEncode32(value)); break; default: @@ -6888,20 +7780,21 @@ void DescriptorBuilder::OptionInterpreter::SetInt32(int number, int32 value, } } -void DescriptorBuilder::OptionInterpreter::SetInt64(int number, int64 value, - FieldDescriptor::Type type, UnknownFieldSet* unknown_fields) { +void DescriptorBuilder::OptionInterpreter::SetInt64( + int number, int64_t value, FieldDescriptor::Type type, + UnknownFieldSet* unknown_fields) { switch (type) { case FieldDescriptor::TYPE_INT64: - unknown_fields->AddVarint(number, static_cast(value)); + unknown_fields->AddVarint(number, static_cast(value)); break; case FieldDescriptor::TYPE_SFIXED64: - unknown_fields->AddFixed64(number, static_cast(value)); + unknown_fields->AddFixed64(number, static_cast(value)); break; case FieldDescriptor::TYPE_SINT64: - unknown_fields->AddVarint(number, - google::protobuf::internal::WireFormatLite::ZigZagEncode64(value)); + unknown_fields->AddVarint( + number, internal::WireFormatLite::ZigZagEncode64(value)); break; default: @@ -6910,15 +7803,16 @@ void DescriptorBuilder::OptionInterpreter::SetInt64(int number, int64 value, } } -void DescriptorBuilder::OptionInterpreter::SetUInt32(int number, uint32 value, - FieldDescriptor::Type type, UnknownFieldSet* unknown_fields) { +void DescriptorBuilder::OptionInterpreter::SetUInt32( + int number, uint32_t value, FieldDescriptor::Type type, + UnknownFieldSet* unknown_fields) { switch (type) { case FieldDescriptor::TYPE_UINT32: - unknown_fields->AddVarint(number, static_cast(value)); + unknown_fields->AddVarint(number, static_cast(value)); break; case FieldDescriptor::TYPE_FIXED32: - unknown_fields->AddFixed32(number, static_cast(value)); + unknown_fields->AddFixed32(number, static_cast(value)); break; default: @@ -6927,8 +7821,9 @@ void DescriptorBuilder::OptionInterpreter::SetUInt32(int number, uint32 value, } } -void DescriptorBuilder::OptionInterpreter::SetUInt64(int number, uint64 value, - FieldDescriptor::Type type, UnknownFieldSet* unknown_fields) { +void DescriptorBuilder::OptionInterpreter::SetUInt64( + int number, uint64_t value, FieldDescriptor::Type type, + UnknownFieldSet* unknown_fields) { switch (type) { case FieldDescriptor::TYPE_UINT64: unknown_fields->AddVarint(number, value); @@ -6946,43 +7841,31 @@ void DescriptorBuilder::OptionInterpreter::SetUInt64(int number, uint64 value, void DescriptorBuilder::LogUnusedDependency(const FileDescriptorProto& proto, const FileDescriptor* result) { + (void)result; // Parameter is used by Google-internal code. if (!unused_dependency_.empty()) { - std::set annotation_extensions; - annotation_extensions.insert("google.protobuf.MessageOptions"); - annotation_extensions.insert("google.protobuf.FileOptions"); - annotation_extensions.insert("google.protobuf.FieldOptions"); - annotation_extensions.insert("google.protobuf.EnumOptions"); - annotation_extensions.insert("google.protobuf.EnumValueOptions"); - annotation_extensions.insert("google.protobuf.EnumValueOptions"); - annotation_extensions.insert("google.protobuf.ServiceOptions"); - annotation_extensions.insert("google.protobuf.MethodOptions"); - annotation_extensions.insert("google.protobuf.StreamOptions"); - for (std::set::const_iterator - it = unused_dependency_.begin(); + auto itr = pool_->unused_import_track_files_.find(proto.name()); + bool is_error = + itr != pool_->unused_import_track_files_.end() && itr->second; + for (std::set::const_iterator it = + unused_dependency_.begin(); it != unused_dependency_.end(); ++it) { - // Do not log warnings for proto files which extend annotations. - int i; - for (i = 0 ; i < (*it)->extension_count(); ++i) { - if (annotation_extensions.find( - (*it)->extension(i)->containing_type()->full_name()) - != annotation_extensions.end()) { - break; - } - } - // Log warnings for unused imported files. - if (i == (*it)->extension_count()) { - string error_message = "Import " + (*it)->name() + " but not used."; - AddWarning((*it)->name(), proto, DescriptorPool::ErrorCollector::OTHER, + std::string error_message = "Import " + (*it)->name() + " is unused."; + if (is_error) { + AddError((*it)->name(), proto, DescriptorPool::ErrorCollector::IMPORT, + error_message); + } else { + AddWarning((*it)->name(), proto, DescriptorPool::ErrorCollector::IMPORT, error_message); } } } } -Symbol DescriptorPool::CrossLinkOnDemandHelper(const string& name, +Symbol DescriptorPool::CrossLinkOnDemandHelper(StringPiece name, bool expecting_enum) const { - string lookup_name = name; + (void)expecting_enum; // Parameter is used by Google-internal code. + auto lookup_name = std::string(name); if (!lookup_name.empty() && lookup_name[0] == '.') { lookup_name = lookup_name.substr(1); } @@ -6996,39 +7879,39 @@ Symbol DescriptorPool::CrossLinkOnDemandHelper(const string& name, // enum_type_, message_type_, and default_value_enum_ appropriately. void FieldDescriptor::InternalTypeOnceInit() const { GOOGLE_CHECK(file()->finished_building_ == true); - if (type_name_) { - Symbol result = file()->pool()->CrossLinkOnDemandHelper( - *type_name_, type_ == FieldDescriptor::TYPE_ENUM); - if (result.type == Symbol::MESSAGE) { - type_ = FieldDescriptor::TYPE_MESSAGE; - message_type_ = result.descriptor; - } else if (result.type == Symbol::ENUM) { - type_ = FieldDescriptor::TYPE_ENUM; - enum_type_ = result.enum_descriptor; - } + const EnumDescriptor* enum_type = nullptr; + Symbol result = file()->pool()->CrossLinkOnDemandHelper( + type_descriptor_.lazy_type_name, type_ == FieldDescriptor::TYPE_ENUM); + if (result.type() == Symbol::MESSAGE) { + type_ = FieldDescriptor::TYPE_MESSAGE; + type_descriptor_.message_type = result.descriptor(); + } else if (result.type() == Symbol::ENUM) { + type_ = FieldDescriptor::TYPE_ENUM; + enum_type = type_descriptor_.enum_type = result.enum_descriptor(); } - if (enum_type_ && !default_value_enum_) { - if (default_value_enum_name_) { + + if (enum_type) { + if (lazy_default_value_enum_name_) { // Have to build the full name now instead of at CrossLink time, - // because enum_type_ may not be known at the time. - string name = enum_type_->full_name(); + // because enum_type may not be known at the time. + std::string name = enum_type->full_name(); // Enum values reside in the same scope as the enum type. - string::size_type last_dot = name.find_last_of('.'); - if (last_dot != string::npos) { - name = name.substr(0, last_dot) + "." + *default_value_enum_name_; + std::string::size_type last_dot = name.find_last_of('.'); + if (last_dot != std::string::npos) { + name = name.substr(0, last_dot) + "." + lazy_default_value_enum_name_; } else { - name = *default_value_enum_name_; + name = lazy_default_value_enum_name_; } Symbol result = file()->pool()->CrossLinkOnDemandHelper(name, true); - if (result.type == Symbol::ENUM_VALUE) { - default_value_enum_ = result.enum_value_descriptor; - } + default_value_enum_ = result.enum_value_descriptor(); + } else { + default_value_enum_ = nullptr; } if (!default_value_enum_) { // We use the first defined value as the default // if a default is not explicitly defined. - GOOGLE_CHECK(enum_type_->value_count()); - default_value_enum_ = enum_type_->value(0); + GOOGLE_CHECK(enum_type->value_count()); + default_value_enum_ = enum_type->value(0); } } } @@ -7038,34 +7921,46 @@ void FieldDescriptor::TypeOnceInit(const FieldDescriptor* to_init) { } // message_type(), enum_type(), default_value_enum(), and type() -// all share the same GoogleOnceDynamic init path to do lazy +// all share the same internal::call_once init path to do lazy // import building and cross linking of a field of a message. const Descriptor* FieldDescriptor::message_type() const { if (type_once_) { - type_once_->Init(&FieldDescriptor::TypeOnceInit, this); + internal::call_once(*type_once_, FieldDescriptor::TypeOnceInit, this); } - return message_type_; + return type_ == TYPE_MESSAGE || type_ == TYPE_GROUP + ? type_descriptor_.message_type + : nullptr; } const EnumDescriptor* FieldDescriptor::enum_type() const { if (type_once_) { - type_once_->Init(&FieldDescriptor::TypeOnceInit, this); + internal::call_once(*type_once_, FieldDescriptor::TypeOnceInit, this); } - return enum_type_; + return type_ == TYPE_ENUM ? type_descriptor_.enum_type : nullptr; } const EnumValueDescriptor* FieldDescriptor::default_value_enum() const { if (type_once_) { - type_once_->Init(&FieldDescriptor::TypeOnceInit, this); + internal::call_once(*type_once_, FieldDescriptor::TypeOnceInit, this); } return default_value_enum_; } +const std::string& FieldDescriptor::PrintableNameForExtension() const { + const bool is_message_set_extension = + is_extension() && + containing_type()->options().message_set_wire_format() && + type() == FieldDescriptor::TYPE_MESSAGE && is_optional() && + extension_scope() == message_type(); + return is_message_set_extension ? message_type()->full_name() : full_name(); +} + void FileDescriptor::InternalDependenciesOnceInit() const { GOOGLE_CHECK(finished_building_ == true); + auto* names = dependencies_once_->dependencies_names; for (int i = 0; i < dependency_count(); i++) { - if (dependencies_names_[i]) { - dependencies_[i] = pool_->FindFileByName(*dependencies_names_[i]); + if (names[i]) { + dependencies_[i] = pool_->FindFileByName(names[i]); } } } @@ -7076,62 +7971,55 @@ void FileDescriptor::DependenciesOnceInit(const FileDescriptor* to_init) { const FileDescriptor* FileDescriptor::dependency(int index) const { if (dependencies_once_) { - // Do once init for all indicies, as it's unlikely only a single index would - // be called, and saves on GoogleOnceDynamic allocations. - dependencies_once_->Init(&FileDescriptor::DependenciesOnceInit, this); + // Do once init for all indices, as it's unlikely only a single index would + // be called, and saves on internal::call_once allocations. + internal::call_once(dependencies_once_->once, + FileDescriptor::DependenciesOnceInit, this); } return dependencies_[index]; } const Descriptor* MethodDescriptor::input_type() const { - return input_type_.Get(); + return input_type_.Get(service()); } const Descriptor* MethodDescriptor::output_type() const { - return output_type_.Get(); + return output_type_.Get(service()); } namespace internal { void LazyDescriptor::Set(const Descriptor* descriptor) { - GOOGLE_CHECK(!name_); GOOGLE_CHECK(!once_); - GOOGLE_CHECK(!file_); descriptor_ = descriptor; } -void LazyDescriptor::SetLazy(const string& name, const FileDescriptor* file) { +void LazyDescriptor::SetLazy(StringPiece name, + const FileDescriptor* file) { // verify Init() has been called and Set hasn't been called yet. GOOGLE_CHECK(!descriptor_); - GOOGLE_CHECK(!file_); - GOOGLE_CHECK(!name_); GOOGLE_CHECK(!once_); GOOGLE_CHECK(file && file->pool_); GOOGLE_CHECK(file->pool_->lazily_build_dependencies_); GOOGLE_CHECK(!file->finished_building_); - file_ = file; - name_ = file->pool_->tables_->AllocateString(name); - once_ = file->pool_->tables_->AllocateOnceDynamic(); + once_ = file->pool_->tables_->Create(); + lazy_name_ = file->pool_->tables_->Strdup(name); } -void LazyDescriptor::Once() { +void LazyDescriptor::Once(const ServiceDescriptor* service) { if (once_) { - once_->Init(&LazyDescriptor::OnceStatic, this); + internal::call_once(*once_, [&] { + auto* file = service->file(); + GOOGLE_CHECK(file->finished_building_); + descriptor_ = + file->pool_->CrossLinkOnDemandHelper(lazy_name_, false).descriptor(); + }); } } -void LazyDescriptor::OnceStatic(LazyDescriptor* lazy) { lazy->OnceInternal(); } - -void LazyDescriptor::OnceInternal() { - GOOGLE_CHECK(file_->finished_building_); - if (!descriptor_ && name_) { - Symbol result = file_->pool_->CrossLinkOnDemandHelper(*name_, false); - if (!result.IsNull() && result.type == Symbol::MESSAGE) { - descriptor_ = result.descriptor; - } - } -} } // namespace internal } // namespace protobuf } // namespace google + +#include diff --git a/3rdparty/protobuf/src/google/protobuf/descriptor.h b/3rdparty/protobuf/src/google/protobuf/descriptor.h index 5f5159a8ad..e74e355b5a 100644 --- a/3rdparty/protobuf/src/google/protobuf/descriptor.h +++ b/3rdparty/protobuf/src/google/protobuf/descriptor.h @@ -54,22 +54,30 @@ #ifndef GOOGLE_PROTOBUF_DESCRIPTOR_H__ #define GOOGLE_PROTOBUF_DESCRIPTOR_H__ +#include +#include #include -#ifndef _SHARED_PTR_H -#include -#endif #include #include #include + #include +#include #include #include +#include +#include // TYPE_BOOL is defined in the MacOS's ConditionalMacros.h. #ifdef TYPE_BOOL #undef TYPE_BOOL #endif // TYPE_BOOL +#ifdef SWIG +#define PROTOBUF_EXPORT +#endif + + namespace google { namespace protobuf { @@ -87,6 +95,7 @@ class DescriptorPool; // Defined in descriptor.proto class DescriptorProto; +class DescriptorProto_ExtensionRange; class FieldDescriptorProto; class OneofDescriptorProto; class EnumDescriptorProto; @@ -108,23 +117,23 @@ class SourceCodeInfo; // Defined in message.h class Message; +class Reflection; // Defined in descriptor.cc class DescriptorBuilder; class FileDescriptorTables; -struct Symbol; +class Symbol; // Defined in unknown_field_set.h. class UnknownField; -// Defined in generated_message_reflection.h. -namespace internal { -class GeneratedMessageReflection; -} // namespace internal - // Defined in command_line_interface.cc namespace compiler { class CommandLineInterface; +namespace cpp { +// Defined in helpers.h +class Formatter; +} // namespace cpp } // namespace compiler namespace descriptor_unittest { @@ -145,9 +154,9 @@ struct SourceLocation { // Doc comments found at the source location. // See the comments in SourceCodeInfo.Location (descriptor.proto) for details. - string leading_comments; - string trailing_comments; - std::vector leading_detached_comments; + std::string leading_comments; + std::string trailing_comments; + std::vector leading_detached_comments; }; // Options when generating machine-parsable output from a descriptor with @@ -165,7 +174,8 @@ struct DebugStringOptions { DebugStringOptions() : include_comments(false), elide_group_body(false), - elide_oneof_body(false) {} + elide_oneof_body(false) { + } }; // A class to handle the simplest cases of a lazily linked descriptor @@ -173,15 +183,14 @@ struct DebugStringOptions { // which is needed when a pool has lazily_build_dependencies_ set. // Must be instantiated as mutable in a descriptor. namespace internal { -class LIBPROTOBUF_EXPORT LazyDescriptor { + +class PROTOBUF_EXPORT LazyDescriptor { public: // Init function to be called at init time of a descriptor containing // a LazyDescriptor. void Init() { - descriptor_ = NULL; - name_ = NULL; - once_ = NULL; - file_ = NULL; + descriptor_ = nullptr; + once_ = nullptr; } // Sets the value of the descriptor if it is known during the descriptor @@ -195,26 +204,39 @@ class LIBPROTOBUF_EXPORT LazyDescriptor { // build time if the symbol wasn't found and building of the file containing // that type is delayed because lazily_build_dependencies_ is set on the pool. // Should not be called after Set() has been called. - void SetLazy(const string& name, const FileDescriptor* file); + void SetLazy(StringPiece name, const FileDescriptor* file); // Returns the current value of the descriptor, thread-safe. If SetLazy(...) // has been called, will do a one-time cross link of the type specified, // building the descriptor file that contains the type if necessary. - inline const Descriptor* Get() { - Once(); + inline const Descriptor* Get(const ServiceDescriptor* service) { + Once(service); return descriptor_; } private: - static void OnceStatic(LazyDescriptor* lazy); - void OnceInternal(); - void Once(); + void Once(const ServiceDescriptor* service); - const Descriptor* descriptor_; - const string* name_; - GoogleOnceDynamic* once_; - const FileDescriptor* file_; + union { + const Descriptor* descriptor_; + const char* lazy_name_; + }; + internal::once_flag* once_; }; + +class PROTOBUF_EXPORT SymbolBase { + private: + friend class google::protobuf::Symbol; + uint8_t symbol_type_; +}; + +// Some types have more than one SymbolBase because they have multiple +// identities in the table. We can't have duplicate direct bases, so we use this +// intermediate base to do so. +// See BuildEnumValue for details. +template +class PROTOBUF_EXPORT SymbolBaseN : public SymbolBase {}; + } // namespace internal // Describes a type of protocol message, or a particular group within a @@ -222,34 +244,35 @@ class LIBPROTOBUF_EXPORT LazyDescriptor { // Message::GetDescriptor(). Generated message classes also have a // static method called descriptor() which returns the type's descriptor. // Use DescriptorPool to construct your own descriptors. -class LIBPROTOBUF_EXPORT Descriptor { +class PROTOBUF_EXPORT Descriptor : private internal::SymbolBase { public: + typedef DescriptorProto Proto; + // The name of the message type, not including its scope. - const string& name() const; + const std::string& name() const; // The fully-qualified name of the message type, scope delimited by // periods. For example, message type "Foo" which is declared in package // "bar" has full name "bar.Foo". If a type "Baz" is nested within // Foo, Baz's full_name is "bar.Foo.Baz". To get only the part that // comes after the last '.', use name(). - const string& full_name() const; + const std::string& full_name() const; // Index of this descriptor within the file or containing type's message // type array. int index() const; - // The .proto file in which this message type was defined. Never NULL. + // The .proto file in which this message type was defined. Never nullptr. const FileDescriptor* file() const; // If this Descriptor describes a nested type, this returns the type - // in which it is nested. Otherwise, returns NULL. + // in which it is nested. Otherwise, returns nullptr. const Descriptor* containing_type() const; // Get options for this message type. These are specified in the .proto file // by placing lines like "option foo = 1234;" in the message definition. - // Allowed options are defined by MessageOptions in - // google/protobuf/descriptor.proto, and any available extensions of that - // message. + // Allowed options are defined by MessageOptions in descriptor.proto, and any + // available extensions of that message. const MessageOptions& options() const; // Write the contents of this Descriptor into the given DescriptorProto. @@ -257,19 +280,49 @@ class LIBPROTOBUF_EXPORT Descriptor { // isn't, the result may be garbage. void CopyTo(DescriptorProto* proto) const; - // Write the contents of this decriptor in a human-readable form. Output + // Write the contents of this descriptor in a human-readable form. Output // will be suitable for re-parsing. - string DebugString() const; + std::string DebugString() const; // Similar to DebugString(), but additionally takes options (e.g., // include original user comments in output). - string DebugStringWithOptions(const DebugStringOptions& options) const; + std::string DebugStringWithOptions(const DebugStringOptions& options) const; // Returns true if this is a placeholder for an unknown type. This will // only be the case if this descriptor comes from a DescriptorPool // with AllowUnknownDependencies() set. bool is_placeholder() const; + enum WellKnownType { + WELLKNOWNTYPE_UNSPECIFIED, // Not a well-known type. + + // Wrapper types. + WELLKNOWNTYPE_DOUBLEVALUE, // google.protobuf.DoubleValue + WELLKNOWNTYPE_FLOATVALUE, // google.protobuf.FloatValue + WELLKNOWNTYPE_INT64VALUE, // google.protobuf.Int64Value + WELLKNOWNTYPE_UINT64VALUE, // google.protobuf.UInt64Value + WELLKNOWNTYPE_INT32VALUE, // google.protobuf.Int32Value + WELLKNOWNTYPE_UINT32VALUE, // google.protobuf.UInt32Value + WELLKNOWNTYPE_STRINGVALUE, // google.protobuf.StringValue + WELLKNOWNTYPE_BYTESVALUE, // google.protobuf.BytesValue + WELLKNOWNTYPE_BOOLVALUE, // google.protobuf.BoolValue + + // Other well known types. + WELLKNOWNTYPE_ANY, // google.protobuf.Any + WELLKNOWNTYPE_FIELDMASK, // google.protobuf.FieldMask + WELLKNOWNTYPE_DURATION, // google.protobuf.Duration + WELLKNOWNTYPE_TIMESTAMP, // google.protobuf.Timestamp + WELLKNOWNTYPE_VALUE, // google.protobuf.Value + WELLKNOWNTYPE_LISTVALUE, // google.protobuf.ListValue + WELLKNOWNTYPE_STRUCT, // google.protobuf.Struct + + // New well-known types may be added in the future. + // Please make sure any switch() statements have a 'default' case. + __WELLKNOWNTYPE__DO_NOT_USE__ADD_DEFAULT_INSTEAD__, + }; + + WellKnownType well_known_type() const; + // Field stuff ----------------------------------------------------- // The number of fields in this message type. @@ -278,33 +331,37 @@ class LIBPROTOBUF_EXPORT Descriptor { // These are returned in the order they were defined in the .proto file. const FieldDescriptor* field(int index) const; - // Looks up a field by declared tag number. Returns NULL if no such field + // Looks up a field by declared tag number. Returns nullptr if no such field // exists. const FieldDescriptor* FindFieldByNumber(int number) const; - // Looks up a field by name. Returns NULL if no such field exists. - const FieldDescriptor* FindFieldByName(const string& name) const; + // Looks up a field by name. Returns nullptr if no such field exists. + const FieldDescriptor* FindFieldByName(ConstStringParam name) const; // Looks up a field by lowercased name (as returned by lowercase_name()). // This lookup may be ambiguous if multiple field names differ only by case, // in which case the field returned is chosen arbitrarily from the matches. const FieldDescriptor* FindFieldByLowercaseName( - const string& lowercase_name) const; + ConstStringParam lowercase_name) const; // Looks up a field by camel-case name (as returned by camelcase_name()). // This lookup may be ambiguous if multiple field names differ in a way that // leads them to have identical camel-case names, in which case the field // returned is chosen arbitrarily from the matches. const FieldDescriptor* FindFieldByCamelcaseName( - const string& camelcase_name) const; + ConstStringParam camelcase_name) const; // The number of oneofs in this message type. int oneof_decl_count() const; + // The number of oneofs in this message type, excluding synthetic oneofs. + // Real oneofs always come first, so iterating up to real_oneof_decl_cout() + // will yield all real oneofs. + int real_oneof_decl_count() const; // Get a oneof by index, where 0 <= index < oneof_decl_count(). // These are returned in the order they were defined in the .proto file. const OneofDescriptor* oneof_decl(int index) const; - // Looks up a oneof by name. Returns NULL if no such oneof exists. - const OneofDescriptor* FindOneofByName(const string& name) const; + // Looks up a oneof by name. Returns nullptr if no such oneof exists. + const OneofDescriptor* FindOneofByName(ConstStringParam name) const; // Nested type stuff ----------------------------------------------- @@ -314,9 +371,9 @@ class LIBPROTOBUF_EXPORT Descriptor { // These are returned in the order they were defined in the .proto file. const Descriptor* nested_type(int index) const; - // Looks up a nested type by name. Returns NULL if no such nested type + // Looks up a nested type by name. Returns nullptr if no such nested type // exists. - const Descriptor* FindNestedTypeByName(const string& name) const; + const Descriptor* FindNestedTypeByName(ConstStringParam name) const; // Enum stuff ------------------------------------------------------ @@ -326,20 +383,26 @@ class LIBPROTOBUF_EXPORT Descriptor { // These are returned in the order they were defined in the .proto file. const EnumDescriptor* enum_type(int index) const; - // Looks up an enum type by name. Returns NULL if no such enum type exists. - const EnumDescriptor* FindEnumTypeByName(const string& name) const; + // Looks up an enum type by name. Returns nullptr if no such enum type + // exists. + const EnumDescriptor* FindEnumTypeByName(ConstStringParam name) const; // Looks up an enum value by name, among all enum types in this message. - // Returns NULL if no such value exists. - const EnumValueDescriptor* FindEnumValueByName(const string& name) const; + // Returns nullptr if no such value exists. + const EnumValueDescriptor* FindEnumValueByName(ConstStringParam name) const; // Extensions ------------------------------------------------------ // A range of field numbers which are designated for third-party // extensions. struct ExtensionRange { + typedef DescriptorProto_ExtensionRange Proto; + typedef ExtensionRangeOptions OptionsType; + // See Descriptor::CopyTo(). + void CopyTo(DescriptorProto_ExtensionRange* proto) const; + int start; // inclusive int end; // exclusive @@ -356,11 +419,33 @@ class LIBPROTOBUF_EXPORT Descriptor { // Returns true if the number is in one of the extension ranges. bool IsExtensionNumber(int number) const; - // Returns NULL if no extension range contains the given number. + // Returns nullptr if no extension range contains the given number. const ExtensionRange* FindExtensionRangeContainingNumber(int number) const; - // The number of extensions -- extending *other* messages -- that were - // defined nested within this message type's scope. + // The number of extensions defined nested within this message type's scope. + // See doc: + // https://developers.google.com/protocol-buffers/docs/proto#nested-extensions + // + // Note that the extensions may be extending *other* messages. + // + // For example: + // message M1 { + // extensions 1 to max; + // } + // + // message M2 { + // extend M1 { + // optional int32 foo = 1; + // } + // } + // + // In this case, + // DescriptorPool::generated_pool() + // ->FindMessageTypeByName("M2") + // ->extension(0) + // will return "foo", even though "foo" is an extension of M1. + // To find all known extensions of a given message, instead use + // DescriptorPool::FindAllExtensions. int extension_count() const; // Get an extension by index, where 0 <= index < extension_count(). // These are returned in the order they were defined in the .proto file. @@ -368,15 +453,17 @@ class LIBPROTOBUF_EXPORT Descriptor { // Looks up a named extension (which extends some *other* message type) // defined within this message type's scope. - const FieldDescriptor* FindExtensionByName(const string& name) const; + const FieldDescriptor* FindExtensionByName(ConstStringParam name) const; // Similar to FindFieldByLowercaseName(), but finds extensions defined within // this message type's scope. - const FieldDescriptor* FindExtensionByLowercaseName(const string& name) const; + const FieldDescriptor* FindExtensionByLowercaseName( + ConstStringParam name) const; // Similar to FindFieldByCamelcaseName(), but finds extensions defined within // this message type's scope. - const FieldDescriptor* FindExtensionByCamelcaseName(const string& name) const; + const FieldDescriptor* FindExtensionByCamelcaseName( + ConstStringParam name) const; // Reserved fields ------------------------------------------------- @@ -396,17 +483,17 @@ class LIBPROTOBUF_EXPORT Descriptor { // Returns true if the number is in one of the reserved ranges. bool IsReservedNumber(int number) const; - // Returns NULL if no reserved range contains the given number. + // Returns nullptr if no reserved range contains the given number. const ReservedRange* FindReservedRangeContainingNumber(int number) const; // The number of reserved field names in this message type. int reserved_name_count() const; // Gets a reserved name by index, where 0 <= index < reserved_name_count(). - const string& reserved_name(int index) const; + const std::string& reserved_name(int index) const; // Returns true if the field name is reserved. - bool IsReservedName(const string& name) const; + bool IsReservedName(ConstStringParam name) const; // Source Location --------------------------------------------------- @@ -415,14 +502,26 @@ class LIBPROTOBUF_EXPORT Descriptor { // |*out_location| unchanged iff location information was not available. bool GetSourceLocation(SourceLocation* out_location) const; + // Maps -------------------------------------------------------------- + + // Returns the FieldDescriptor for the "key" field. If this isn't a map entry + // field, returns nullptr. + const FieldDescriptor* map_key() const; + + // Returns the FieldDescriptor for the "value" field. If this isn't a map + // entry field, returns nullptr. + const FieldDescriptor* map_value() const; + private: + friend class Symbol; typedef MessageOptions OptionsType; // Allows tests to test CopyTo(proto, true). - friend class ::google::protobuf::descriptor_unittest::DescriptorTest; + friend class descriptor_unittest::DescriptorTest; // Allows access to GetLocationPath for annotations. - friend class ::google::protobuf::io::Printer; + friend class io::Printer; + friend class compiler::cpp::Formatter; // Fill the json_name field of FieldDescriptorProto. void CopyJsonNameTo(DescriptorProto* proto) const; @@ -431,7 +530,7 @@ class LIBPROTOBUF_EXPORT Descriptor { // correct depth. Takes |options| to control debug-string options, and // |include_opening_clause| to indicate whether the "message ... " part of the // clause has already been generated (this varies depending on context). - void DebugString(int depth, string *contents, + void DebugString(int depth, std::string* contents, const DebugStringOptions& options, bool include_opening_clause) const; @@ -439,8 +538,26 @@ class LIBPROTOBUF_EXPORT Descriptor { // to this descriptor from the file root. void GetLocationPath(std::vector* output) const; - const string* name_; - const string* full_name_; + // True if this is a placeholder for an unknown type. + bool is_placeholder_ : 1; + // True if this is a placeholder and the type name wasn't fully-qualified. + bool is_unqualified_placeholder_ : 1; + // Well known type. Stored like this to conserve space. + uint8_t well_known_type_ : 5; + + // This points to the last field _number_ that is part of the sequence + // starting at 1, where + // `desc->field(i)->number() == i + 1` + // A value of `0` means no field matches. That is, there are no fields or the + // first field is not field `1`. + // Uses 16-bit to avoid extra padding. Unlikely to have more than 2^16 + // sequentially numbered fields in a message. + uint16_t sequential_field_limit_; + + int field_count_; + + // all_names_ = [name, full_name] + const std::string* all_names_; const FileDescriptor* file_; const Descriptor* containing_type_; const MessageOptions* options_; @@ -453,10 +570,10 @@ class LIBPROTOBUF_EXPORT Descriptor { ExtensionRange* extension_ranges_; FieldDescriptor* extensions_; ReservedRange* reserved_ranges_; - const string** reserved_names_; + const std::string** reserved_names_; - int field_count_; int oneof_decl_count_; + int real_oneof_decl_count_; int nested_type_count_; int enum_type_count_; int extension_range_count_; @@ -464,11 +581,6 @@ class LIBPROTOBUF_EXPORT Descriptor { int reserved_range_count_; int reserved_name_count_; - // True if this is a placeholder for an unknown type. - bool is_placeholder_; - // True if this is a placeholder and the type name wasn't fully-qualified. - bool is_unqualified_placeholder_; - // IMPORTANT: If you add a new field, make sure to search for all instances // of Allocate() and AllocateArray() in descriptor.cc // and update them to initialize the field. @@ -479,6 +591,7 @@ class LIBPROTOBUF_EXPORT Descriptor { friend class DescriptorPool; friend class EnumDescriptor; friend class FieldDescriptor; + friend class FileDescriptorTables; friend class OneofDescriptor; friend class MethodDescriptor; friend class FileDescriptor; @@ -493,72 +606,72 @@ class LIBPROTOBUF_EXPORT Descriptor { // - Get the Descriptor or FileDescriptor for its containing scope, then // call Descriptor::FindExtensionByName() or // FileDescriptor::FindExtensionByName(). -// - Given a DescriptorPool, call DescriptorPool::FindExtensionByNumber(). -// - Given a Reflection for a message object, call -// Reflection::FindKnownExtensionByName() or -// Reflection::FindKnownExtensionByNumber(). +// - Given a DescriptorPool, call DescriptorPool::FindExtensionByNumber() or +// DescriptorPool::FindExtensionByPrintableName(). // Use DescriptorPool to construct your own descriptors. -class LIBPROTOBUF_EXPORT FieldDescriptor { +class PROTOBUF_EXPORT FieldDescriptor : private internal::SymbolBase { public: + typedef FieldDescriptorProto Proto; + // Identifies a field type. 0 is reserved for errors. The order is weird // for historical reasons. Types 12 and up are new in proto2. enum Type { - TYPE_DOUBLE = 1, // double, exactly eight bytes on the wire. - TYPE_FLOAT = 2, // float, exactly four bytes on the wire. - TYPE_INT64 = 3, // int64, varint on the wire. Negative numbers - // take 10 bytes. Use TYPE_SINT64 if negative - // values are likely. - TYPE_UINT64 = 4, // uint64, varint on the wire. - TYPE_INT32 = 5, // int32, varint on the wire. Negative numbers - // take 10 bytes. Use TYPE_SINT32 if negative - // values are likely. - TYPE_FIXED64 = 6, // uint64, exactly eight bytes on the wire. - TYPE_FIXED32 = 7, // uint32, exactly four bytes on the wire. - TYPE_BOOL = 8, // bool, varint on the wire. - TYPE_STRING = 9, // UTF-8 text. - TYPE_GROUP = 10, // Tag-delimited message. Deprecated. - TYPE_MESSAGE = 11, // Length-delimited message. + TYPE_DOUBLE = 1, // double, exactly eight bytes on the wire. + TYPE_FLOAT = 2, // float, exactly four bytes on the wire. + TYPE_INT64 = 3, // int64, varint on the wire. Negative numbers + // take 10 bytes. Use TYPE_SINT64 if negative + // values are likely. + TYPE_UINT64 = 4, // uint64, varint on the wire. + TYPE_INT32 = 5, // int32, varint on the wire. Negative numbers + // take 10 bytes. Use TYPE_SINT32 if negative + // values are likely. + TYPE_FIXED64 = 6, // uint64, exactly eight bytes on the wire. + TYPE_FIXED32 = 7, // uint32, exactly four bytes on the wire. + TYPE_BOOL = 8, // bool, varint on the wire. + TYPE_STRING = 9, // UTF-8 text. + TYPE_GROUP = 10, // Tag-delimited message. Deprecated. + TYPE_MESSAGE = 11, // Length-delimited message. - TYPE_BYTES = 12, // Arbitrary byte array. - TYPE_UINT32 = 13, // uint32, varint on the wire - TYPE_ENUM = 14, // Enum, varint on the wire - TYPE_SFIXED32 = 15, // int32, exactly four bytes on the wire - TYPE_SFIXED64 = 16, // int64, exactly eight bytes on the wire - TYPE_SINT32 = 17, // int32, ZigZag-encoded varint on the wire - TYPE_SINT64 = 18, // int64, ZigZag-encoded varint on the wire + TYPE_BYTES = 12, // Arbitrary byte array. + TYPE_UINT32 = 13, // uint32, varint on the wire + TYPE_ENUM = 14, // Enum, varint on the wire + TYPE_SFIXED32 = 15, // int32, exactly four bytes on the wire + TYPE_SFIXED64 = 16, // int64, exactly eight bytes on the wire + TYPE_SINT32 = 17, // int32, ZigZag-encoded varint on the wire + TYPE_SINT64 = 18, // int64, ZigZag-encoded varint on the wire - MAX_TYPE = 18, // Constant useful for defining lookup tables - // indexed by Type. + MAX_TYPE = 18, // Constant useful for defining lookup tables + // indexed by Type. }; // Specifies the C++ data type used to represent the field. There is a // fixed mapping from Type to CppType where each Type maps to exactly one // CppType. 0 is reserved for errors. enum CppType { - CPPTYPE_INT32 = 1, // TYPE_INT32, TYPE_SINT32, TYPE_SFIXED32 - CPPTYPE_INT64 = 2, // TYPE_INT64, TYPE_SINT64, TYPE_SFIXED64 - CPPTYPE_UINT32 = 3, // TYPE_UINT32, TYPE_FIXED32 - CPPTYPE_UINT64 = 4, // TYPE_UINT64, TYPE_FIXED64 - CPPTYPE_DOUBLE = 5, // TYPE_DOUBLE - CPPTYPE_FLOAT = 6, // TYPE_FLOAT - CPPTYPE_BOOL = 7, // TYPE_BOOL - CPPTYPE_ENUM = 8, // TYPE_ENUM - CPPTYPE_STRING = 9, // TYPE_STRING, TYPE_BYTES - CPPTYPE_MESSAGE = 10, // TYPE_MESSAGE, TYPE_GROUP + CPPTYPE_INT32 = 1, // TYPE_INT32, TYPE_SINT32, TYPE_SFIXED32 + CPPTYPE_INT64 = 2, // TYPE_INT64, TYPE_SINT64, TYPE_SFIXED64 + CPPTYPE_UINT32 = 3, // TYPE_UINT32, TYPE_FIXED32 + CPPTYPE_UINT64 = 4, // TYPE_UINT64, TYPE_FIXED64 + CPPTYPE_DOUBLE = 5, // TYPE_DOUBLE + CPPTYPE_FLOAT = 6, // TYPE_FLOAT + CPPTYPE_BOOL = 7, // TYPE_BOOL + CPPTYPE_ENUM = 8, // TYPE_ENUM + CPPTYPE_STRING = 9, // TYPE_STRING, TYPE_BYTES + CPPTYPE_MESSAGE = 10, // TYPE_MESSAGE, TYPE_GROUP - MAX_CPPTYPE = 10, // Constant useful for defining lookup tables - // indexed by CppType. + MAX_CPPTYPE = 10, // Constant useful for defining lookup tables + // indexed by CppType. }; // Identifies whether the field is optional, required, or repeated. 0 is // reserved for errors. enum Label { - LABEL_OPTIONAL = 1, // optional - LABEL_REQUIRED = 2, // required - LABEL_REPEATED = 3, // repeated + LABEL_OPTIONAL = 1, // optional + LABEL_REQUIRED = 2, // required + LABEL_REPEATED = 3, // repeated - MAX_LABEL = 3, // Constant useful for defining lookup tables - // indexed by Label. + MAX_LABEL = 3, // Constant useful for defining lookup tables + // indexed by Label. }; // Valid field numbers are positive integers up to kMaxNumber. @@ -569,14 +682,14 @@ class LIBPROTOBUF_EXPORT FieldDescriptor { static const int kFirstReservedNumber = 19000; // Last field number reserved for the protocol buffer library implementation. // Users may not declare fields that use reserved numbers. - static const int kLastReservedNumber = 19999; + static const int kLastReservedNumber = 19999; - const string& name() const; // Name of this field within the message. - const string& full_name() const; // Fully-qualified name of the field. - const string& json_name() const; // JSON name of this field. - const FileDescriptor* file() const;// File in which this field was defined. - bool is_extension() const; // Is this an extension field? - int number() const; // Declared tag number. + const std::string& name() const; // Name of this field within the message. + const std::string& full_name() const; // Fully-qualified name of the field. + const std::string& json_name() const; // JSON name of this field. + const FileDescriptor* file() const; // File in which this field was defined. + bool is_extension() const; // Is this an extension field? + int number() const; // Declared tag number. // Same as name() except converted to lower-case. This (and especially the // FindFieldByLowercaseName() method) can be useful when parsing formats @@ -584,7 +697,7 @@ class LIBPROTOBUF_EXPORT FieldDescriptor { // field names should be lowercased anyway according to the protobuf style // guide, so this only makes a difference when dealing with old .proto files // which do not follow the guide.) - const string& lowercase_name() const; + const std::string& lowercase_name() const; // Same as name() except converted to camel-case. In this conversion, any // time an underscore appears in the name, it is removed and the next @@ -595,7 +708,7 @@ class LIBPROTOBUF_EXPORT FieldDescriptor { // fooBar -> fooBar // This (and especially the FindFieldByCamelcaseName() method) can be useful // when parsing formats which prefer to use camel-case naming style. - const string& camelcase_name() const; + const std::string& camelcase_name() const; Type type() const; // Declared type of this field. const char* type_name() const; // Name of the declared type. @@ -603,15 +716,28 @@ class LIBPROTOBUF_EXPORT FieldDescriptor { const char* cpp_type_name() const; // Name of the C++ type. Label label() const; // optional/required/repeated - bool is_required() const; // shorthand for label() == LABEL_REQUIRED - bool is_optional() const; // shorthand for label() == LABEL_OPTIONAL - bool is_repeated() const; // shorthand for label() == LABEL_REPEATED - bool is_packable() const; // shorthand for is_repeated() && - // IsTypePackable(type()) - bool is_packed() const; // shorthand for is_packable() && - // options().packed() - bool is_map() const; // shorthand for type() == TYPE_MESSAGE && - // message_type()->options().map_entry() + bool is_required() const; // shorthand for label() == LABEL_REQUIRED + bool is_optional() const; // shorthand for label() == LABEL_OPTIONAL + bool is_repeated() const; // shorthand for label() == LABEL_REPEATED + bool is_packable() const; // shorthand for is_repeated() && + // IsTypePackable(type()) + bool is_packed() const; // shorthand for is_packable() && + // options().packed() + bool is_map() const; // shorthand for type() == TYPE_MESSAGE && + // message_type()->options().map_entry() + + // Returns true if this field was syntactically written with "optional" in the + // .proto file. Excludes singular proto3 fields that do not have a label. + bool has_optional_keyword() const; + + // Returns true if this field tracks presence, ie. does the field + // distinguish between "unset" and "present with default value." + // This includes required, optional, and oneof fields. It excludes maps, + // repeated fields, and singular proto3 fields without "optional". + // + // For fields where has_presence() == true, the return value of + // Reflection::HasField() is semantically meaningful. + bool has_presence() const; // Index of this field within the message's field array, or the file or // extension scope's extensions array. @@ -626,16 +752,20 @@ class LIBPROTOBUF_EXPORT FieldDescriptor { // Get the field default value if cpp_type() == CPPTYPE_INT32. If no // explicit default was defined, the default is 0. - int32 default_value_int32() const; + int32_t default_value_int32_t() const; + int32_t default_value_int32() const { return default_value_int32_t(); } // Get the field default value if cpp_type() == CPPTYPE_INT64. If no // explicit default was defined, the default is 0. - int64 default_value_int64() const; + int64_t default_value_int64_t() const; + int64_t default_value_int64() const { return default_value_int64_t(); } // Get the field default value if cpp_type() == CPPTYPE_UINT32. If no // explicit default was defined, the default is 0. - uint32 default_value_uint32() const; + uint32_t default_value_uint32_t() const; + uint32_t default_value_uint32() const { return default_value_uint32_t(); } // Get the field default value if cpp_type() == CPPTYPE_UINT64. If no // explicit default was defined, the default is 0. - uint64 default_value_uint64() const; + uint64_t default_value_uint64_t() const; + uint64_t default_value_uint64() const { return default_value_uint64_t(); } // Get the field default value if cpp_type() == CPPTYPE_FLOAT. If no // explicit default was defined, the default is 0.0. float default_value_float() const; @@ -648,26 +778,30 @@ class LIBPROTOBUF_EXPORT FieldDescriptor { // Get the field default value if cpp_type() == CPPTYPE_ENUM. If no // explicit default was defined, the default is the first value defined // in the enum type (all enum types are required to have at least one value). - // This never returns NULL. + // This never returns nullptr. const EnumValueDescriptor* default_value_enum() const; // Get the field default value if cpp_type() == CPPTYPE_STRING. If no // explicit default was defined, the default is the empty string. - const string& default_value_string() const; + const std::string& default_value_string() const; // The Descriptor for the message of which this is a field. For extensions, - // this is the extended type. Never NULL. + // this is the extended type. Never nullptr. const Descriptor* containing_type() const; // If the field is a member of a oneof, this is the one, otherwise this is - // NULL. + // nullptr. const OneofDescriptor* containing_oneof() const; + // If the field is a member of a non-synthetic oneof, returns the descriptor + // for the oneof, otherwise returns nullptr. + const OneofDescriptor* real_containing_oneof() const; + // If the field is a member of a oneof, returns the index in that oneof. int index_in_oneof() const; // An extension may be declared within the scope of another message. If this // field is an extension (is_extension() is true), then extension_scope() - // returns that message, or NULL if the extension was declared at global + // returns that message, or nullptr if the extension was declared at global // scope. If this is not an extension, extension_scope() is undefined (may // assert-fail). const Descriptor* extension_scope() const; @@ -682,19 +816,18 @@ class LIBPROTOBUF_EXPORT FieldDescriptor { // Get the FieldOptions for this field. This includes things listed in // square brackets after the field definition. E.g., the field: // optional string text = 1 [ctype=CORD]; - // has the "ctype" option set. Allowed options are defined by FieldOptions - // in google/protobuf/descriptor.proto, and any available extensions of that - // message. + // has the "ctype" option set. Allowed options are defined by FieldOptions in + // descriptor.proto, and any available extensions of that message. const FieldOptions& options() const; // See Descriptor::CopyTo(). void CopyTo(FieldDescriptorProto* proto) const; // See Descriptor::DebugString(). - string DebugString() const; + std::string DebugString() const; // See Descriptor::DebugStringWithOptions(). - string DebugStringWithOptions(const DebugStringOptions& options) const; + std::string DebugStringWithOptions(const DebugStringOptions& options) const; // Helper method to get the CppType for a particular Type. static CppType TypeToCppType(Type type); @@ -708,6 +841,21 @@ class LIBPROTOBUF_EXPORT FieldDescriptor { // Return true iff [packed = true] is valid for fields of this type. static inline bool IsTypePackable(Type field_type); + // Returns full_name() except if the field is a MessageSet extension, + // in which case it returns the full_name() of the containing message type + // for backwards compatibility with proto1. + // + // A MessageSet extension is defined as an optional message extension + // whose containing type has the message_set_wire_format option set. + // This should be true of extensions of google.protobuf.bridge.MessageSet; + // by convention, such extensions are named "message_set_extension". + // + // The opposite operation (looking up an extension's FieldDescriptor given + // its printable name) can be accomplished with + // message->file()->pool()->FindExtensionByPrintableName(message, name) + // where the extension extends "message". + const std::string& PrintableNameForExtension() const; + // Source Location --------------------------------------------------- // Updates |*out_location| to the source location of the complete @@ -716,26 +864,28 @@ class LIBPROTOBUF_EXPORT FieldDescriptor { bool GetSourceLocation(SourceLocation* out_location) const; private: + friend class Symbol; typedef FieldOptions OptionsType; // Allows access to GetLocationPath for annotations. - friend class ::google::protobuf::io::Printer; + friend class io::Printer; + friend class compiler::cpp::Formatter; + friend class Reflection; // Fill the json_name field of FieldDescriptorProto. void CopyJsonNameTo(FieldDescriptorProto* proto) const; // See Descriptor::DebugString(). - enum PrintLabelFlag { PRINT_LABEL, OMIT_LABEL }; - void DebugString(int depth, PrintLabelFlag print_label_flag, - string* contents, const DebugStringOptions& options) const; + void DebugString(int depth, std::string* contents, + const DebugStringOptions& options) const; // formats the default value appropriately and returns it as a string. // Must have a default value to call this. If quote_string_type is true, then // types of CPPTYPE_STRING whill be surrounded by quotes and CEscaped. - string DefaultValueAsString(bool quote_string_type) const; + std::string DefaultValueAsString(bool quote_string_type) const; // Helper function that returns the field type name for DebugString. - string FieldTypeNameDebugString() const; + std::string FieldTypeNameDebugString() const; // Walks up the descriptor tree to generate the source location path // to this descriptor from the file root. @@ -744,58 +894,76 @@ class LIBPROTOBUF_EXPORT FieldDescriptor { // Returns true if this is a map message type. bool is_map_message_type() const; - const string* name_; - const string* full_name_; - const string* lowercase_name_; - const string* camelcase_name_; - // If has_json_name_ is true, it's the value specified by the user. - // Otherwise, it has the same value as camelcase_name_. - const string* json_name_; - const FileDescriptor* file_; - GoogleOnceDynamic* type_once_; - static void TypeOnceInit(const FieldDescriptor* to_init); - void InternalTypeOnceInit() const; - mutable Type type_; - Label label_; - bool has_default_value_; + bool has_default_value_ : 1; + bool proto3_optional_ : 1; // Whether the user has specified the json_name field option in the .proto // file. - bool has_json_name_; - bool is_extension_; + bool has_json_name_ : 1; + bool is_extension_ : 1; + bool is_oneof_ : 1; + + // Actually a `Label` but stored as uint8_t to save space. + uint8_t label_ : 2; + + // Actually a `Type`, but stored as uint8_t to save space. + mutable uint8_t type_; + + // Logically: + // all_names_ = [name, full_name, lower, camel, json] + // However: + // duplicates will be omitted, so lower/camel/json might be in the same + // position. + // We store the true offset for each name here, and the bit width must be + // large enough to account for the worst case where all names are present. + uint8_t lowercase_name_index_ : 2; + uint8_t camelcase_name_index_ : 2; + uint8_t json_name_index_ : 3; + // Sadly, `number_` located here to reduce padding. Unrelated to all_names_ + // and its indices above. int number_; - int index_in_oneof_; + const std::string* all_names_; + const FileDescriptor* file_; + + internal::once_flag* type_once_; + static void TypeOnceInit(const FieldDescriptor* to_init); + void InternalTypeOnceInit() const; const Descriptor* containing_type_; - const OneofDescriptor* containing_oneof_; - const Descriptor* extension_scope_; - mutable const Descriptor* message_type_; - mutable const EnumDescriptor* enum_type_; + union { + const OneofDescriptor* containing_oneof; + const Descriptor* extension_scope; + } scope_; + union { + mutable const Descriptor* message_type; + mutable const EnumDescriptor* enum_type; + const char* lazy_type_name; + } type_descriptor_; const FieldOptions* options_; - const string* type_name_; - const string* default_value_enum_name_; // IMPORTANT: If you add a new field, make sure to search for all instances // of Allocate() and AllocateArray() in // descriptor.cc and update them to initialize the field. union { - int32 default_value_int32_; - int64 default_value_int64_; - uint32 default_value_uint32_; - uint64 default_value_uint64_; - float default_value_float_; + int32_t default_value_int32_t_; + int64_t default_value_int64_t_; + uint32_t default_value_uint32_t_; + uint64_t default_value_uint64_t_; + float default_value_float_; double default_value_double_; - bool default_value_bool_; + bool default_value_bool_; mutable const EnumValueDescriptor* default_value_enum_; - const string* default_value_string_; + const char* lazy_default_value_enum_name_; + const std::string* default_value_string_; + mutable std::atomic default_generated_instance_; }; static const CppType kTypeToCppTypeMap[MAX_TYPE + 1]; - static const char * const kTypeToName[MAX_TYPE + 1]; + static const char* const kTypeToName[MAX_TYPE + 1]; - static const char * const kCppTypeToName[MAX_CPPTYPE + 1]; + static const char* const kCppTypeToName[MAX_CPPTYPE + 1]; - static const char * const kLabelToName[MAX_LABEL + 1]; + static const char* const kLabelToName[MAX_LABEL + 1]; // Must be constructed using DescriptorPool. FieldDescriptor() {} @@ -808,15 +976,21 @@ class LIBPROTOBUF_EXPORT FieldDescriptor { // Describes a oneof defined in a message type. -class LIBPROTOBUF_EXPORT OneofDescriptor { +class PROTOBUF_EXPORT OneofDescriptor : private internal::SymbolBase { public: - const string& name() const; // Name of this oneof. - const string& full_name() const; // Fully-qualified name of the oneof. + typedef OneofDescriptorProto Proto; + + const std::string& name() const; // Name of this oneof. + const std::string& full_name() const; // Fully-qualified name of the oneof. // Index of this oneof within the message's oneof array. int index() const; - // The .proto file in which this oneof was defined. Never NULL. + // Returns whether this oneof was inserted by the compiler to wrap a proto3 + // optional field. If this returns true, code generators should *not* emit it. + bool is_synthetic() const; + + // The .proto file in which this oneof was defined. Never nullptr. const FileDescriptor* file() const; // The Descriptor for the message containing this oneof. const Descriptor* containing_type() const; @@ -833,10 +1007,10 @@ class LIBPROTOBUF_EXPORT OneofDescriptor { void CopyTo(OneofDescriptorProto* proto) const; // See Descriptor::DebugString(). - string DebugString() const; + std::string DebugString() const; // See Descriptor::DebugStringWithOptions(). - string DebugStringWithOptions(const DebugStringOptions& options) const; + std::string DebugStringWithOptions(const DebugStringOptions& options) const; // Source Location --------------------------------------------------- @@ -846,26 +1020,28 @@ class LIBPROTOBUF_EXPORT OneofDescriptor { bool GetSourceLocation(SourceLocation* out_location) const; private: + friend class Symbol; typedef OneofOptions OptionsType; // Allows access to GetLocationPath for annotations. - friend class ::google::protobuf::io::Printer; + friend class io::Printer; + friend class compiler::cpp::Formatter; // See Descriptor::DebugString(). - void DebugString(int depth, string* contents, + void DebugString(int depth, std::string* contents, const DebugStringOptions& options) const; // Walks up the descriptor tree to generate the source location path // to this descriptor from the file root. void GetLocationPath(std::vector* output) const; - const string* name_; - const string* full_name_; - const Descriptor* containing_type_; - bool is_extendable_; int field_count_; - const FieldDescriptor** fields_; + + // all_names_ = [name, full_name] + const std::string* all_names_; + const Descriptor* containing_type_; const OneofOptions* options_; + const FieldDescriptor* fields_; // IMPORTANT: If you add a new field, make sure to search for all instances // of Allocate() and AllocateArray() @@ -881,18 +1057,20 @@ class LIBPROTOBUF_EXPORT OneofDescriptor { // Describes an enum type defined in a .proto file. To get the EnumDescriptor // for a generated enum type, call TypeName_descriptor(). Use DescriptorPool // to construct your own descriptors. -class LIBPROTOBUF_EXPORT EnumDescriptor { +class PROTOBUF_EXPORT EnumDescriptor : private internal::SymbolBase { public: + typedef EnumDescriptorProto Proto; + // The name of this enum type in the containing scope. - const string& name() const; + const std::string& name() const; // The fully-qualified name of the enum type, scope delimited by periods. - const string& full_name() const; + const std::string& full_name() const; // Index of this enum within the file or containing message's enum array. int index() const; - // The .proto file in which this enum type was defined. Never NULL. + // The .proto file in which this enum type was defined. Never nullptr. const FileDescriptor* file() const; // The number of values for this EnumDescriptor. Guaranteed to be greater @@ -902,30 +1080,30 @@ class LIBPROTOBUF_EXPORT EnumDescriptor { // These are returned in the order they were defined in the .proto file. const EnumValueDescriptor* value(int index) const; - // Looks up a value by name. Returns NULL if no such value exists. - const EnumValueDescriptor* FindValueByName(const string& name) const; - // Looks up a value by number. Returns NULL if no such value exists. If + // Looks up a value by name. Returns nullptr if no such value exists. + const EnumValueDescriptor* FindValueByName(ConstStringParam name) const; + // Looks up a value by number. Returns nullptr if no such value exists. If // multiple values have this number, the first one defined is returned. const EnumValueDescriptor* FindValueByNumber(int number) const; // If this enum type is nested in a message type, this is that message type. - // Otherwise, NULL. + // Otherwise, nullptr. const Descriptor* containing_type() const; // Get options for this enum type. These are specified in the .proto file by // placing lines like "option foo = 1234;" in the enum definition. Allowed - // options are defined by EnumOptions in google/protobuf/descriptor.proto, - // and any available extensions of that message. + // options are defined by EnumOptions in descriptor.proto, and any available + // extensions of that message. const EnumOptions& options() const; // See Descriptor::CopyTo(). void CopyTo(EnumDescriptorProto* proto) const; // See Descriptor::DebugString(). - string DebugString() const; + std::string DebugString() const; // See Descriptor::DebugStringWithOptions(). - string DebugStringWithOptions(const DebugStringOptions& options) const; + std::string DebugStringWithOptions(const DebugStringOptions& options) const; // Returns true if this is a placeholder for an unknown enum. This will // only be the case if this descriptor comes from a DescriptorPool @@ -950,18 +1128,18 @@ class LIBPROTOBUF_EXPORT EnumDescriptor { // Returns true if the number is in one of the reserved ranges. bool IsReservedNumber(int number) const; - // Returns NULL if no reserved range contains the given number. - const EnumDescriptor::ReservedRange* - FindReservedRangeContainingNumber(int number) const; + // Returns nullptr if no reserved range contains the given number. + const EnumDescriptor::ReservedRange* FindReservedRangeContainingNumber( + int number) const; // The number of reserved field names in this message type. int reserved_name_count() const; // Gets a reserved name by index, where 0 <= index < reserved_name_count(). - const string& reserved_name(int index) const; + const std::string& reserved_name(int index) const; // Returns true if the field name is reserved. - bool IsReservedName(const string& name) const; + bool IsReservedName(ConstStringParam name) const; // Source Location --------------------------------------------------- @@ -971,49 +1149,62 @@ class LIBPROTOBUF_EXPORT EnumDescriptor { bool GetSourceLocation(SourceLocation* out_location) const; private: + friend class Symbol; typedef EnumOptions OptionsType; // Allows access to GetLocationPath for annotations. - friend class ::google::protobuf::io::Printer; + friend class io::Printer; + friend class compiler::cpp::Formatter; + + // Allow access to FindValueByNumberCreatingIfUnknown. + friend class descriptor_unittest::DescriptorTest; // Looks up a value by number. If the value does not exist, dynamically // creates a new EnumValueDescriptor for that value, assuming that it was // unknown. If a new descriptor is created, this is done in a thread-safe way, // and future calls will return the same value descriptor pointer. // - // This is private but is used by GeneratedMessageReflection (which is - // friended below) to return a valid EnumValueDescriptor from GetEnum() when - // this feature is enabled. - const EnumValueDescriptor* - FindValueByNumberCreatingIfUnknown(int number) const; - + // This is private but is used by Reflection (which is friended below) to + // return a valid EnumValueDescriptor from GetEnum() when this feature is + // enabled. + const EnumValueDescriptor* FindValueByNumberCreatingIfUnknown( + int number) const; // See Descriptor::DebugString(). - void DebugString(int depth, string *contents, + void DebugString(int depth, std::string* contents, const DebugStringOptions& options) const; // Walks up the descriptor tree to generate the source location path // to this descriptor from the file root. void GetLocationPath(std::vector* output) const; - const string* name_; - const string* full_name_; + // True if this is a placeholder for an unknown type. + bool is_placeholder_ : 1; + // True if this is a placeholder and the type name wasn't fully-qualified. + bool is_unqualified_placeholder_ : 1; + + // This points to the last value _index_ that is part of the sequence starting + // with the first label, where + // `enum->value(i)->number() == enum->value(0)->number() + i` + // We measure relative to the first label to adapt to enum labels starting at + // 0 or 1. + // Uses 16-bit to avoid extra padding. Unlikely to have more than 2^15 + // sequentially numbered labels in an enum. + int16_t sequential_value_limit_; + + int value_count_; + + // all_names_ = [name, full_name] + const std::string* all_names_; const FileDescriptor* file_; const Descriptor* containing_type_; const EnumOptions* options_; - - // True if this is a placeholder for an unknown type. - bool is_placeholder_; - // True if this is a placeholder and the type name wasn't fully-qualified. - bool is_unqualified_placeholder_; - - int value_count_; EnumValueDescriptor* values_; int reserved_range_count_; int reserved_name_count_; EnumDescriptor::ReservedRange* reserved_ranges_; - const string** reserved_names_; + const std::string** reserved_names_; // IMPORTANT: If you add a new field, make sure to search for all instances // of Allocate() and AllocateArray() in @@ -1024,10 +1215,11 @@ class LIBPROTOBUF_EXPORT EnumDescriptor { friend class DescriptorBuilder; friend class Descriptor; friend class FieldDescriptor; + friend class FileDescriptorTables; friend class EnumValueDescriptor; friend class FileDescriptor; friend class DescriptorPool; - friend class internal::GeneratedMessageReflection; + friend class Reflection; GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(EnumDescriptor); }; @@ -1036,40 +1228,41 @@ class LIBPROTOBUF_EXPORT EnumDescriptor { // for its type, then use EnumDescriptor::FindValueByName() or // EnumDescriptor::FindValueByNumber(). Use DescriptorPool to construct // your own descriptors. -class LIBPROTOBUF_EXPORT EnumValueDescriptor { +class PROTOBUF_EXPORT EnumValueDescriptor : private internal::SymbolBaseN<0>, + private internal::SymbolBaseN<1> { public: - const string& name() const; // Name of this enum constant. - int index() const; // Index within the enums's Descriptor. - int number() const; // Numeric value of this enum constant. + typedef EnumValueDescriptorProto Proto; + + const std::string& name() const; // Name of this enum constant. + int index() const; // Index within the enums's Descriptor. + int number() const; // Numeric value of this enum constant. // The full_name of an enum value is a sibling symbol of the enum type. // e.g. the full name of FieldDescriptorProto::TYPE_INT32 is actually // "google.protobuf.FieldDescriptorProto.TYPE_INT32", NOT // "google.protobuf.FieldDescriptorProto.Type.TYPE_INT32". This is to conform // with C++ scoping rules for enums. - const string& full_name() const; + const std::string& full_name() const; - // The .proto file in which this value was defined. Never NULL. + // The .proto file in which this value was defined. Never nullptr. const FileDescriptor* file() const; - // The type of this value. Never NULL. + // The type of this value. Never nullptr. const EnumDescriptor* type() const; - // Get options for this enum value. These are specified in the .proto file - // by adding text like "[foo = 1234]" after an enum value definition. - // Allowed options are defined by EnumValueOptions in - // google/protobuf/descriptor.proto, and any available extensions of that - // message. + // Get options for this enum value. These are specified in the .proto file by + // adding text like "[foo = 1234]" after an enum value definition. Allowed + // options are defined by EnumValueOptions in descriptor.proto, and any + // available extensions of that message. const EnumValueOptions& options() const; // See Descriptor::CopyTo(). void CopyTo(EnumValueDescriptorProto* proto) const; // See Descriptor::DebugString(). - string DebugString() const; + std::string DebugString() const; // See Descriptor::DebugStringWithOptions(). - string DebugStringWithOptions(const DebugStringOptions& options) const; - + std::string DebugStringWithOptions(const DebugStringOptions& options) const; // Source Location --------------------------------------------------- @@ -1079,22 +1272,24 @@ class LIBPROTOBUF_EXPORT EnumValueDescriptor { bool GetSourceLocation(SourceLocation* out_location) const; private: + friend class Symbol; typedef EnumValueOptions OptionsType; // Allows access to GetLocationPath for annotations. - friend class ::google::protobuf::io::Printer; + friend class io::Printer; + friend class compiler::cpp::Formatter; // See Descriptor::DebugString(). - void DebugString(int depth, string *contents, + void DebugString(int depth, std::string* contents, const DebugStringOptions& options) const; // Walks up the descriptor tree to generate the source location path // to this descriptor from the file root. void GetLocationPath(std::vector* output) const; - const string* name_; - const string* full_name_; int number_; + // all_names_ = [name, full_name] + const std::string* all_names_; const EnumDescriptor* type_; const EnumValueOptions* options_; // IMPORTANT: If you add a new field, make sure to search for all instances @@ -1107,30 +1302,30 @@ class LIBPROTOBUF_EXPORT EnumValueDescriptor { friend class EnumDescriptor; friend class DescriptorPool; friend class FileDescriptorTables; + friend class Reflection; GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(EnumValueDescriptor); }; -// Describes an RPC service. To get the ServiceDescriptor for a service, -// call Service::GetDescriptor(). Generated service classes also have a -// static method called descriptor() which returns the type's -// ServiceDescriptor. Use DescriptorPool to construct your own descriptors. -class LIBPROTOBUF_EXPORT ServiceDescriptor { +// Describes an RPC service. Use DescriptorPool to construct your own +// descriptors. +class PROTOBUF_EXPORT ServiceDescriptor : private internal::SymbolBase { public: + typedef ServiceDescriptorProto Proto; + // The name of the service, not including its containing scope. - const string& name() const; + const std::string& name() const; // The fully-qualified name of the service, scope delimited by periods. - const string& full_name() const; + const std::string& full_name() const; // Index of this service within the file's services array. int index() const; - // The .proto file in which this service was defined. Never NULL. + // The .proto file in which this service was defined. Never nullptr. const FileDescriptor* file() const; // Get options for this service type. These are specified in the .proto file // by placing lines like "option foo = 1234;" in the service definition. - // Allowed options are defined by ServiceOptions in - // google/protobuf/descriptor.proto, and any available extensions of that - // message. + // Allowed options are defined by ServiceOptions in descriptor.proto, and any + // available extensions of that message. const ServiceOptions& options() const; // The number of methods this service defines. @@ -1140,16 +1335,15 @@ class LIBPROTOBUF_EXPORT ServiceDescriptor { const MethodDescriptor* method(int index) const; // Look up a MethodDescriptor by name. - const MethodDescriptor* FindMethodByName(const string& name) const; + const MethodDescriptor* FindMethodByName(ConstStringParam name) const; // See Descriptor::CopyTo(). void CopyTo(ServiceDescriptorProto* proto) const; // See Descriptor::DebugString(). - string DebugString() const; + std::string DebugString() const; // See Descriptor::DebugStringWithOptions(). - string DebugStringWithOptions(const DebugStringOptions& options) const; - + std::string DebugStringWithOptions(const DebugStringOptions& options) const; // Source Location --------------------------------------------------- @@ -1159,20 +1353,23 @@ class LIBPROTOBUF_EXPORT ServiceDescriptor { bool GetSourceLocation(SourceLocation* out_location) const; private: + friend class Symbol; typedef ServiceOptions OptionsType; // Allows access to GetLocationPath for annotations. - friend class ::google::protobuf::io::Printer; + friend class io::Printer; + friend class compiler::cpp::Formatter; // See Descriptor::DebugString(). - void DebugString(string *contents, const DebugStringOptions& options) const; + void DebugString(std::string* contents, + const DebugStringOptions& options) const; // Walks up the descriptor tree to generate the source location path // to this descriptor from the file root. void GetLocationPath(std::vector* output) const; - const string* name_; - const string* full_name_; + // all_names_ = [name, full_name] + const std::string* all_names_; const FileDescriptor* file_; const ServiceOptions* options_; MethodDescriptor* methods_; @@ -1194,18 +1391,20 @@ class LIBPROTOBUF_EXPORT ServiceDescriptor { // a service, first get its ServiceDescriptor, then call // ServiceDescriptor::FindMethodByName(). Use DescriptorPool to construct your // own descriptors. -class LIBPROTOBUF_EXPORT MethodDescriptor { +class PROTOBUF_EXPORT MethodDescriptor : private internal::SymbolBase { public: + typedef MethodDescriptorProto Proto; + // Name of this method, not including containing scope. - const string& name() const; + const std::string& name() const; // The fully-qualified name of the method, scope delimited by periods. - const string& full_name() const; + const std::string& full_name() const; // Index within the service's Descriptor. int index() const; - // The .proto file in which this method was defined. Never NULL. + // The .proto file in which this method was defined. Never nullptr. const FileDescriptor* file() const; - // Gets the service to which this method belongs. Never NULL. + // Gets the service to which this method belongs. Never nullptr. const ServiceDescriptor* service() const; // Gets the type of protocol message which this method accepts as input. @@ -1221,19 +1420,17 @@ class LIBPROTOBUF_EXPORT MethodDescriptor { // Get options for this method. These are specified in the .proto file by // placing lines like "option foo = 1234;" in curly-braces after a method // declaration. Allowed options are defined by MethodOptions in - // google/protobuf/descriptor.proto, and any available extensions of that - // message. + // descriptor.proto, and any available extensions of that message. const MethodOptions& options() const; // See Descriptor::CopyTo(). void CopyTo(MethodDescriptorProto* proto) const; // See Descriptor::DebugString(). - string DebugString() const; + std::string DebugString() const; // See Descriptor::DebugStringWithOptions(). - string DebugStringWithOptions(const DebugStringOptions& options) const; - + std::string DebugStringWithOptions(const DebugStringOptions& options) const; // Source Location --------------------------------------------------- @@ -1243,27 +1440,29 @@ class LIBPROTOBUF_EXPORT MethodDescriptor { bool GetSourceLocation(SourceLocation* out_location) const; private: + friend class Symbol; typedef MethodOptions OptionsType; // Allows access to GetLocationPath for annotations. - friend class ::google::protobuf::io::Printer; + friend class io::Printer; + friend class compiler::cpp::Formatter; // See Descriptor::DebugString(). - void DebugString(int depth, string *contents, + void DebugString(int depth, std::string* contents, const DebugStringOptions& options) const; // Walks up the descriptor tree to generate the source location path // to this descriptor from the file root. void GetLocationPath(std::vector* output) const; - const string* name_; - const string* full_name_; + bool client_streaming_; + bool server_streaming_; + // all_names_ = [name, full_name] + const std::string* all_names_; const ServiceDescriptor* service_; mutable internal::LazyDescriptor input_type_; mutable internal::LazyDescriptor output_type_; const MethodOptions* options_; - bool client_streaming_; - bool server_streaming_; // IMPORTANT: If you add a new field, make sure to search for all instances // of Allocate() and AllocateArray() in // descriptor.cc and update them to initialize the field. @@ -1279,17 +1478,19 @@ class LIBPROTOBUF_EXPORT MethodDescriptor { // Describes a whole .proto file. To get the FileDescriptor for a compiled-in // file, get the descriptor for something defined in that file and call // descriptor->file(). Use DescriptorPool to construct your own descriptors. -class LIBPROTOBUF_EXPORT FileDescriptor { +class PROTOBUF_EXPORT FileDescriptor { public: + typedef FileDescriptorProto Proto; + // The filename, relative to the source tree. - // e.g. "google/protobuf/descriptor.proto" - const string& name() const; + // e.g. "foo/bar/baz.proto" + const std::string& name() const; // The package, e.g. "google.protobuf.compiler". - const string& package() const; + const std::string& package() const; // The DescriptorPool in which this FileDescriptor and all its contents were - // allocated. Never NULL. + // allocated. Never nullptr. const DescriptorPool* pool() const; // The number of files imported by this one. @@ -1344,36 +1545,39 @@ class LIBPROTOBUF_EXPORT FileDescriptor { // Get options for this file. These are specified in the .proto file by // placing lines like "option foo = 1234;" at the top level, outside of any // other definitions. Allowed options are defined by FileOptions in - // google/protobuf/descriptor.proto, and any available extensions of that - // message. + // descriptor.proto, and any available extensions of that message. const FileOptions& options() const; // Syntax of this file. enum Syntax { SYNTAX_UNKNOWN = 0, - SYNTAX_PROTO2 = 2, - SYNTAX_PROTO3 = 3, + SYNTAX_PROTO2 = 2, + SYNTAX_PROTO3 = 3, }; Syntax syntax() const; static const char* SyntaxName(Syntax syntax); - // Find a top-level message type by name. Returns NULL if not found. - const Descriptor* FindMessageTypeByName(const string& name) const; - // Find a top-level enum type by name. Returns NULL if not found. - const EnumDescriptor* FindEnumTypeByName(const string& name) const; - // Find an enum value defined in any top-level enum by name. Returns NULL if + // Find a top-level message type by name (not full_name). Returns nullptr if // not found. - const EnumValueDescriptor* FindEnumValueByName(const string& name) const; - // Find a service definition by name. Returns NULL if not found. - const ServiceDescriptor* FindServiceByName(const string& name) const; - // Find a top-level extension definition by name. Returns NULL if not found. - const FieldDescriptor* FindExtensionByName(const string& name) const; + const Descriptor* FindMessageTypeByName(ConstStringParam name) const; + // Find a top-level enum type by name. Returns nullptr if not found. + const EnumDescriptor* FindEnumTypeByName(ConstStringParam name) const; + // Find an enum value defined in any top-level enum by name. Returns nullptr + // if not found. + const EnumValueDescriptor* FindEnumValueByName(ConstStringParam name) const; + // Find a service definition by name. Returns nullptr if not found. + const ServiceDescriptor* FindServiceByName(ConstStringParam name) const; + // Find a top-level extension definition by name. Returns nullptr if not + // found. + const FieldDescriptor* FindExtensionByName(ConstStringParam name) const; // Similar to FindExtensionByName(), but searches by lowercased-name. See // Descriptor::FindFieldByLowercaseName(). - const FieldDescriptor* FindExtensionByLowercaseName(const string& name) const; + const FieldDescriptor* FindExtensionByLowercaseName( + ConstStringParam name) const; // Similar to FindExtensionByName(), but searches by camelcased-name. See // Descriptor::FindFieldByCamelcaseName(). - const FieldDescriptor* FindExtensionByCamelcaseName(const string& name) const; + const FieldDescriptor* FindExtensionByCamelcaseName( + ConstStringParam name) const; // See Descriptor::CopyTo(). // Notes: @@ -1388,10 +1592,10 @@ class LIBPROTOBUF_EXPORT FileDescriptor { void CopyJsonNameTo(FileDescriptorProto* proto) const; // See Descriptor::DebugString(). - string DebugString() const; + std::string DebugString() const; // See Descriptor::DebugStringWithOptions(). - string DebugStringWithOptions(const DebugStringOptions& options) const; + std::string DebugStringWithOptions(const DebugStringOptions& options) const; // Returns true if this is a placeholder for an unknown file. This will // only be the case if this descriptor comes from a DescriptorPool @@ -1413,31 +1617,41 @@ class LIBPROTOBUF_EXPORT FileDescriptor { private: typedef FileOptions OptionsType; - const string* name_; - const string* package_; + const std::string* name_; + const std::string* package_; const DescriptorPool* pool_; - GoogleOnceDynamic* dependencies_once_; + + // Data required to do lazy initialization. + struct PROTOBUF_EXPORT LazyInitData { +#ifndef SWIG + internal::once_flag once; +#endif + const char** dependencies_names; + }; + + LazyInitData* dependencies_once_; static void DependenciesOnceInit(const FileDescriptor* to_init); void InternalDependenciesOnceInit() const; - // These are arranged to minimze padding on 64-bit. + // These are arranged to minimize padding on 64-bit. int dependency_count_; int public_dependency_count_; int weak_dependency_count_; int message_type_count_; int enum_type_count_; int service_count_; - int extension_count_; - Syntax syntax_; - bool is_placeholder_; + bool is_placeholder_; // Indicates the FileDescriptor is completed building. Used to verify // that type accessor functions that can possibly build a dependent file // aren't called during the process of building the file. bool finished_building_; + // Actually a `Syntax` but stored as uint8_t to save space. + uint8_t syntax_; + // This one is here to fill the padding. + int extension_count_; mutable const FileDescriptor** dependencies_; - const string** dependencies_names_; int* public_dependencies_; int* weak_dependencies_; Descriptor* message_types_; @@ -1494,7 +1708,7 @@ class LIBPROTOBUF_EXPORT FileDescriptor { // // You can also search for descriptors within a DescriptorPool by name, and // extensions by number. -class LIBPROTOBUF_EXPORT DescriptorPool { +class PROTOBUF_EXPORT DescriptorPool { public: // Create a normal, empty DescriptorPool. DescriptorPool(); @@ -1524,7 +1738,7 @@ class LIBPROTOBUF_EXPORT DescriptorPool { // in subsequent lookups in the DescriptorPool. class ErrorCollector; explicit DescriptorPool(DescriptorDatabase* fallback_database, - ErrorCollector* error_collector = NULL); + ErrorCollector* error_collector = nullptr); ~DescriptorPool(); @@ -1534,36 +1748,44 @@ class LIBPROTOBUF_EXPORT DescriptorPool { static const DescriptorPool* generated_pool(); - // Find a FileDescriptor in the pool by file name. Returns NULL if not + // Find a FileDescriptor in the pool by file name. Returns nullptr if not // found. - const FileDescriptor* FindFileByName(const string& name) const; + const FileDescriptor* FindFileByName(ConstStringParam name) const; // Find the FileDescriptor in the pool which defines the given symbol. // If any of the Find*ByName() methods below would succeed, then this is // equivalent to calling that method and calling the result's file() method. - // Otherwise this returns NULL. + // Otherwise this returns nullptr. const FileDescriptor* FindFileContainingSymbol( - const string& symbol_name) const; + ConstStringParam symbol_name) const; // Looking up descriptors ------------------------------------------ // These find descriptors by fully-qualified name. These will find both - // top-level descriptors and nested descriptors. They return NULL if not + // top-level descriptors and nested descriptors. They return nullptr if not // found. - const Descriptor* FindMessageTypeByName(const string& name) const; - const FieldDescriptor* FindFieldByName(const string& name) const; - const FieldDescriptor* FindExtensionByName(const string& name) const; - const OneofDescriptor* FindOneofByName(const string& name) const; - const EnumDescriptor* FindEnumTypeByName(const string& name) const; - const EnumValueDescriptor* FindEnumValueByName(const string& name) const; - const ServiceDescriptor* FindServiceByName(const string& name) const; - const MethodDescriptor* FindMethodByName(const string& name) const; + const Descriptor* FindMessageTypeByName(ConstStringParam name) const; + const FieldDescriptor* FindFieldByName(ConstStringParam name) const; + const FieldDescriptor* FindExtensionByName(ConstStringParam name) const; + const OneofDescriptor* FindOneofByName(ConstStringParam name) const; + const EnumDescriptor* FindEnumTypeByName(ConstStringParam name) const; + const EnumValueDescriptor* FindEnumValueByName(ConstStringParam name) const; + const ServiceDescriptor* FindServiceByName(ConstStringParam name) const; + const MethodDescriptor* FindMethodByName(ConstStringParam name) const; // Finds an extension of the given type by number. The extendee must be // a member of this DescriptorPool or one of its underlays. const FieldDescriptor* FindExtensionByNumber(const Descriptor* extendee, int number) const; + // Finds an extension of the given type by its printable name. + // See comments above PrintableNameForExtension() for the definition of + // "printable name". The extendee must be a member of this DescriptorPool + // or one of its underlays. Returns nullptr if there is no known message + // extension with the given printable name. + const FieldDescriptor* FindExtensionByPrintableName( + const Descriptor* extendee, ConstStringParam printable_name) const; + // Finds extensions of extendee. The extensions will be appended to // out in an undefined order. Only extensions defined directly in // this DescriptorPool or one of its underlays are guaranteed to be @@ -1577,7 +1799,7 @@ class LIBPROTOBUF_EXPORT DescriptorPool { // When converting a FileDescriptorProto to a FileDescriptor, various // errors might be detected in the input. The caller may handle these // programmatically by implementing an ErrorCollector. - class LIBPROTOBUF_EXPORT ErrorCollector { + class PROTOBUF_EXPORT ErrorCollector { public: inline ErrorCollector() {} virtual ~ErrorCollector(); @@ -1586,37 +1808,40 @@ class LIBPROTOBUF_EXPORT DescriptorPool { // This is useful e.g. for mapping the error back to an exact location // in a .proto file. enum ErrorLocation { - NAME, // the symbol name, or the package name for files - NUMBER, // field or extension range number - TYPE, // field type - EXTENDEE, // field extendee - DEFAULT_VALUE, // field default value - INPUT_TYPE, // method input type - OUTPUT_TYPE, // method output type - OPTION_NAME, // name in assignment - OPTION_VALUE, // value in option assignment - OTHER // some other problem + NAME, // the symbol name, or the package name for files + NUMBER, // field or extension range number + TYPE, // field type + EXTENDEE, // field extendee + DEFAULT_VALUE, // field default value + INPUT_TYPE, // method input type + OUTPUT_TYPE, // method output type + OPTION_NAME, // name in assignment + OPTION_VALUE, // value in option assignment + IMPORT, // import error + OTHER // some other problem }; // Reports an error in the FileDescriptorProto. Use this function if the // problem occurred should interrupt building the FileDescriptorProto. virtual void AddError( - const string& filename, // File name in which the error occurred. - const string& element_name, // Full name of the erroneous element. - const Message* descriptor, // Descriptor of the erroneous element. - ErrorLocation location, // One of the location constants, above. - const string& message // Human-readable error message. - ) = 0; + const std::string& filename, // File name in which the error occurred. + const std::string& element_name, // Full name of the erroneous element. + const Message* descriptor, // Descriptor of the erroneous element. + ErrorLocation location, // One of the location constants, above. + const std::string& message // Human-readable error message. + ) = 0; // Reports a warning in the FileDescriptorProto. Use this function if the // problem occurred should NOT interrupt building the FileDescriptorProto. virtual void AddWarning( - const string& /*filename*/, // File name in which the error occurred. - const string& /*element_name*/, // Full name of the erroneous element. - const Message* /*descriptor*/, // Descriptor of the erroneous element. - ErrorLocation /*location*/, // One of the location constants, above. - const string& /*message*/ // Human-readable error message. - ) {} + const std::string& /*filename*/, // File name in which the error + // occurred. + const std::string& /*element_name*/, // Full name of the erroneous + // element. + const Message* /*descriptor*/, // Descriptor of the erroneous element. + ErrorLocation /*location*/, // One of the location constants, above. + const std::string& /*message*/ // Human-readable error message. + ) {} private: GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ErrorCollector); @@ -1624,15 +1849,14 @@ class LIBPROTOBUF_EXPORT DescriptorPool { // Convert the FileDescriptorProto to real descriptors and place them in // this DescriptorPool. All dependencies of the file must already be in - // the pool. Returns the resulting FileDescriptor, or NULL if there were + // the pool. Returns the resulting FileDescriptor, or nullptr if there were // problems with the input (e.g. the message was invalid, or dependencies // were missing). Details about the errors are written to GOOGLE_LOG(ERROR). const FileDescriptor* BuildFile(const FileDescriptorProto& proto); // Same as BuildFile() except errors are sent to the given ErrorCollector. const FileDescriptor* BuildFileCollectingErrors( - const FileDescriptorProto& proto, - ErrorCollector* error_collector); + const FileDescriptorProto& proto, ErrorCollector* error_collector); // By default, it is an error if a FileDescriptorProto contains references // to types or other files that are not found in the DescriptorPool (or its @@ -1687,8 +1911,8 @@ class LIBPROTOBUF_EXPORT DescriptorPool { // Called by generated classes at init time to add their descriptors to // generated_pool. Do NOT call this in your own code! filename must be a // permanent string (e.g. a string literal). - static void InternalAddGeneratedFile( - const void* encoded_file_descriptor, int size); + static void InternalAddGeneratedFile(const void* encoded_file_descriptor, + int size); // Disallow [enforce_utf8 = false] in .proto files. void DisallowEnforceUtf8() { disallow_enforce_utf8_ = true; } @@ -1700,6 +1924,11 @@ class LIBPROTOBUF_EXPORT DescriptorPool { // the underlay takes precedence. static DescriptorPool* internal_generated_pool(); + // For internal use only: Gets a non-const pointer to the generated + // descriptor database. + // Only used for testing. + static DescriptorDatabase* internal_generated_database(); + // For internal use only: Changes the behavior of BuildFile() such that it // allows the file to make reference to message types declared in other files // which it did not officially declare as dependencies. @@ -1709,7 +1938,7 @@ class LIBPROTOBUF_EXPORT DescriptorPool { // Delay the building of dependencies of a file descriptor until absolutely // necessary, like when message_type() is called on a field that is defined // in that dependency's file. This will cause functional issues if a proto - // or one of it's dependencies has errors. Should only be enabled for the + // or one of its dependencies has errors. Should only be enabled for the // generated_pool_ (because no descriptor build errors are guaranteed by // the compilation generation process), testing, or if a lack of descriptor // build errors can be guaranteed for a pool. @@ -1728,12 +1957,12 @@ class LIBPROTOBUF_EXPORT DescriptorPool { // For internal (unit test) use only: Returns true if a FileDescriptor has // been constructed for the given file, false otherwise. Useful for testing // lazy descriptor initialization behavior. - bool InternalIsFileLoaded(const string& filename) const; - + bool InternalIsFileLoaded(ConstStringParam filename) const; // Add a file to unused_import_track_files_. DescriptorBuilder will log - // warnings for those files if there is any unused import. - void AddUnusedImportTrackFile(const string& file_name); + // warnings or errors for those files if there is any unused import. + void AddUnusedImportTrackFile(ConstStringParam file_name, + bool is_error = false); void ClearUnusedImportTrackFiles(); private: @@ -1751,33 +1980,40 @@ class LIBPROTOBUF_EXPORT DescriptorPool { // Return true if the given name is a sub-symbol of any non-package // descriptor that already exists in the descriptor pool. (The full // definition of such types is already known.) - bool IsSubSymbolOfBuiltType(const string& name) const; + bool IsSubSymbolOfBuiltType(StringPiece name) const; // Tries to find something in the fallback database and link in the // corresponding proto file. Returns true if successful, in which case // the caller should search for the thing again. These are declared // const because they are called by (semantically) const methods. - bool TryFindFileInFallbackDatabase(const string& name) const; - bool TryFindSymbolInFallbackDatabase(const string& name) const; + bool TryFindFileInFallbackDatabase(StringPiece name) const; + bool TryFindSymbolInFallbackDatabase(StringPiece name) const; bool TryFindExtensionInFallbackDatabase(const Descriptor* containing_type, int field_number) const; + // This internal find extension method only check with its table and underlay + // descriptor_pool's table. It does not check with fallback DB and no + // additional proto file will be build in this method. + const FieldDescriptor* InternalFindExtensionByNumberNoLock( + const Descriptor* extendee, int number) const; + // Like BuildFile() but called internally when the file has been loaded from // fallback_database_. Declared const because it is called by (semantically) // const methods. const FileDescriptor* BuildFileFromDatabase( - const FileDescriptorProto& proto) const; + const FileDescriptorProto& proto) const; // Helper for when lazily_build_dependencies_ is set, can look up a symbol // after the file's descriptor is built, and can build the file where that // symbol is defined if necessary. Will create a placeholder if the type // doesn't exist in the fallback database, or the file doesn't build // successfully. - Symbol CrossLinkOnDemandHelper(const string& name, bool expecting_enum) const; + Symbol CrossLinkOnDemandHelper(StringPiece name, + bool expecting_enum) const; // Create a placeholder FileDescriptor of the specified name - FileDescriptor* NewPlaceholderFile(const string& name) const; - FileDescriptor* NewPlaceholderFileWithMutexHeld(const string& name) const; + FileDescriptor* NewPlaceholderFile(StringPiece name) const; + FileDescriptor* NewPlaceholderFileWithMutexHeld(StringPiece name) const; enum PlaceholderType { PLACEHOLDER_MESSAGE, @@ -1785,14 +2021,14 @@ class LIBPROTOBUF_EXPORT DescriptorPool { PLACEHOLDER_EXTENDABLE_MESSAGE }; // Create a placeholder Descriptor of the specified name - Symbol NewPlaceholder(const string& name, + Symbol NewPlaceholder(StringPiece name, PlaceholderType placeholder_type) const; - Symbol NewPlaceholderWithMutexHeld(const string& name, + Symbol NewPlaceholderWithMutexHeld(StringPiece name, PlaceholderType placeholder_type) const; - // If fallback_database_ is NULL, this is NULL. Otherwise, this is a mutex - // which must be locked while accessing tables_. - Mutex* mutex_; + // If fallback_database_ is nullptr, this is nullptr. Otherwise, this is a + // mutex which must be locked while accessing tables_. + internal::WrappedMutex* mutex_; // See constructor. DescriptorDatabase* fallback_database_; @@ -1802,14 +2038,17 @@ class LIBPROTOBUF_EXPORT DescriptorPool { // This class contains a lot of hash maps with complicated types that // we'd like to keep out of the header. class Tables; - google::protobuf::scoped_ptr tables_; + std::unique_ptr tables_; bool enforce_dependencies_; bool lazily_build_dependencies_; bool allow_unknown_; bool enforce_weak_; bool disallow_enforce_utf8_; - std::set unused_import_track_files_; + + // Set of files to track for unused imports. The bool value when true means + // unused imports are treated as errors (and as warnings when false). + std::map unused_import_track_files_; GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(DescriptorPool); }; @@ -1823,7 +2062,12 @@ class LIBPROTOBUF_EXPORT DescriptorPool { // Strings fields are stored as pointers but returned as const references. #define PROTOBUF_DEFINE_STRING_ACCESSOR(CLASS, FIELD) \ - inline const string& CLASS::FIELD() const { return *FIELD##_; } + inline const std::string& CLASS::FIELD() const { return *FIELD##_; } + +// Name and full name are stored in a single array to save space. +#define PROTOBUF_DEFINE_NAME_ACCESSOR(CLASS) \ + inline const std::string& CLASS::name() const { return all_names_[0]; } \ + inline const std::string& CLASS::full_name() const { return all_names_[1]; } // Arrays take an index parameter, obviously. #define PROTOBUF_DEFINE_ARRAY_ACCESSOR(CLASS, FIELD, TYPE) \ @@ -1832,13 +2076,13 @@ class LIBPROTOBUF_EXPORT DescriptorPool { #define PROTOBUF_DEFINE_OPTIONS_ACCESSOR(CLASS, TYPE) \ inline const TYPE& CLASS::options() const { return *options_; } -PROTOBUF_DEFINE_STRING_ACCESSOR(Descriptor, name) -PROTOBUF_DEFINE_STRING_ACCESSOR(Descriptor, full_name) +PROTOBUF_DEFINE_NAME_ACCESSOR(Descriptor) PROTOBUF_DEFINE_ACCESSOR(Descriptor, file, const FileDescriptor*) PROTOBUF_DEFINE_ACCESSOR(Descriptor, containing_type, const Descriptor*) PROTOBUF_DEFINE_ACCESSOR(Descriptor, field_count, int) PROTOBUF_DEFINE_ACCESSOR(Descriptor, oneof_decl_count, int) +PROTOBUF_DEFINE_ACCESSOR(Descriptor, real_oneof_decl_count, int) PROTOBUF_DEFINE_ACCESSOR(Descriptor, nested_type_count, int) PROTOBUF_DEFINE_ACCESSOR(Descriptor, enum_type_count, int) @@ -1851,8 +2095,7 @@ PROTOBUF_DEFINE_ACCESSOR(Descriptor, extension_range_count, int) PROTOBUF_DEFINE_ACCESSOR(Descriptor, extension_count, int) PROTOBUF_DEFINE_ARRAY_ACCESSOR(Descriptor, extension_range, const Descriptor::ExtensionRange*) -PROTOBUF_DEFINE_ARRAY_ACCESSOR(Descriptor, extension, - const FieldDescriptor*) +PROTOBUF_DEFINE_ARRAY_ACCESSOR(Descriptor, extension, const FieldDescriptor*) PROTOBUF_DEFINE_ACCESSOR(Descriptor, reserved_range_count, int) PROTOBUF_DEFINE_ARRAY_ACCESSOR(Descriptor, reserved_range, @@ -1862,40 +2105,30 @@ PROTOBUF_DEFINE_ACCESSOR(Descriptor, reserved_name_count, int) PROTOBUF_DEFINE_OPTIONS_ACCESSOR(Descriptor, MessageOptions) PROTOBUF_DEFINE_ACCESSOR(Descriptor, is_placeholder, bool) -PROTOBUF_DEFINE_STRING_ACCESSOR(FieldDescriptor, name) -PROTOBUF_DEFINE_STRING_ACCESSOR(FieldDescriptor, full_name) -PROTOBUF_DEFINE_STRING_ACCESSOR(FieldDescriptor, json_name) -PROTOBUF_DEFINE_STRING_ACCESSOR(FieldDescriptor, lowercase_name) -PROTOBUF_DEFINE_STRING_ACCESSOR(FieldDescriptor, camelcase_name) +PROTOBUF_DEFINE_NAME_ACCESSOR(FieldDescriptor) PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, file, const FileDescriptor*) PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, number, int) PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, is_extension, bool) -PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, label, FieldDescriptor::Label) PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, containing_type, const Descriptor*) -PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, containing_oneof, - const OneofDescriptor*) -PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, index_in_oneof, int) -PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, extension_scope, const Descriptor*) PROTOBUF_DEFINE_OPTIONS_ACCESSOR(FieldDescriptor, FieldOptions) PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, has_default_value, bool) PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, has_json_name, bool) -PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, default_value_int32 , int32 ) -PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, default_value_int64 , int64 ) -PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, default_value_uint32, uint32) -PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, default_value_uint64, uint64) -PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, default_value_float , float ) +PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, default_value_int32_t, int32_t) +PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, default_value_int64_t, int64_t) +PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, default_value_uint32_t, uint32_t) +PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, default_value_uint64_t, uint64_t) +PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, default_value_float, float) PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, default_value_double, double) -PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, default_value_bool , bool ) +PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, default_value_bool, bool) PROTOBUF_DEFINE_STRING_ACCESSOR(FieldDescriptor, default_value_string) -PROTOBUF_DEFINE_STRING_ACCESSOR(OneofDescriptor, name) -PROTOBUF_DEFINE_STRING_ACCESSOR(OneofDescriptor, full_name) +PROTOBUF_DEFINE_NAME_ACCESSOR(OneofDescriptor) PROTOBUF_DEFINE_ACCESSOR(OneofDescriptor, containing_type, const Descriptor*) PROTOBUF_DEFINE_ACCESSOR(OneofDescriptor, field_count, int) +PROTOBUF_DEFINE_ARRAY_ACCESSOR(OneofDescriptor, field, const FieldDescriptor*) PROTOBUF_DEFINE_OPTIONS_ACCESSOR(OneofDescriptor, OneofOptions) -PROTOBUF_DEFINE_STRING_ACCESSOR(EnumDescriptor, name) -PROTOBUF_DEFINE_STRING_ACCESSOR(EnumDescriptor, full_name) +PROTOBUF_DEFINE_NAME_ACCESSOR(EnumDescriptor) PROTOBUF_DEFINE_ACCESSOR(EnumDescriptor, file, const FileDescriptor*) PROTOBUF_DEFINE_ACCESSOR(EnumDescriptor, containing_type, const Descriptor*) PROTOBUF_DEFINE_ACCESSOR(EnumDescriptor, value_count, int) @@ -1908,22 +2141,19 @@ PROTOBUF_DEFINE_ARRAY_ACCESSOR(EnumDescriptor, reserved_range, const EnumDescriptor::ReservedRange*) PROTOBUF_DEFINE_ACCESSOR(EnumDescriptor, reserved_name_count, int) -PROTOBUF_DEFINE_STRING_ACCESSOR(EnumValueDescriptor, name) -PROTOBUF_DEFINE_STRING_ACCESSOR(EnumValueDescriptor, full_name) +PROTOBUF_DEFINE_NAME_ACCESSOR(EnumValueDescriptor) PROTOBUF_DEFINE_ACCESSOR(EnumValueDescriptor, number, int) PROTOBUF_DEFINE_ACCESSOR(EnumValueDescriptor, type, const EnumDescriptor*) PROTOBUF_DEFINE_OPTIONS_ACCESSOR(EnumValueDescriptor, EnumValueOptions) -PROTOBUF_DEFINE_STRING_ACCESSOR(ServiceDescriptor, name) -PROTOBUF_DEFINE_STRING_ACCESSOR(ServiceDescriptor, full_name) +PROTOBUF_DEFINE_NAME_ACCESSOR(ServiceDescriptor) PROTOBUF_DEFINE_ACCESSOR(ServiceDescriptor, file, const FileDescriptor*) PROTOBUF_DEFINE_ACCESSOR(ServiceDescriptor, method_count, int) PROTOBUF_DEFINE_ARRAY_ACCESSOR(ServiceDescriptor, method, const MethodDescriptor*) PROTOBUF_DEFINE_OPTIONS_ACCESSOR(ServiceDescriptor, ServiceOptions) -PROTOBUF_DEFINE_STRING_ACCESSOR(MethodDescriptor, name) -PROTOBUF_DEFINE_STRING_ACCESSOR(MethodDescriptor, full_name) +PROTOBUF_DEFINE_NAME_ACCESSOR(MethodDescriptor) PROTOBUF_DEFINE_ACCESSOR(MethodDescriptor, service, const ServiceDescriptor*) PROTOBUF_DEFINE_OPTIONS_ACCESSOR(MethodDescriptor, MethodOptions) PROTOBUF_DEFINE_ACCESSOR(MethodDescriptor, client_streaming, bool) @@ -1955,17 +2185,21 @@ PROTOBUF_DEFINE_ARRAY_ACCESSOR(FileDescriptor, extension, // A few accessors differ from the macros... +inline Descriptor::WellKnownType Descriptor::well_known_type() const { + return static_cast(well_known_type_); +} + inline bool Descriptor::IsExtensionNumber(int number) const { - return FindExtensionRangeContainingNumber(number) != NULL; + return FindExtensionRangeContainingNumber(number) != nullptr; } inline bool Descriptor::IsReservedNumber(int number) const { - return FindReservedRangeContainingNumber(number) != NULL; + return FindReservedRangeContainingNumber(number) != nullptr; } -inline bool Descriptor::IsReservedName(const string& name) const { +inline bool Descriptor::IsReservedName(ConstStringParam name) const { for (int i = 0; i < reserved_name_count(); i++) { - if (name == reserved_name(i)) { + if (name == static_cast(reserved_name(i))) { return true; } } @@ -1974,17 +2208,17 @@ inline bool Descriptor::IsReservedName(const string& name) const { // Can't use PROTOBUF_DEFINE_ARRAY_ACCESSOR because reserved_names_ is actually // an array of pointers rather than the usual array of objects. -inline const string& Descriptor::reserved_name(int index) const { +inline const std::string& Descriptor::reserved_name(int index) const { return *reserved_names_[index]; } inline bool EnumDescriptor::IsReservedNumber(int number) const { - return FindReservedRangeContainingNumber(number) != NULL; + return FindReservedRangeContainingNumber(number) != nullptr; } -inline bool EnumDescriptor::IsReservedName(const string& name) const { +inline bool EnumDescriptor::IsReservedName(ConstStringParam name) const { for (int i = 0; i < reserved_name_count(); i++) { - if (name == reserved_name(i)) { + if (name == static_cast(reserved_name(i))) { return true; } } @@ -1993,15 +2227,45 @@ inline bool EnumDescriptor::IsReservedName(const string& name) const { // Can't use PROTOBUF_DEFINE_ARRAY_ACCESSOR because reserved_names_ is actually // an array of pointers rather than the usual array of objects. -inline const string& EnumDescriptor::reserved_name(int index) const { +inline const std::string& EnumDescriptor::reserved_name(int index) const { return *reserved_names_[index]; } +inline const std::string& FieldDescriptor::lowercase_name() const { + return all_names_[lowercase_name_index_]; +} + +inline const std::string& FieldDescriptor::camelcase_name() const { + return all_names_[camelcase_name_index_]; +} + +inline const std::string& FieldDescriptor::json_name() const { + return all_names_[json_name_index_]; +} + +inline const OneofDescriptor* FieldDescriptor::containing_oneof() const { + return is_oneof_ ? scope_.containing_oneof : nullptr; +} + +inline int FieldDescriptor::index_in_oneof() const { + GOOGLE_DCHECK(is_oneof_); + return static_cast(this - scope_.containing_oneof->field(0)); +} + +inline const Descriptor* FieldDescriptor::extension_scope() const { + GOOGLE_CHECK(is_extension_); + return scope_.extension_scope; +} + +inline FieldDescriptor::Label FieldDescriptor::label() const { + return static_cast

Image Classification Example

+

+ This tutorial shows you how to write an image classification example with OpenCV.js.
+ To try the example you should click the modelFile button(and configFile button if needed) to upload inference model. + You can find the model URLs and parameters in the
model info section. + Then You should change the parameters in the first code snippet according to the uploaded model. + Finally click Try it button to see the result. You can choose any other images.
+

+ +
+
+ + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+
+
+ canvasInput +
+
+
+ modelFile +
+
+
+ configFile +
+
+
+ +
+

+
+ +
+

Help function

+

1.The parameters for model inference which you can modify to investigate more models.

+ +

2.Main loop in which will read the image from canvas and do inference once.

+ +

3.Load labels from txt file and process it into an array.

+ +

4.Get blob from image as input for net, and standardize it with mean and std.

+ +

5.Fetch model file and save to emscripten file system once click the input button.

+ +

6.The post-processing, including softmax if needed and get the top classes from the output vector.

+ +
+ +
+

Model Info:

+
+ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/js_tutorials/js_assets/webnn-electron/js_image_classification_webnn_electron.html b/doc/js_tutorials/js_assets/webnn-electron/js_image_classification_webnn_electron.html new file mode 100644 index 0000000000..0da44330c9 --- /dev/null +++ b/doc/js_tutorials/js_assets/webnn-electron/js_image_classification_webnn_electron.html @@ -0,0 +1,268 @@ + + + + + + Image Classification Example + + + + +

Image Classification Example

+

+ This tutorial shows you how to write an image classification example with OpenCV.js.
+ To try the example you should click the modelFile button(and configFile button if needed) to upload inference model. + You can find the model URLs and parameters in the model info section. + Then You should change the parameters in the first code snippet according to the uploaded model. + Finally click Try it button to see the result. You can choose any other images.
+

+ +
+
+ + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+
+
+ canvasInput +
+
+
+ modelFile +
+
+
+ configFile +
+
+
+ +
+

+
+ +
+

Help function

+

1.The parameters for model inference which you can modify to investigate more models.

+ +

2.Main loop in which will read the image from canvas and do inference once.

+ +

3.Load labels from txt file and process it into an array.

+ +

4.Get blob from image as input for net, and standardize it with mean and std.

+ +

5.Fetch model file and save to emscripten file system once click the input button.

+ +

6.The post-processing, including softmax if needed and get the top classes from the output vector.

+ +
+ +
+

Model Info:

+
+ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/js_tutorials/js_assets/webnn-electron/main.js b/doc/js_tutorials/js_assets/webnn-electron/main.js new file mode 100644 index 0000000000..b73878b07e --- /dev/null +++ b/doc/js_tutorials/js_assets/webnn-electron/main.js @@ -0,0 +1,56 @@ +// Modules to control application life and create native browser window +const {app, BrowserWindow} = require('electron') +const path = require('path') + +// Keep a global reference of the window object, if you don't, the window will +// be closed automatically when the JavaScript object is garbage collected. +let mainWindow = {} + +function createWindow() { + // Create the browser window. + mainWindow = new BrowserWindow({ + width: 1220, + height: 840, + webPreferences: { + nodeIntegration: true, + contextIsolation: false, + preload: app.getAppPath()+"/node_setup.js" + } + }) + + // Load the index.html with 'numRunsParm' to run inference multiple times. + let url = `file://${__dirname}/js_image_classification_webnn_electron.html` + const numRunsParm = '?' + process.argv[2] + mainWindow.loadURL(url + numRunsParm) + + // Emitted when the window is closed. + mainWindow.on('closed', function() { + // Dereference the window object, usually you would store windows + // in an array if your app supports multi windows, this is the time + // when you should delete the corresponding element. + mainWindow = null + }) +} + +// This method will be called when Electron has finished +// initialization and is ready to create browser windows. +// Some APIs can only be used after this event occurs. +app.on('ready', createWindow) + +// Quit when all windows are closed. +app.on('window-all-closed', function() { + // On macOS it is common for applications and their menu bar + // to stay active until the user quits explicitly with Cmd + Q + if (process.platform !== 'darwin') app.quit() +}) + +app.on( + 'activate', + function() { + // On macOS it's common to re-create a window in the app when the + // dock icon is clicked and there are no other windows open. + if (mainWindow === null) createWindow() + }) + + // In this file you can include the rest of your app's specific main process + // code. You can also put them in separate files and require them here. diff --git a/doc/js_tutorials/js_assets/webnn-electron/node_setup.js b/doc/js_tutorials/js_assets/webnn-electron/node_setup.js new file mode 100644 index 0000000000..c269b5ce94 --- /dev/null +++ b/doc/js_tutorials/js_assets/webnn-electron/node_setup.js @@ -0,0 +1,12 @@ +const cv = require('./opencv'); +const webnn = require(process.env.WEBNN_NATIVE_DIR+'/../../node/lib/webnn'); +// navigator is undefined in node.js, but defined in electron.js. +if (global.navigator === undefined) { + global.navigator = {}; +} +global.navigator.ml = webnn.ml; +global.MLContext = webnn.MLContext +global.MLGraphBuilder = webnn.MLGraphBuilder +global.MLGraph = webnn.MLGraph +global.MLOperand = webnn.MLOperand +global.cv = cv; \ No newline at end of file diff --git a/doc/js_tutorials/js_assets/webnn-electron/package.json b/doc/js_tutorials/js_assets/webnn-electron/package.json new file mode 100644 index 0000000000..e6a258ee40 --- /dev/null +++ b/doc/js_tutorials/js_assets/webnn-electron/package.json @@ -0,0 +1,14 @@ +{ + "name": "image_classification", + "version": "0.0.1", + "description": "An Electon.js example of image_classification using webnn-native", + "main": "main.js", + "author": "WebNN-native Authors", + "license": "Apache-2.0", + "scripts": { + "start": "electron ." + }, + "dependencies": { + "electron": "^15.1.2" + } +} diff --git a/doc/js_tutorials/js_assets/webnn-electron/utils_webnn_electron.js b/doc/js_tutorials/js_assets/webnn-electron/utils_webnn_electron.js new file mode 100644 index 0000000000..2c4e981715 --- /dev/null +++ b/doc/js_tutorials/js_assets/webnn-electron/utils_webnn_electron.js @@ -0,0 +1,159 @@ +function Utils(errorOutputId) { // eslint-disable-line no-unused-vars + let self = this; + this.errorOutput = document.getElementById(errorOutputId); + + const OPENCV_URL = 'opencv.js'; + this.loadOpenCv = async function(onloadCallback) { + if (cv.getBuildInformation) + { + console.log(cv.getBuildInformation()); + onloadCallback(); + } + else + { + // WASM + if (cv instanceof Promise) { + cv = await cv; + console.log(cv.getBuildInformation()); + onloadCallback(); + } else { + cv['onRuntimeInitialized']=()=>{ + console.log(cv.getBuildInformation()); + onloadCallback(); + } + } + } + }; + + this.createFileFromUrl = function(path, url, callback) { + let request = new XMLHttpRequest(); + request.open('GET', url, true); + request.responseType = 'arraybuffer'; + request.onload = function(ev) { + if (request.readyState === 4) { + if (request.status === 200) { + let data = new Uint8Array(request.response); + cv.FS_createDataFile('/', path, data, true, false, false); + callback(); + } else { + self.printError('Failed to load ' + url + ' status: ' + request.status); + } + } + }; + request.send(); + }; + + this.loadImageToCanvas = function(url, cavansId) { + let canvas = document.getElementById(cavansId); + let ctx = canvas.getContext('2d'); + let img = new Image(); + img.crossOrigin = 'anonymous'; + img.onload = function() { + canvas.width = img.width; + canvas.height = img.height; + ctx.drawImage(img, 0, 0, img.width, img.height); + }; + img.src = url; + }; + + this.executeCode = function(textAreaId) { + try { + this.clearError(); + let code = document.getElementById(textAreaId).value; + eval(code); + } catch (err) { + this.printError(err); + } + }; + + this.clearError = function() { + this.errorOutput.innerHTML = ''; + }; + + this.printError = function(err) { + if (typeof err === 'undefined') { + err = ''; + } else if (typeof err === 'number') { + if (!isNaN(err)) { + if (typeof cv !== 'undefined') { + err = 'Exception: ' + cv.exceptionFromPtr(err).msg; + } + } + } else if (typeof err === 'string') { + let ptr = Number(err.split(' ')[0]); + if (!isNaN(ptr)) { + if (typeof cv !== 'undefined') { + err = 'Exception: ' + cv.exceptionFromPtr(ptr).msg; + } + } + } else if (err instanceof Error) { + err = err.stack.replace(/\n/g, '
'); + } + this.errorOutput.innerHTML = err; + }; + + this.loadCode = function(scriptId, textAreaId) { + let scriptNode = document.getElementById(scriptId); + let textArea = document.getElementById(textAreaId); + if (scriptNode.type !== 'text/code-snippet') { + throw Error('Unknown code snippet type'); + } + textArea.value = scriptNode.text.replace(/^\n/, ''); + }; + + this.addFileInputHandler = function(fileInputId, canvasId) { + let inputElement = document.getElementById(fileInputId); + inputElement.addEventListener('change', (e) => { + let files = e.target.files; + if (files.length > 0) { + let imgUrl = URL.createObjectURL(files[0]); + self.loadImageToCanvas(imgUrl, canvasId); + } + }, false); + }; + + function onVideoCanPlay() { + if (self.onCameraStartedCallback) { + self.onCameraStartedCallback(self.stream, self.video); + } + }; + + this.startCamera = function(resolution, callback, videoId) { + const constraints = { + 'qvga': {width: {exact: 320}, height: {exact: 240}}, + 'vga': {width: {exact: 640}, height: {exact: 480}}}; + let video = document.getElementById(videoId); + if (!video) { + video = document.createElement('video'); + } + + let videoConstraint = constraints[resolution]; + if (!videoConstraint) { + videoConstraint = true; + } + + navigator.mediaDevices.getUserMedia({video: videoConstraint, audio: false}) + .then(function(stream) { + video.srcObject = stream; + video.play(); + self.video = video; + self.stream = stream; + self.onCameraStartedCallback = callback; + video.addEventListener('canplay', onVideoCanPlay, false); + }) + .catch(function(err) { + self.printError('Camera Error: ' + err.name + ' ' + err.message); + }); + }; + + this.stopCamera = function() { + if (this.video) { + this.video.pause(); + this.video.srcObject = null; + this.video.removeEventListener('canplay', onVideoCanPlay); + } + if (this.stream) { + this.stream.getVideoTracks()[0].stop(); + } + }; +}; diff --git a/doc/js_tutorials/js_setup/js_setup/js_setup.markdown b/doc/js_tutorials/js_setup/js_setup/js_setup.markdown index ad14185a35..9927477443 100644 --- a/doc/js_tutorials/js_setup/js_setup/js_setup.markdown +++ b/doc/js_tutorials/js_setup/js_setup/js_setup.markdown @@ -145,6 +145,12 @@ Building OpenCV.js from Source python ./platforms/js/build_js.py build_js --cmake_option="-DOPENCV_EXTRA_MODULES_PATH=opencv_contrib/modules" @endcode +-# [optional] To enable WebNN backend, append `--webnn` option. + + For example: + @code{.bash} + emcmake python ./opencv/platforms/js/build_js.py build_js --webnn + @endcode Running OpenCV.js Tests --------------------------------------- diff --git a/modules/dnn/CMakeLists.txt b/modules/dnn/CMakeLists.txt index 398ed5ba48..88a8546aa0 100644 --- a/modules/dnn/CMakeLists.txt +++ b/modules/dnn/CMakeLists.txt @@ -19,6 +19,10 @@ if(OPENCV_DNN_OPENCL AND HAVE_OPENCL) add_definitions(-DCV_OCL4DNN=1) endif() +if(WITH_WEBNN AND HAVE_WEBNN) + add_definitions(-DHAVE_WEBNN=1) +endif() + ocv_option(OPENCV_DNN_CUDA "Build with CUDA support" HAVE_CUDA AND HAVE_CUBLAS @@ -142,6 +146,16 @@ if(HAVE_TENGINE) list(APPEND libs -Wl,--whole-archive ${TENGINE_LIBRARIES} -Wl,--no-whole-archive) endif() +set(webnn_srcs "") +if(NOT EMSCRIPTEN) + if(HAVE_WEBNN) + list(APPEND include_dirs ${WEBNN_HEADER_DIRS}) + list(APPEND include_dirs ${WEBNN_INCLUDE_DIRS}) + list(APPEND libs -Wl,--whole-archive ${WEBNN_LIBRARIES} -Wl,--no-whole-archive) + list(APPEND webnn_srcs $ENV{WEBNN_NATIVE_DIR}/gen/src/webnn/webnn_cpp.cpp) + endif() +endif() + ocv_module_include_directories(${include_dirs}) if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") ocv_append_source_files_cxx_compiler_options(fw_srcs "-Wno-suggest-override") # GCC @@ -171,7 +185,7 @@ if(HAVE_NGRAPH) list(APPEND dnn_runtime_libs ngraph::ngraph) endif() -ocv_glob_module_sources(${sources_options} SOURCES ${fw_srcs}) +ocv_glob_module_sources(${sources_options} SOURCES ${fw_srcs} ${webnn_srcs}) ocv_create_module(${libs} ${dnn_runtime_libs}) ocv_add_samples() ocv_add_accuracy_tests(${dnn_runtime_libs}) diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index 8b4e5e21d4..d6b29cfcf3 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -74,6 +74,7 @@ CV__DNN_INLINE_NS_BEGIN DNN_BACKEND_OPENCV, DNN_BACKEND_VKCOM, DNN_BACKEND_CUDA, + DNN_BACKEND_WEBNN, #ifdef __OPENCV_BUILD DNN_BACKEND_INFERENCE_ENGINE_NGRAPH = 1000000, // internal - use DNN_BACKEND_INFERENCE_ENGINE + setInferenceEngineBackendType() DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019, // internal - use DNN_BACKEND_INFERENCE_ENGINE + setInferenceEngineBackendType() @@ -307,6 +308,8 @@ CV__DNN_INLINE_NS_BEGIN virtual Ptr initVkCom(const std::vector > &inputs); + virtual Ptr initWebnn(const std::vector > &inputs, const std::vector >& nodes); + /** * @brief Returns a CUDA backend node * diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index 62607e247a..a76e81e8fb 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -45,6 +45,7 @@ #include "ie_ngraph.hpp" #include "op_vkcom.hpp" #include "op_cuda.hpp" +#include "op_webnn.hpp" #ifdef HAVE_CUDA #include "cuda4dnn/init.hpp" @@ -224,6 +225,13 @@ private: #endif #endif // HAVE_INF_ENGINE +#ifdef HAVE_WEBNN + if (haveWebnn()) + { + backends.push_back(std::make_pair(DNN_BACKEND_WEBNN, DNN_TARGET_CPU)); + } +#endif // HAVE_WEBNN + #ifdef HAVE_OPENCL if (cv::ocl::useOpenCL()) { @@ -1114,6 +1122,14 @@ static Ptr wrapMat(int backendId, int targetId, cv::Mat& m) return Ptr(new NgraphBackendWrapper(targetId, m)); #else CV_Error(Error::StsNotImplemented, "This OpenCV version is built without support of Inference Engine + nGraph"); +#endif + } + else if (backendId == DNN_BACKEND_WEBNN) + { +#ifdef HAVE_WEBNN + return Ptr(new WebnnBackendWrapper(targetId, m)); +#else + CV_Error(Error::StsNotImplemented, "This OpenCV version is built without support of WebNN"); #endif } else if (backendId == DNN_BACKEND_VKCOM) @@ -1259,6 +1275,12 @@ struct Net::Impl : public detail::NetImplBase { return wrapMat(preferableBackend, preferableTarget, host); } + else if (preferableBackend == DNN_BACKEND_WEBNN) + { +#ifdef HAVE_WEBNN + return wrapMat(preferableBackend, preferableTarget, host); +#endif + } else if (preferableBackend == DNN_BACKEND_VKCOM) { #ifdef HAVE_VULKAN @@ -1399,6 +1421,13 @@ struct Net::Impl : public detail::NetImplBase preferableTarget == DNN_TARGET_FPGA ); } +#endif +#ifdef HAVE_WEBNN + if (preferableBackend == DNN_BACKEND_WEBNN) + { + CV_Assert(preferableTarget == DNN_TARGET_CPU || + preferableTarget == DNN_TARGET_OPENCL); + } #endif CV_Assert(preferableBackend != DNN_BACKEND_VKCOM || preferableTarget == DNN_TARGET_VULKAN); @@ -1622,6 +1651,14 @@ struct Net::Impl : public detail::NetImplBase initNgraphBackend(blobsToKeep_); #else CV_Error(Error::StsNotImplemented, "This OpenCV version is built without support of Inference Engine + nGraph"); +#endif + } + else if (preferableBackend == DNN_BACKEND_WEBNN) + { +#ifdef HAVE_WEBNN + initWebnnBackend(blobsToKeep_); +#else + CV_Error(Error::StsNotImplemented, "This OpenCV version is built without support of WebNN"); #endif } else if (preferableBackend == DNN_BACKEND_VKCOM) @@ -2341,6 +2378,270 @@ struct Net::Impl : public detail::NetImplBase } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + void addWebnnOutputs(LayerData &ld) + { + CV_TRACE_FUNCTION(); + + Ptr layerNet; + auto it = ld.backendNodes.find(preferableBackend); + if (it != ld.backendNodes.end()) + { + Ptr node = it->second; + if (!node.empty()) + { + Ptr webnnNode = node.dynamicCast(); + CV_Assert(!webnnNode.empty()); CV_Assert(!webnnNode->net.empty()); + layerNet = webnnNode->net; + } + } + + for (int i = 0; i < ld.inputBlobsId.size(); ++i) + { + LayerData &inpLd = layers[ld.inputBlobsId[i].lid]; + Ptr inpNode = inpLd.backendNodes[preferableBackend]; + if (!inpNode.empty()) + { + Ptr webnnInpNode = inpNode.dynamicCast(); + CV_Assert(!webnnInpNode.empty()); CV_Assert(!webnnInpNode->net.empty()); + if (layerNet != webnnInpNode->net) + { + webnnInpNode->net->addOutput(webnnInpNode->name); + webnnInpNode->net->setUnconnectedNodes(webnnInpNode); + } + } + } + } + + void initWebnnBackend(const std::vector& blobsToKeep_) + { + CV_TRACE_FUNCTION(); + CV_Assert_N(preferableBackend == DNN_BACKEND_WEBNN, haveWebnn()); + + MapIdToLayerData::iterator it; + Ptr net; + + for (it = layers.begin(); it != layers.end(); ++it) + { + LayerData &ld = it->second; + if (ld.id == 0) + { + CV_Assert((netInputLayer->outNames.empty() && ld.outputBlobsWrappers.size() == 1) || + (netInputLayer->outNames.size() == ld.outputBlobsWrappers.size())); + for (int i = 0; i < ld.outputBlobsWrappers.size(); ++i) + { + Ptr wrapper = ld.outputBlobsWrappers[i].dynamicCast(); + std::string outputName = netInputLayer->outNames.empty() ? ld.name : netInputLayer->outNames[i]; + outputName = ld.outputBlobsWrappers.size() > 1 ? (outputName + "." + std::to_string(i)) : outputName; + wrapper->name = outputName; + } + } + else + { + for (int i = 0; i < ld.outputBlobsWrappers.size(); ++i) + { + Ptr wrapper = ld.outputBlobsWrappers[i].dynamicCast(); + std::string outputName = ld.outputBlobsWrappers.size() > 1 ? (ld.name + "." + std::to_string(i)) : ld.name; + wrapper->name = outputName; + } + } + } + + // Build WebNN networks from sets of layers that support this + // backend. Split a whole model on several WebNN networks if + // some of layers are not implemented. + for (it = layers.begin(); it != layers.end(); ++it) + { + LayerData &ld = it->second; + + if (ld.id == 0 && ld.skip) + continue; + + bool fused = ld.skip; + Ptr layer = ld.layerInstance; + if (!fused && !layer->supportBackend(preferableBackend)) + { + // For test use. when not using WebNN, the test case will fail + // with the following code. + CV_LOG_WARNING(NULL, "Layer " + ld.type + " name " + ld.name + " is unsupported by WebNN backend."); + + addWebnnOutputs(ld); + net = Ptr(); + layer->preferableTarget = DNN_TARGET_CPU; + + for (int i = 0; i < ld.inputBlobsId.size(); ++i) + { + LayerData &inpLd = layers[ld.inputBlobsId[i].lid]; + Ptr inpNode = inpLd.backendNodes[preferableBackend]; + if (!inpNode.empty()) { + Ptr webnnNode = inpNode.dynamicCast(); + CV_Assert(!webnnNode.empty()); + webnnNode->net->setUnconnectedNodes(webnnNode); + } + } + continue; + } + ld.skip = true; // Initially skip all WebNN supported layers. + + // Create a new network if one of inputs from different WebNN graph. + std::vector> inputNodes; + for (int i = 0; i < ld.inputBlobsId.size(); ++i) + { + // Layer_Test_ROIPooling.Accuracy has 2 inputs inpLD = 0, 0 -> has 4 inputNodes (input, rois, input, rois) + if (inputNodes.size() == ld.inputBlobsId.size()) { + break; + } + LayerData &inpLd = layers[ld.inputBlobsId[i].lid]; + Ptr inpNode = inpLd.backendNodes[preferableBackend]; + if (!inpNode.empty()) + { + Ptr webnnInpNode = inpNode.dynamicCast(); + CV_Assert(!webnnInpNode.empty()); CV_Assert(!webnnInpNode->net.empty()); + if (webnnInpNode->net == net && !fused) { + inputNodes.push_back(inpNode); + continue; + } + } + + if (net.empty()) { + net = Ptr(new WebnnNet()); + } + + if (!fused) { + std::vector inputNames; + std::vector inputs; + + auto curr_pos = inpLd.consumers.begin(); + auto compare = [&ld] (const LayerPin& lp) { return lp.lid == ld.id; }; + auto cons = curr_pos; + while ((cons = std::find_if(curr_pos, inpLd.consumers.end(), compare)) != + inpLd.consumers.end()) { + int cons_inp = cons->oid; + Ptr inpWrapper = inpLd.outputBlobsWrappers[cons_inp]. + dynamicCast(); + CV_Assert(!inpWrapper.empty()); + auto iter = std::find(inputNames.begin(), inputNames.end(), + inpWrapper->name); + if (iter == inputNames.end()) { + inputNames.push_back(inpWrapper->name); + inputs.push_back(inpLd.outputBlobs[cons_inp]); + } + curr_pos = cons + 1; + } + + auto inps = net->setInputs(inputs, inputNames); + for (auto& inp : inps) { + WebnnBackendNode* node = new WebnnBackendNode(inp); + node->net = net; + inputNodes.emplace_back(Ptr(node)); + } + } + } + + Ptr node; + if (!net.empty()) + { + if (fused) + { + bool inPlace = ld.inputBlobsId.size() == 1 && ld.outputBlobs.size() == 1 && + ld.inputBlobs[0]->data == ld.outputBlobs[0].data; + CV_Assert(inPlace); + node = layers[ld.inputBlobsId[0].lid].backendNodes[preferableBackend]; + ld.inputBlobsWrappers = layers[ld.inputBlobsId[0].lid].inputBlobsWrappers; + } + } + else { + net = Ptr(new WebnnNet()); + } + + if (!fused) + { + CV_Assert(ld.inputBlobsId.size() == inputNodes.size()); + for (int i = 0; i < ld.inputBlobsId.size(); ++i) + { + int lid = ld.inputBlobsId[i].lid; + int oid = ld.inputBlobsId[i].oid; + if (oid == 0 || lid == 0) + continue; + + auto webnnInpNode = inputNodes[i].dynamicCast(); + inputNodes[i] = Ptr(new WebnnBackendNode(webnnInpNode->operand)); + } + + if (layer->supportBackend(preferableBackend)) + { + if (ld.type == "Const") { + ml::Operand fake_operand; + Ptr fake_input_node = Ptr(new WebnnBackendNode(fake_operand)); + fake_input_node->net = net; + inputNodes.push_back(fake_input_node); + } + node = layer->initWebnn(ld.inputBlobsWrappers, inputNodes); + for (int i = 0; i < ld.outputBlobsWrappers.size(); ++i) + { + Ptr wrapper = ld.outputBlobsWrappers[i].dynamicCast(); + node.dynamicCast()->name = wrapper->name; + } + } + else + { + continue; + } + } + else if (node.empty()) + continue; + + ld.backendNodes[preferableBackend] = node; + + Ptr webnnNode = node.dynamicCast(); + CV_Assert(!webnnNode.empty()); + webnnNode->net = net; + + if (ld.consumers.empty()) { + // TF EAST_text_detection + webnnNode->net->setUnconnectedNodes(webnnNode); + } + for (const auto& pin : blobsToKeep_) + { + if (pin.lid == ld.id) + { + webnnNode->net->addOutput(webnnNode->name); + break; + } + } + net->addBlobs(ld.inputBlobsWrappers); + net->addBlobs(ld.outputBlobsWrappers); + addWebnnOutputs(ld); + } + + // Initialize all networks. + for (MapIdToLayerData::reverse_iterator it = layers.rbegin(); it != layers.rend(); ++it) + { + LayerData &ld = it->second; + auto iter = ld.backendNodes.find(preferableBackend); + if (iter == ld.backendNodes.end()) + continue; + + Ptr& node = iter->second; + if (node.empty()) + continue; + + Ptr webnnNode = node.dynamicCast(); + if (webnnNode.empty()) + continue; + + CV_Assert(!webnnNode->net.empty()); + + if (!webnnNode->net->isInitialized()) + { + webnnNode->net->setUnconnectedNodes(webnnNode); + webnnNode->net->createNet((Target)preferableTarget); + ld.skip = false; + } + } + } +#endif + void initVkComBackend() { CV_TRACE_FUNCTION(); @@ -3393,6 +3694,10 @@ struct Net::Impl : public detail::NetImplBase else if (preferableBackend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) { forwardNgraph(ld.outputBlobsWrappers, node, isAsync); + } + else if (preferableBackend == DNN_BACKEND_WEBNN) + { + forwardWebnn(ld.outputBlobsWrappers, node, isAsync); } else if (preferableBackend == DNN_BACKEND_VKCOM) { @@ -4830,6 +5135,7 @@ string Net::Impl::dump() case DNN_BACKEND_OPENCV: backend = "OCV/"; break; case DNN_BACKEND_VKCOM: backend = "VULKAN/"; break; case DNN_BACKEND_CUDA: backend = "CUDA/"; break; + case DNN_BACKEND_WEBNN: backend = "WEBNN/"; break; // don't use default: } out << "digraph G {\n"; @@ -5421,6 +5727,13 @@ Ptr Layer::initNgraph(const std::vector > & inp return Ptr(); } +Ptr Layer::initWebnn(const std::vector > & inputs, const std::vector >& nodes) +{ + CV_Error(Error::StsNotImplemented, "WebNN pipeline of " + type + + " layers is not defined."); + return Ptr(); +} + void Layer::applyHalideScheduler(Ptr& node, const std::vector &inputs, const std::vector &outputs, int targetId) const { diff --git a/modules/dnn/src/layers/batch_norm_layer.cpp b/modules/dnn/src/layers/batch_norm_layer.cpp index 57a95b15ee..d22a070805 100644 --- a/modules/dnn/src/layers/batch_norm_layer.cpp +++ b/modules/dnn/src/layers/batch_norm_layer.cpp @@ -15,6 +15,7 @@ Implementation of Batch Normalization layer. #include "../op_halide.hpp" #include "../op_inf_engine.hpp" #include "../ie_ngraph.hpp" +#include "../op_webnn.hpp" #include @@ -172,6 +173,7 @@ public: return (backendId == DNN_BACKEND_OPENCV) || backendId == DNN_BACKEND_CUDA || (backendId == DNN_BACKEND_HALIDE && haveHalide()) || + backendId == DNN_BACKEND_WEBNN || ((backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && haveInfEngine() && (preferableTarget == DNN_TARGET_CPU || dims == 4)); } @@ -421,6 +423,27 @@ public: return true; } +#ifdef HAVE_WEBNN + virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE + { + Ptr node = nodes[0].dynamicCast(); + auto& webnnInpOperand = node->operand; + auto& webnnGraphBuilder = node->net->builder; + std::vector weights_shape = webnn::getShape(weights_); + ml::Operand weights = webnn::BuildConstant(webnnGraphBuilder, weights_shape, weights_.data, weights_.total()*weights_.elemSize(), ml::OperandType::Float32); + std::vector shape(dims, 1); + shape[1] = weights_shape[1]; + ml::Operand weights_reshaped = webnnGraphBuilder.Reshape(weights, shape.data(), shape.size()); + ml::Operand mul_res = webnnGraphBuilder.Mul(webnnInpOperand, weights_reshaped); + std::vector bias_shape = webnn::getShape(bias_); + ml::Operand bias = webnn::BuildConstant(webnnGraphBuilder, bias_shape, bias_.data, bias_.total()*bias_.elemSize(), ml::OperandType::Float32); + shape[1] = bias_shape[1]; + ml::Operand bias_reshaped = webnnGraphBuilder.Reshape(bias, shape.data(), shape.size()); + ml::Operand add_res = webnnGraphBuilder.Add(mul_res, bias_reshaped); + return Ptr(new WebnnBackendNode(add_res)); + } +#endif + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { diff --git a/modules/dnn/src/layers/concat_layer.cpp b/modules/dnn/src/layers/concat_layer.cpp index 536114fcd7..f620d66a39 100644 --- a/modules/dnn/src/layers/concat_layer.cpp +++ b/modules/dnn/src/layers/concat_layer.cpp @@ -47,6 +47,7 @@ #include "../op_inf_engine.hpp" #include "../ie_ngraph.hpp" #include "../op_vkcom.hpp" +#include "../op_webnn.hpp" #ifdef HAVE_OPENCL #include "opencl_kernels_dnn.hpp" @@ -117,6 +118,7 @@ public: (backendId == DNN_BACKEND_HALIDE && haveHalide() && axis == 1 && !padding) || // By channels (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && haveInfEngine() && !padding) || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH || + (backendId == DNN_BACKEND_WEBNN && !padding) || (backendId == DNN_BACKEND_VKCOM && haveVulkan() && !padding); } @@ -408,6 +410,22 @@ public: params.set("padding_value", zeropoints[1][0]); return true; } + +#ifdef HAVE_WEBNN + virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE + { + Ptr node = nodes[0].dynamicCast(); + auto& webnnGraphBuilder = node->net->builder; + std::vector inputsOperand; + for (int i = 0; i < nodes.size(); i++) + { + inputsOperand.push_back(nodes[i].dynamicCast()->operand); + } + auto operand = webnnGraphBuilder.Concat(inputsOperand.size(), inputsOperand.data(), axis); + return Ptr(new WebnnBackendNode(operand)); + } +#endif + }; Ptr ConcatLayer::create(const LayerParams& params) diff --git a/modules/dnn/src/layers/const_layer.cpp b/modules/dnn/src/layers/const_layer.cpp index 18f190b36b..1f307b8fa6 100644 --- a/modules/dnn/src/layers/const_layer.cpp +++ b/modules/dnn/src/layers/const_layer.cpp @@ -10,6 +10,7 @@ #include "../op_cuda.hpp" #include "layers_common.hpp" #include "../ie_ngraph.hpp" +#include "../op_webnn.hpp" #ifdef HAVE_OPENCL #include "opencl_kernels_dnn.hpp" @@ -36,6 +37,7 @@ public: return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH || + backendId == DNN_BACKEND_WEBNN || backendId == DNN_BACKEND_CUDA; } @@ -97,6 +99,16 @@ public: } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE + { + ml::Operand operand = nullptr; + Ptr node = nodes[0].dynamicCast(); + auto& webnnGraphBuilder = node->net->builder; + operand = webnn::BuildConstant(webnnGraphBuilder, webnn::getShape(blobs[0]), blobs[0].data, blobs[0].total()*blobs[0].elemSize(), ml::OperandType::Float32); + return Ptr(new WebnnBackendNode(operand)); + } +#endif #ifdef HAVE_CUDA Ptr initCUDA( diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index bb6fd5f5cc..50b06a19c0 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -47,6 +47,7 @@ #include "../op_inf_engine.hpp" #include "../ie_ngraph.hpp" #include "../op_vkcom.hpp" +#include "../op_webnn.hpp" #include #include @@ -80,6 +81,9 @@ class BaseConvolutionLayerImpl : public ConvolutionLayer public: bool fusedWeights, fusedBias; std::vector weightsMultipliers; +#ifdef HAVE_WEBNN + int groups; +#endif BaseConvolutionLayerImpl(const LayerParams ¶ms) { setParamsFrom(params); @@ -87,6 +91,9 @@ public: numOutput = params.get("num_output"); int ngroups = params.get("group", 1); +#ifdef HAVE_WEBNN + groups = ngroups; +#endif CV_Assert(numOutput % ngroups == 0); if (kernel_size.size() == 2) { @@ -347,6 +354,17 @@ public: #ifdef HAVE_VULKAN if (backendId == DNN_BACKEND_VKCOM) return ksize == 2; +#endif +#ifdef HAVE_WEBNN + if (backendId == DNN_BACKEND_WEBNN) + { + if (ksize != 2) + { + CV_LOG_WARNING(NULL, "WebNN only supports Conv2d."); + return false; + } + return true; + } #endif return false; } @@ -896,6 +914,108 @@ public: } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE + { + CV_Assert_N(inputs.size() >= 1, nodes.size() >= 1); + Ptr node = nodes[0].dynamicCast(); + auto& webnnInpOperand = node->operand; + auto& webnnGraphBuilder = node->net->builder; + ml::Operand webnnWeights = nodes.size() > 1 ? nodes[1].dynamicCast()->operand : nullptr; + if (nodes.size() > 1) + CV_Assert(webnnWeights); + const int inpCn = weightsMat.total()/(kernel_size[0]*kernel_size[1]*numOutput); + const int group = groups; + const int inpGroupCn = inpCn / group; + std::vector kernel_shape; + if (group != 1) + { + kernel_shape.push_back(group); + } + kernel_shape.push_back(numOutput / group); + kernel_shape.push_back(inpGroupCn); + std::copy(kernel_size.begin(), kernel_size.end(), back_inserter(kernel_shape)); + + if (nodes.size() == 1) + { + webnnWeights = webnn::BuildConstant(webnnGraphBuilder, webnn::getShape(blobs[0]), blobs[0].data, blobs[0].total()*blobs[0].elemSize(), ml::OperandType::Float32); + if (fusedWeights) + { + if (weightsMat.isContinuous()) + { + webnnWeights = webnn::BuildConstant(webnnGraphBuilder, webnn::getShape(weightsMat), weightsMat.data, weightsMat.total()*weightsMat.elemSize(), ml::OperandType::Float32); + } + else + { + Mat newWeights; + Mat cvWeights = weightsMat.colRange(0, blobs[0].total() / numOutput); + cvWeights.copyTo(newWeights); + webnnWeights = webnn::BuildConstant(webnnGraphBuilder, webnn::getShape(newWeights), newWeights.data, newWeights.total()*newWeights.elemSize(), ml::OperandType::Float32); + } + } + } + else + { + webnnWeights = webnnGraphBuilder.Reshape(webnnWeights, kernel_shape.data(), kernel_shape.size()); + } + + ml::AutoPad pad_type = ml::AutoPad::Explicit; + if (!padMode.empty()) + pad_type = padMode == "VALID" ? ml::AutoPad::Explicit : ml::AutoPad::SameUpper; + + ml::Conv2dOptions options = {}; + options.groups = group; + options.autoPad = pad_type; + std::vector Strides(strides.begin(), strides.end()); + if (!Strides.empty()) + { + options.stridesCount = Strides.size(); + options.strides = Strides.data(); + } + std::vector Padding; + if (padMode.empty()) + { + Padding = {static_cast(pads_begin[0]), + static_cast(pads_end[0]), + static_cast(pads_begin[1]), + static_cast(pads_end[1])}; + } + else if (padMode == "VALID") + { + Padding = {0, 0, 0, 0}; + } + if (!Padding.empty()) + { + options.paddingCount = Padding.size(); + options.padding = Padding.data(); + } + std::vector Dilations(dilations.begin(), dilations.end()); + if (!Dilations.empty()) + { + options.dilationsCount = Dilations.size(); + options.dilations = Dilations.data(); + } + ml::Operand operand = webnnGraphBuilder.Conv2d(webnnInpOperand, webnnWeights, &options); + + // ml::Operand result = operand; + if (hasBias() || fusedBias || nodes.size() == 3) + { + ml::Operand webnnBias = nullptr; + if (nodes.size() == 3) + { + std::vector bias_shape = {1, numOutput, 1, 1}; + webnnBias = webnnGraphBuilder.Reshape(nodes[2].dynamicCast()->operand, bias_shape.data(), bias_shape.size()); + } + else + { + webnnBias = webnn::BuildConstant(webnnGraphBuilder, {1, numOutput, 1, 1}, biasvec.data(), (numOutput) * sizeof(float), ml::OperandType::Float32); + } + operand = webnnGraphBuilder.Add(operand, webnnBias); + } + return Ptr(new WebnnBackendNode(operand)); + } +#endif // HAVE_WEBNN + class ParallelConv : public cv::ParallelLoopBody { public: diff --git a/modules/dnn/src/layers/elementwise_layers.cpp b/modules/dnn/src/layers/elementwise_layers.cpp index c95dbbc933..56e82cc3d1 100644 --- a/modules/dnn/src/layers/elementwise_layers.cpp +++ b/modules/dnn/src/layers/elementwise_layers.cpp @@ -47,8 +47,11 @@ #include "../op_inf_engine.hpp" #include "../ie_ngraph.hpp" #include "../op_vkcom.hpp" +#include "../op_webnn.hpp" #include +#include +#include #include #ifdef HAVE_OPENCL @@ -59,6 +62,7 @@ #include "../cuda4dnn/primitives/activation.hpp" using namespace cv::dnn::cuda4dnn; #endif +#include namespace cv { @@ -186,6 +190,17 @@ public: } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE + { + Ptr node = nodes[0].dynamicCast(); + auto& webnnInpOperand = node->operand; + auto& webnnGraphBuilder = node->net->builder; + auto operand = func.initWebnnAPI(webnnGraphBuilder, webnnInpOperand); + return Ptr(new WebnnBackendNode(operand)); + } +#endif + virtual Ptr initVkCom(const std::vector >& inputs) CV_OVERRIDE { #ifdef HAVE_VULKAN @@ -319,6 +334,16 @@ struct ReLUFunctor : public BaseFunctor #ifdef HAVE_DNN_NGRAPH if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) return true; +#endif +#ifdef HAVE_WEBNN + if (backendId == DNN_BACKEND_WEBNN) { + // TODO: support PRELU + if (slope != 0) + { + CV_LOG_WARNING(NULL, "PRELU is not supported now."); + } + return slope == 0; + } #endif return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA || @@ -441,6 +466,13 @@ struct ReLUFunctor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) + { + return builder.Relu(input); + } +#endif + #ifdef HAVE_VULKAN std::shared_ptr initVkCom() { @@ -491,6 +523,7 @@ struct ReLU6Functor : public BaseFunctor return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA || backendId == DNN_BACKEND_HALIDE || + backendId == DNN_BACKEND_WEBNN || backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; } @@ -587,6 +620,18 @@ struct ReLU6Functor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH + + +#ifdef HAVE_WEBNN + ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) + { + ml::ClampOptions clampOptions; + clampOptions.minValue = minValue; + clampOptions.maxValue = maxValue; + return builder.Clamp(input, &clampOptions); + } +#endif + #ifdef HAVE_VULKAN std::shared_ptr initVkCom() { @@ -684,6 +729,15 @@ struct BaseDefaultFunctor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) + { + CV_Error(Error::StsNotImplemented, ""); + ml::Operand operand; + return operand; + } +#endif + #ifdef HAVE_VULKAN std::shared_ptr initVkCom() { @@ -742,6 +796,15 @@ struct TanHFunctor : public BaseDefaultFunctor } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) + { + CV_Error(Error::StsNotImplemented, ""); + ml::Operand operand; + return operand; + } +#endif + int64 getFLOPSPerElement() const { return 1; } }; @@ -845,6 +908,15 @@ struct MishFunctor : public BaseDefaultFunctor } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) + { + CV_Error(Error::StsNotImplemented, ""); + ml::Operand operand; + return operand; + } +#endif + int64 getFLOPSPerElement() const { return 3; } }; @@ -897,6 +969,15 @@ struct SigmoidFunctor : public BaseDefaultFunctor } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) + { + CV_Error(Error::StsNotImplemented, ""); + ml::Operand operand; + return operand; + } +#endif + int64 getFLOPSPerElement() const { return 3; } }; @@ -1007,6 +1088,15 @@ struct AbsValFunctor : public BaseDefaultFunctor } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) + { + CV_Error(Error::StsNotImplemented, ""); + ml::Operand operand; + return operand; + } +#endif + int64 getFLOPSPerElement() const { return 1; } }; @@ -1136,6 +1226,15 @@ struct LogFunctor : public BaseDefaultFunctor return log(x); } +#ifdef HAVE_WEBNN + ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) + { + CV_Error(Error::StsNotImplemented, ""); + ml::Operand operand; + return operand; + } +#endif + #ifdef HAVE_CUDA Ptr initCUDA(int target, csl::Stream stream) { @@ -1233,6 +1332,15 @@ struct SqrtFunctor : public BaseDefaultFunctor } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) + { + CV_Error(Error::StsNotImplemented, ""); + ml::Operand operand; + return operand; + } +#endif + int64 getFLOPSPerElement() const { return 1; } }; @@ -1415,6 +1523,15 @@ struct PowerFunctor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) + { + CV_Error(Error::StsNotImplemented, ""); + ml::Operand operand; + return operand; + } +#endif + #ifdef HAVE_VULKAN std::shared_ptr initVkCom() { @@ -1517,6 +1634,15 @@ struct ExpFunctor : public BaseDefaultFunctor } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) + { + CV_Error(Error::StsNotImplemented, ""); + ml::Operand operand; + return operand; + } +#endif + int64 getFLOPSPerElement() const { return 3; } }; @@ -1649,6 +1775,15 @@ struct ChannelsPReLUFunctor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) + { + CV_Error(Error::StsNotImplemented, ""); + ml::Operand operand; + return operand; + } +#endif + #ifdef HAVE_VULKAN std::shared_ptr initVkCom() { diff --git a/modules/dnn/src/layers/fully_connected_layer.cpp b/modules/dnn/src/layers/fully_connected_layer.cpp index 28ea7f347f..1e8c9f5489 100644 --- a/modules/dnn/src/layers/fully_connected_layer.cpp +++ b/modules/dnn/src/layers/fully_connected_layer.cpp @@ -46,6 +46,7 @@ #include "../op_halide.hpp" #include "../op_inf_engine.hpp" #include "../ie_ngraph.hpp" +#include "../op_webnn.hpp" #include @@ -150,6 +151,7 @@ public: return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA || (backendId == DNN_BACKEND_HALIDE && haveHalide() && axis == 1) || + (backendId == DNN_BACKEND_WEBNN && axis == 1) || (((backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && !blobs.empty()) || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && axis == 1); } @@ -657,6 +659,40 @@ public: return true; } +#ifdef HAVE_WEBNN + virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE + { + Ptr node = nodes[0].dynamicCast(); + auto& webnnInpOperand = node->operand; + auto& webnnGraphBuilder = node->net->builder; + ml::GemmOptions gemmOptions = {}; + if (bias) + { + std::vector biasDims = {(int32_t)blobs[1].size[1]}; + ml::Operand bias = webnn::BuildConstant(webnnGraphBuilder, biasDims, blobs[1].data, blobs[1].total()*blobs[1].elemSize(), ml::OperandType::Float32); + gemmOptions.c = bias; + } + ml::Operand result = nullptr; + if (nodes.size() == 2) + { + auto& inp2 = nodes[1].dynamicCast()->operand; + result = webnnGraphBuilder.Gemm(webnnInpOperand, inp2, &gemmOptions); + } + else + { + std::vector input_shape(2, -1); + input_shape[1] = blobs[0].size[1]; + ml::Operand webnnInpOperand_reshaped = webnnGraphBuilder.Reshape(webnnInpOperand, input_shape.data(), input_shape.size()); + std::vector weight_shape = {(int32_t)blobs[0].size[0], (int32_t)blobs[0].size[1]}; + // std::cout<<"weight size: "<(new WebnnBackendNode(result)); + } +#endif // HAVE_WEBNN + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { diff --git a/modules/dnn/src/layers/permute_layer.cpp b/modules/dnn/src/layers/permute_layer.cpp index 77c2469c05..9e66eb6a64 100644 --- a/modules/dnn/src/layers/permute_layer.cpp +++ b/modules/dnn/src/layers/permute_layer.cpp @@ -46,6 +46,7 @@ #include "../op_inf_engine.hpp" #include "../ie_ngraph.hpp" #include "../op_vkcom.hpp" +#include "../op_webnn.hpp" #include #include @@ -119,6 +120,7 @@ public: #endif return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA || + backendId == DNN_BACKEND_WEBNN || ((backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && haveInfEngine()) || (backendId == DNN_BACKEND_VKCOM && haveVulkan()); } @@ -439,6 +441,20 @@ public: } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE + { + Ptr node = nodes[0].dynamicCast(); + auto& webnnInpOperand = node->operand; + auto& webnnGraphBuilder = node->net->builder; + std::vector permutation(_order.begin(), _order.end()); + ml::TransposeOptions options; + options.permutation = permutation.data(); + options.permutationCount = permutation.size(); + auto operand = webnnGraphBuilder.Transpose(webnnInpOperand, &options); + return Ptr(new WebnnBackendNode(operand)); + } +#endif #ifdef HAVE_CUDA Ptr initCUDA( diff --git a/modules/dnn/src/layers/pooling_layer.cpp b/modules/dnn/src/layers/pooling_layer.cpp index 7653e53668..0b9b94fa57 100644 --- a/modules/dnn/src/layers/pooling_layer.cpp +++ b/modules/dnn/src/layers/pooling_layer.cpp @@ -46,6 +46,7 @@ #include "../op_cuda.hpp" #include "../op_halide.hpp" #include "../op_inf_engine.hpp" +#include "../op_webnn.hpp" #ifdef HAVE_DNN_NGRAPH #include "../ie_ngraph.hpp" @@ -85,6 +86,7 @@ typedef int HALIDE_DIFF_T; #include "../cuda4dnn/primitives/max_unpooling.hpp" using namespace cv::dnn::cuda4dnn; #endif +#include namespace cv @@ -246,6 +248,51 @@ public: (type == MAX || type == AVE); return false; } + else if (backendId == DNN_BACKEND_WEBNN) + { + if (kernel_size.empty() || kernel_size.size() == 2) + { + if (!haveWebnn()) + { + return false; + } + else + { + if (!ceilMode) + { + CV_LOG_WARNING(NULL, "ceilMode is not supported by WebNN backend."); + return false; + } + if (computeMaxIdx) + { + CV_LOG_WARNING(NULL, "Mask is not supported by WebNN backend."); + return false; + } + if (type != MAX && type != AVE) + { + if (type == STOCHASTIC) + { + CV_LOG_WARNING(NULL, "Stochastic Pooling is not supported by WebNN backend."); + } + if (type == SUM) + { + CV_LOG_WARNING(NULL, "Sum Pooling is not supported by WebNN backend."); + } + if (type == ROI) + { + CV_LOG_WARNING(NULL, "ROI Pooling is not supported by WebNN backend."); + } + if (type == PSROI) + { + CV_LOG_WARNING(NULL, "Position-sensitive ROI Pooling is not supported by WebNN backend."); + } + CV_LOG_WARNING(NULL, "WebNN backend only supports MaxPooling and AveragePooling currently."); + return false; + } + } + return true; + } + } return false; } @@ -607,6 +654,45 @@ public: } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE + { + // std::cout << "Use WebNN Pooling Layer's Implementation." << std::endl; + Ptr node = nodes[0].dynamicCast(); + auto& webnnInpOperand = node->operand; + auto& webnnGraphBuilder = node->net->builder; + webnn::Pool2dOptions options; + std::vector kernelSize(kernel_size.begin(), kernel_size.end()); + std::vector Strides(strides.begin(), strides.end()); + std::vector Padding; + if (padMode.empty()) { + Padding = {static_cast(pads_begin[0]), + static_cast(pads_end[0]), + static_cast(pads_begin[1]), + static_cast(pads_end[1])}; + } else if (padMode == "VALID") { + Padding = {0, 0, 0, 0}; + } else if (padMode == "SAME") { + options.autoPad = ml::AutoPad::SameUpper; + } + // std::cout << "padMode: " << padMode << std::endl; + options.windowDimensions = kernelSize; + options.strides = Strides; + options.padding = Padding; + if (type == MAX) + { + auto operand = webnnGraphBuilder.MaxPool2d(webnnInpOperand, options.AsPtr()); + return Ptr(new WebnnBackendNode(operand)); + } + else if (type == AVE) + { + auto operand = webnnGraphBuilder.AveragePool2d(webnnInpOperand, options.AsPtr()); + return Ptr(new WebnnBackendNode(operand)); + } else { + CV_Error(Error::StsNotImplemented, "Unsupported pooling type"); + } + } +#endif // HAVE_WEBNN class PoolingInvoker : public ParallelLoopBody { diff --git a/modules/dnn/src/layers/reshape_layer.cpp b/modules/dnn/src/layers/reshape_layer.cpp index 4c10d155c8..0ba3abf047 100644 --- a/modules/dnn/src/layers/reshape_layer.cpp +++ b/modules/dnn/src/layers/reshape_layer.cpp @@ -45,6 +45,7 @@ #include "../op_cuda.hpp" #include "../op_inf_engine.hpp" #include "../ie_ngraph.hpp" +#include "../op_webnn.hpp" #include @@ -203,6 +204,7 @@ public: { return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA || + backendId == DNN_BACKEND_WEBNN || ((backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && haveInfEngine()); } @@ -330,6 +332,17 @@ public: } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE + { + Ptr node = nodes[0].dynamicCast(); + auto& webnnInpOperand = node->operand; + auto& webnnGraphBuilder = node->net->builder; + const std::vector out(outShapes[0].begin(), outShapes[0].end()); + auto operand = webnnGraphBuilder.Reshape(webnnInpOperand, out.data(), out.size()); + return Ptr(new WebnnBackendNode(operand)); + } +#endif #ifdef HAVE_CUDA Ptr initCUDA( diff --git a/modules/dnn/src/layers/scale_layer.cpp b/modules/dnn/src/layers/scale_layer.cpp index 003f78dc1d..fcee451556 100644 --- a/modules/dnn/src/layers/scale_layer.cpp +++ b/modules/dnn/src/layers/scale_layer.cpp @@ -15,6 +15,7 @@ Implementation of Scale layer. #include "../op_halide.hpp" #include "../op_inf_engine.hpp" #include "../ie_ngraph.hpp" +#include "../op_webnn.hpp" #include #include @@ -32,6 +33,10 @@ namespace dnn class ScaleLayerImpl CV_FINAL : public ScaleLayer { public: +#ifdef HAVE_WEBNN + mutable int dims; + mutable int numChannels; +#endif ScaleLayerImpl(const LayerParams& params) { setParamsFrom(params); @@ -47,6 +52,15 @@ public: std::vector &internals) const CV_OVERRIDE { outputs.assign(1, inputs[0]); +#ifdef HAVE_WEBNN + dims = inputs[0].size(); + numChannels = 1; + if (inputs.size() > 1) + { + for (const size_t& dim : inputs[1]) + numChannels *= dim; + } +#endif return true; } @@ -68,7 +82,8 @@ public: backendId == DNN_BACKEND_CUDA || backendId == DNN_BACKEND_HALIDE || (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && axis == 1 && !blobs.empty()) || - (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && axis > 0); + (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && axis > 0) || + (backendId == DNN_BACKEND_WEBNN && axis >0); } template @@ -375,6 +390,48 @@ public: } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE + { + Ptr node = nodes[0].dynamicCast(); + auto& webnnInpOperand0 = node->operand; + auto& webnnGraphBuilder = node->net->builder; + auto webnnInpOperand1 = nodes.size() > 1 ? nodes[1].dynamicCast()->operand : nullptr; + auto webnnInpOperand2 = nodes.size() > 2 ? nodes[1].dynamicCast()->operand : nullptr; + std::vector shape(dims, 1); + + size_t channels = 1; + if (blobs.empty()) + channels = numChannels; + else + channels = blobs[0].total(); + + int cAxis = normalize_axis(axis, shape.size()); + shape[cAxis] = channels; + + ml::Operand operand = webnnInpOperand0; + if (hasWeights) + { + ml::Operand webnnWeights = blobs.empty() ? webnnInpOperand1 : webnn::BuildConstant(webnnGraphBuilder, webnn::getShape(blobs[0]), blobs[0].data, blobs[0].total()*blobs[0].elemSize(), ml::OperandType::Float32); + webnnWeights = webnnGraphBuilder.Reshape(webnnWeights, shape.data(), shape.size()); + operand = webnnGraphBuilder.Mul(operand, webnnWeights); + } + if (hasBias) + { + ml::Operand webnnBias; + if(!hasWeights) + webnnBias = blobs.empty() ? webnnInpOperand1 : webnn::BuildConstant(webnnGraphBuilder, webnn::getShape(blobs.back()), blobs.back().data, blobs.back().total()*blobs.back().elemSize(), ml::OperandType::Float32); + else + webnnBias = blobs.empty() ? webnnInpOperand2 : webnn::BuildConstant(webnnGraphBuilder, webnn::getShape(blobs.back()), blobs.back().data, blobs.back().total()*blobs.back().elemSize(), ml::OperandType::Float32); + webnnBias = webnnGraphBuilder.Reshape(webnnBias, shape.data(), shape.size()); + operand = webnnGraphBuilder.Add(operand, webnnBias); + } + + return Ptr(new WebnnBackendNode(operand)); + } +#endif + + void getScaleShift(Mat& scale, Mat& shift) const CV_OVERRIDE { scale = (hasWeights && !blobs.empty()) ? blobs[0] : Mat(); diff --git a/modules/dnn/src/layers/softmax_layer.cpp b/modules/dnn/src/layers/softmax_layer.cpp index e937e98f8c..db2951808f 100644 --- a/modules/dnn/src/layers/softmax_layer.cpp +++ b/modules/dnn/src/layers/softmax_layer.cpp @@ -47,9 +47,11 @@ #include "../op_inf_engine.hpp" #include "../ie_ngraph.hpp" #include "../op_vkcom.hpp" +#include "../op_webnn.hpp" #include #include +#include using std::max; #ifdef HAVE_OPENCL @@ -97,6 +99,16 @@ public: virtual bool supportBackend(int backendId) CV_OVERRIDE { +#ifdef HAVE_WEBNN + if (backendId == DNN_BACKEND_WEBNN) { + // TODO: support logSoftMax + if (logSoftMax) + { + CV_LOG_WARNING(NULL, "logSoftMax is not supported by WebNN backend.") + } + return !logSoftMax; + } +#endif return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA || (backendId == DNN_BACKEND_HALIDE && haveHalide() && axisRaw == 1) || @@ -390,6 +402,18 @@ public: return true; } +#ifdef HAVE_WEBNN + virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE + { + Ptr node = nodes[0].dynamicCast(); + auto& webnnInpOperand = node->operand; + auto& webnnGraphBuilder = node->net->builder; + auto operand = webnnGraphBuilder.Softmax(webnnInpOperand); + return Ptr(new WebnnBackendNode(operand)); + } + +#endif + int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { diff --git a/modules/dnn/src/op_webnn.cpp b/modules/dnn/src/op_webnn.cpp new file mode 100644 index 0000000000..4dba55bcbe --- /dev/null +++ b/modules/dnn/src/op_webnn.cpp @@ -0,0 +1,249 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include +#include "op_webnn.hpp" + +#include +#include + +#include "opencv2/core/utils/filesystem.hpp" +#include "opencv2/core/utils/filesystem.private.hpp" + +#include + +namespace cv { namespace dnn { + +#ifdef HAVE_WEBNN + +namespace webnn { +ml::Operand BuildConstant(const ml::GraphBuilder& builder, + const std::vector& dimensions, + const void* value, + size_t size, + ml::OperandType type) { + ml::OperandDescriptor desc; + desc.type = type; + desc.dimensions = dimensions.data(); + desc.dimensionsCount = (uint32_t)dimensions.size(); + ml::ArrayBufferView resource; + resource.buffer = const_cast(value); + resource.byteLength = size; + return builder.Constant(&desc, &resource); + } +} + +static std::string kDefaultInpLayerName = "opencv_webnn_empty_inp_layer_name"; + +static std::vector > +webnnWrappers(const std::vector >& ptrs) +{ + std::vector > wrappers(ptrs.size()); + for (int i = 0; i < ptrs.size(); ++i) + { + CV_Assert(!ptrs[i].empty()); + wrappers[i] = ptrs[i].dynamicCast(); + CV_Assert(!wrappers[i].empty()); + } + return wrappers; +} + +// WebnnNet +WebnnNet::WebnnNet() +{ + hasNetOwner = false; + device_name = "CPU"; + +#ifdef __EMSCRIPTEN__ + context = ml::Context(emscripten_webnn_create_context()); +#else + WebnnProcTable backendProcs = webnn_native::GetProcs(); + webnnProcSetProcs(&backendProcs); + context = ml::Context(webnn_native::CreateContext()); +#endif + builder = ::ml::CreateGraphBuilder(context); + namedOperands = ::ml::CreateNamedOperands(); +} + +void WebnnNet::addOutput(const std::string& name) +{ + requestedOutputs.push_back(name); +} + +void WebnnNet::createNet(Target targetId) { + init(targetId); +} + +void WebnnNet::init(Target targetId) +{ + switch (targetId) + { + case DNN_TARGET_CPU: + device_name = "CPU"; + break; + case DNN_TARGET_OPENCL: + device_name = "GPU"; + break; + default: + CV_Error(Error::StsNotImplemented, "Unknown target"); + }; + + graph = builder.Build(namedOperands); + CV_Assert(graph!=nullptr); + isInit = true; +} + +std::vector WebnnNet::setInputs(const std::vector& inputs, + const std::vector& names) { + CV_Assert_N(inputs.size() == names.size()); + std::vector current_inp; + for (size_t i = 0; i < inputs.size(); i++) + { + auto& m = inputs[i]; + + std::vector dimensions = webnn::getShape(m); + ml::OperandDescriptor descriptor; + descriptor.dimensions = dimensions.data(); + descriptor.dimensionsCount = dimensions.size(); + if (m.type() == CV_32F) + { + descriptor.type = ml::OperandType::Float32; + } + else + { + CV_Error(Error::StsNotImplemented, format("Unsupported data type %s", typeToString(m.type()).c_str())); + } + ml::Operand inputOperand = builder.Input(names[i].c_str(), &descriptor); + current_inp.push_back(std::move(inputOperand)); + } + inputNames = names; + return current_inp; +} + +void WebnnNet::setUnconnectedNodes(Ptr& node) { + outputNames.push_back(node->name); + namedOperands.Set(outputNames.back().c_str(), node->operand); +} + +bool WebnnNet::isInitialized() +{ + return isInit; +} + +void WebnnNet::reset() +{ + allBlobs.clear(); + isInit = false; +} + +void WebnnNet::addBlobs(const std::vector >& ptrs) +{ + auto wrappers = webnnWrappers(ptrs); + for (const auto& wrapper : wrappers) + { + std::string name = wrapper->name; + name = name.empty() ? kDefaultInpLayerName : name; + allBlobs.insert({name, wrapper}); + } +} + +void WebnnNet::forward(const std::vector >& outBlobsWrappers, bool isAsync) +{ + CV_LOG_DEBUG(NULL, "WebnnNet::forward(" << (isAsync ? "async" : "sync") << ")"); + ml::NamedInputs named_inputs = ::ml::CreateNamedInputs(); + std::vector inputs(inputNames.size()); + for (int i = 0; i < inputNames.size(); ++i) { + const std::string& name = inputNames[i]; + ml::Input& input = inputs[i]; + auto blobIt = allBlobs.find(name); + CV_Assert(blobIt != allBlobs.end()); + const Ptr wrapper = blobIt->second; + input.resource.buffer = wrapper->host->data; + input.resource.byteLength = wrapper->size; + named_inputs.Set(name.c_str(), &input); + } + std::vector > outs = webnnWrappers(outBlobsWrappers); + ml::NamedOutputs named_outputs = ::ml::CreateNamedOutputs(); + std::vector outputs(outs.size()); + for (int i = 0; i < outs.size(); ++i) { + const std::string& name = outs[i]->name; + ml::ArrayBufferView& output = outputs[i]; + output.buffer = outs[i]->host->data; + // std::cout<<"host data size: "<host->total()*outs[i]->host->elemSize()<size; + // std::cout<<"outs[i]->size: "<< outs[i]->size << std::endl; + named_outputs.Set(name.c_str(), &output); + } + ml::ComputeGraphStatus status = graph.Compute(named_inputs, named_outputs); + if (status != ::ml::ComputeGraphStatus::Success) { + CV_Error(Error::StsAssert, format("Failed to compute: %d", int(status))); + } +} + +// WebnnBackendNode +WebnnBackendNode::WebnnBackendNode(ml::Operand&& _operand) + : BackendNode(DNN_BACKEND_WEBNN), operand(std::move(_operand)) {} + +WebnnBackendNode::WebnnBackendNode(ml::Operand& _operand) + : BackendNode(DNN_BACKEND_WEBNN), operand(_operand) {} + +// WebnnBackendWrapper +WebnnBackendWrapper::WebnnBackendWrapper(int targetId, cv::Mat& m) + : BackendWrapper(DNN_BACKEND_WEBNN, targetId) +{ + size = m.total() * m.elemSize(); + // buffer.reset(new char[size]); + // std::memcpy(buffer.get(), m.data, size); + // dimensions = getShape(m); + // descriptor.dimensions = dimensions.data(); + // descriptor.dimensionsCount = dimensions.size(); + if (m.type() == CV_32F) + { + descriptor.type = ml::OperandType::Float32; + } + else + { + CV_Error(Error::StsNotImplemented, format("Unsupported data type %s", typeToString(m.type()).c_str())); + } + host = &m; +} + +WebnnBackendWrapper::~WebnnBackendWrapper() +{ + // nothing +} + +void WebnnBackendWrapper::copyToHost() +{ + CV_LOG_DEBUG(NULL, "WebnnBackendWrapper::copyToHost()"); + //CV_Error(Error::StsNotImplemented, ""); +} + +void WebnnBackendWrapper::setHostDirty() +{ + CV_LOG_DEBUG(NULL, "WebnnBackendWrapper::setHostDirty()"); + //CV_Error(Error::StsNotImplemented, ""); +} + +void forwardWebnn(const std::vector >& outBlobsWrappers, + Ptr& node, bool isAsync) +{ + CV_Assert(!node.empty()); + Ptr webnnNode = node.dynamicCast(); + CV_Assert(!webnnNode.empty()); + webnnNode->net->forward(outBlobsWrappers, isAsync); +} + + +#else +void forwardWebnn(const std::vector >& outBlobsWrappers, + Ptr& operand, bool isAsync) +{ + CV_Assert(false && "WebNN is not enabled in this OpenCV build"); +} + +#endif + +} +} \ No newline at end of file diff --git a/modules/dnn/src/op_webnn.hpp b/modules/dnn/src/op_webnn.hpp new file mode 100644 index 0000000000..5b77b10827 --- /dev/null +++ b/modules/dnn/src/op_webnn.hpp @@ -0,0 +1,171 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef __OPENCV_DNN_OP_WEBNN_HPP__ +#define __OPENCV_DNN_OP_WEBNN_HPP__ + +#include "opencv2/core/cvdef.h" +#include "opencv2/core/cvstd.hpp" +#include "opencv2/dnn.hpp" + +#ifdef HAVE_WEBNN + +#include +#include +#ifdef __EMSCRIPTEN__ +#include +#include +#include +#else +#include +#include +#endif + +#include +#include + +#endif // HAVE_WEBNN + +namespace cv { namespace dnn { + +constexpr bool haveWebnn() { +#ifdef HAVE_WEBNN + return true; +#else + return false; +#endif +} + +#ifdef HAVE_WEBNN + +class WebnnBackendNode; +class WebnnBackendWrapper; + +namespace webnn { +inline std::vector getShape(const Mat& mat) +{ + std::vector result(mat.dims); + for (int i = 0; i < mat.dims; i++) + result[i] = (int32_t)mat.size[i]; + return result; +} + +ml::Operand BuildConstant(const ml::GraphBuilder& builder, + const std::vector& dimensions, + const void* value, + size_t size, + ml::OperandType type); + +struct Pool2dOptions { + public: + std::vector windowDimensions; + std::vector padding; + std::vector strides; + std::vector dilations; + ml::AutoPad autoPad = ml::AutoPad::Explicit; + ml::InputOperandLayout layout = ml::InputOperandLayout::Nchw; + + const ml::Pool2dOptions* AsPtr() { + if (!windowDimensions.empty()) { + mOptions.windowDimensionsCount = windowDimensions.size(); + mOptions.windowDimensions = windowDimensions.data(); + } + if (!padding.empty()) { + mOptions.paddingCount = padding.size(); + mOptions.padding = padding.data(); + } + if (!strides.empty()) { + mOptions.stridesCount = strides.size(); + mOptions.strides = strides.data(); + } + if (!dilations.empty()) { + mOptions.dilationsCount = dilations.size(); + mOptions.dilations = dilations.data(); + } + mOptions.layout = layout; + mOptions.autoPad = autoPad; + return &mOptions; + } + + private: + ml::Pool2dOptions mOptions; + }; +} + +class WebnnNet +{ +public: + WebnnNet(); + + void addOutput(const std::string& name); + + bool isInitialized(); + void init(Target targetId); + + void forward(const std::vector >& outBlobsWrappers, bool isAsync); + + std::vector setInputs(const std::vector& inputs, const std::vector& names); + + void setUnconnectedNodes(Ptr& node); + void addBlobs(const std::vector >& ptrs); + + void createNet(Target targetId); + // void setNodePtr(std::shared_ptr* ptr); + + void reset(); + + ml::GraphBuilder builder; + ml::Context context; + ml::Graph graph; + + std::unordered_map> allBlobs; + + bool hasNetOwner; + std::string device_name; + bool isInit = false; + + std::vector requestedOutputs; + + std::vector inputNames; + std::vector outputNames; + ml::NamedOperands namedOperands; +}; + +class WebnnBackendNode : public BackendNode +{ +public: + WebnnBackendNode(ml::Operand&& operand); + WebnnBackendNode(ml::Operand& operand); + + std::string name; + ml::Operand operand; + Ptr net; +}; + +class WebnnBackendWrapper : public BackendWrapper +{ +public: + WebnnBackendWrapper(int targetId, Mat& m); + ~WebnnBackendWrapper(); + + virtual void copyToHost() CV_OVERRIDE; + virtual void setHostDirty() CV_OVERRIDE; + + std::string name; + Mat* host; + std::unique_ptr buffer; + size_t size; + std::vector dimensions; + ml::OperandDescriptor descriptor; +}; + +#endif // HAVE_WebNN + +void forwardWebnn(const std::vector >& outBlobsWrappers, + Ptr& node, bool isAsync); + +}} // namespace cv::dnn + + +#endif // __OPENCV_DNN_OP_WEBNN_HPP__ diff --git a/modules/dnn/src/webnn/README.md b/modules/dnn/src/webnn/README.md new file mode 100644 index 0000000000..6c6544a65d --- /dev/null +++ b/modules/dnn/src/webnn/README.md @@ -0,0 +1,11 @@ +## Build Instructions + +### Build WebNN-native and set the environment variable + +Refer to [WebNN's build instructions](https://github.com/webmachinelearning/webnn-native) to complete the build of WebNN-native. + +Set environment variable `WEBNN_NATIVE_DIR` to enable native DNN_BACKEND_WEBNN build: `export WEBNN_NATIVE_DIR=${PATH_TO_WebNN}`. Please let `WEBNN_NATIVE_DIR` points the output directory of webnn-native build (e.g. webnn-native/out/Release). + +### Test native DNN_BACKEND_WEBNN backend +Add -DWITH_WEBNN=ON to the cmake command to build the WebNN module such as: +`cmake -DWITH_WEBNN=ON ../opencv` (according to the @ref tutorial_linux_install) \ No newline at end of file diff --git a/modules/dnn/test/test_common.hpp b/modules/dnn/test/test_common.hpp index 139f3d1671..f20aa507c1 100644 --- a/modules/dnn/test/test_common.hpp +++ b/modules/dnn/test/test_common.hpp @@ -135,7 +135,8 @@ testing::internal::ParamGenerator< tuple > dnnBackendsAndTarget bool withCpuOCV = true, bool withVkCom = true, bool withCUDA = true, - bool withNgraph = true + bool withNgraph = true, + bool withWebnn = true ); testing::internal::ParamGenerator< tuple > dnnBackendsAndTargetsIE(); diff --git a/modules/dnn/test/test_common.impl.hpp b/modules/dnn/test/test_common.impl.hpp index 3d56e6f308..c312474256 100644 --- a/modules/dnn/test/test_common.impl.hpp +++ b/modules/dnn/test/test_common.impl.hpp @@ -29,6 +29,7 @@ void PrintTo(const cv::dnn::Backend& v, std::ostream* os) case DNN_BACKEND_CUDA: *os << "CUDA"; return; case DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019: *os << "DLIE"; return; case DNN_BACKEND_INFERENCE_ENGINE_NGRAPH: *os << "NGRAPH"; return; + case DNN_BACKEND_WEBNN: *os << "WEBNN"; return; } // don't use "default:" to emit compiler warnings *os << "DNN_BACKEND_UNKNOWN(" << (int)v << ")"; } @@ -247,7 +248,8 @@ testing::internal::ParamGenerator< tuple > dnnBackendsAndTarget bool withCpuOCV /*= true*/, bool withVkCom /*= true*/, bool withCUDA /*= true*/, - bool withNgraph /*= true*/ + bool withNgraph /*= true*/, + bool withWebnn /*= false*/ ) { #ifdef HAVE_INF_ENGINE @@ -302,6 +304,17 @@ testing::internal::ParamGenerator< tuple > dnnBackendsAndTarget } #endif +#ifdef HAVE_WEBNN + if (withWebnn) + { + for (auto target : getAvailableTargets(DNN_BACKEND_WEBNN)) { + targets.push_back(make_tuple(DNN_BACKEND_WEBNN, target)); + } + } +#else + CV_UNUSED(withWebnn); +#endif + { available = getAvailableTargets(DNN_BACKEND_OPENCV); for (std::vector< Target >::const_iterator i = available.begin(); i != available.end(); ++i) diff --git a/platforms/js/build_js.py b/platforms/js/build_js.py index f490eb58d5..64dc1a6c67 100644 --- a/platforms/js/build_js.py +++ b/platforms/js/build_js.py @@ -67,6 +67,8 @@ class Builder: self.options = options self.build_dir = check_dir(options.build_dir, create=True) self.opencv_dir = check_dir(options.opencv_dir) + print('-----------------------------------------------------------') + print('options.opencv_dir:', options.opencv_dir) self.emscripten_dir = check_dir(options.emscripten_dir) def get_toolchain_file(self): @@ -84,6 +86,7 @@ class Builder: "-DCMAKE_BUILD_TYPE=Release", "-DCMAKE_TOOLCHAIN_FILE='%s'" % self.get_toolchain_file(), "-DCPU_BASELINE=''", + "-DCMAKE_INSTALL_PREFIX=/usr/local", "-DCPU_DISPATCH=''", "-DCV_TRACE=OFF", "-DBUILD_SHARED_LIBS=OFF", @@ -136,10 +139,10 @@ class Builder: "-DBUILD_opencv_js=ON", "-DBUILD_opencv_python2=OFF", "-DBUILD_opencv_python3=OFF", - "-DBUILD_EXAMPLES=OFF", + "-DBUILD_EXAMPLES=ON", "-DBUILD_PACKAGE=OFF", - "-DBUILD_TESTS=OFF", - "-DBUILD_PERF_TESTS=OFF"] + "-DBUILD_TESTS=ON", + "-DBUILD_PERF_TESTS=ON"] if self.options.cmake_option: cmd += self.options.cmake_option if self.options.build_doc: @@ -162,6 +165,9 @@ class Builder: else: cmd.append("-DBUILD_WASM_INTRIN_TESTS=OFF") + if self.options.webnn: + cmd.append("-DWITH_WEBNN=ON") + flags = self.get_build_flags() if flags: cmd += ["-DCMAKE_C_FLAGS='%s'" % flags, @@ -184,6 +190,8 @@ class Builder: flags += "-msimd128 " if self.options.build_flags: flags += self.options.build_flags + if self.options.webnn: + flags += "-s USE_WEBNN=1 " return flags def config(self): @@ -243,6 +251,7 @@ if __name__ == "__main__": # Write a path to modify file like argument of this flag parser.add_argument('--config', default=os.path.join(os.path.dirname(os.path.abspath(__file__)), 'opencv_js.config.py'), help="Specify configuration file with own list of exported into JS functions") + parser.add_argument('--webnn', action="store_true", help="Enable WebNN Backend") args = parser.parse_args() diff --git a/platforms/js/opencv_js.config.py b/platforms/js/opencv_js.config.py index 1ed28af89f..20380fc970 100644 --- a/platforms/js/opencv_js.config.py +++ b/platforms/js/opencv_js.config.py @@ -129,7 +129,7 @@ video = { 'TrackerMIL_Params': [], } -dnn = {'dnn_Net': ['setInput', 'forward'], +dnn = {'dnn_Net': ['setInput', 'forward', 'setPreferableBackend'], '': ['readNetFromCaffe', 'readNetFromTensorflow', 'readNetFromTorch', 'readNetFromDarknet', 'readNetFromONNX', 'readNet', 'blobFromImage']} diff --git a/samples/dnn/classification.cpp b/samples/dnn/classification.cpp index 769d6874be..51893666ab 100644 --- a/samples/dnn/classification.cpp +++ b/samples/dnn/classification.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -17,6 +18,7 @@ std::string keys = "{ std | 0.0 0.0 0.0 | Preprocess input image by dividing on a standard deviation.}" "{ crop | false | Preprocess input image by center cropping.}" "{ framework f | | Optional name of an origin framework of the model. Detect it automatically if it does not set. }" + "{ needSoftmax | false | Use Softmax to post-process the output of the net.}" "{ classes | | Optional path to a text file with names of classes. }" "{ backend | 0 | Choose one of computation backends: " "0: automatically (by default), " @@ -24,7 +26,8 @@ std::string keys = "2: Intel's Deep Learning Inference Engine (https://software.intel.com/openvino-toolkit), " "3: OpenCV implementation, " "4: VKCOM, " - "5: CUDA }," + "5: CUDA, " + "6: WebNN }" "{ target | 0 | Choose one of target computation devices: " "0: CPU target (by default), " "1: OpenCL, " @@ -70,6 +73,9 @@ int main(int argc, char** argv) String framework = parser.get("framework"); int backendId = parser.get("backend"); int targetId = parser.get("target"); + bool needSoftmax = parser.get("needSoftmax"); + std::cout<<"mean: "< layersTimes; - double freq = getTickFrequency() / 1000; - double t = net.getPerfProfile(layersTimes) / freq; - std::string label = format("Inference time: %.2f ms", t); + timeRecorder.reset(); + for(int i = 0; i < 200; i++) { + //! [Make forward pass] + timeRecorder.start(); + prob = net.forward(); + timeRecorder.stop(); + + //! [Get a class with a highest score] + Point classIdPoint; + minMaxLoc(prob.reshape(1, 1), 0, &confidence, 0, &classIdPoint); + classId = classIdPoint.x; + //! [Get a class with a highest score] + + // Put efficiency information. + // std::vector layersTimes; + // double freq = getTickFrequency() / 1000; + // t = net.getPerfProfile(layersTimes) / freq; + // t_sum += t; + } + if (needSoftmax == true) + { + float maxProb = 0.0; + float sum = 0.0; + Mat softmaxProb; + + maxProb = *std::max_element(prob.begin(), prob.end()); + cv::exp(prob-maxProb, softmaxProb); + sum = (float)cv::sum(softmaxProb)[0]; + softmaxProb /= sum; + Point classIdPoint; + minMaxLoc(softmaxProb.reshape(1, 1), 0, &confidence, 0, &classIdPoint); + classId = classIdPoint.x; + } + std::string label = format("Inference time of 1 round: %.2f ms", t1); + std::string label2 = format("Average time of 200 rounds: %.2f ms", timeRecorder.getTimeMilli()/200); putText(frame, label, Point(0, 15), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); + putText(frame, label2, Point(0, 35), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); // Print predicted class. label = format("%s: %.4f", (classes.empty() ? format("Class #%d", classId).c_str() : classes[classId].c_str()), confidence); - putText(frame, label, Point(0, 40), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); + putText(frame, label, Point(0, 55), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); imshow(kWinName, frame); } From 97c6ec6d49cb78321eafe6fa220ff80ebdc5e2f4 Mon Sep 17 00:00:00 2001 From: cudawarped Date: Tue, 23 Nov 2021 21:18:55 +0000 Subject: [PATCH 089/226] Merge pull request #20978 from cudawarped:videocapture_read_raw_enchancement Add capacity to Videocapture to return the extraData from FFmpeg when required * Update rawMode to append any extra data recieved during the initial negotiation of an RTSP stream or during the parsing of an MPEG4 file header. For h264[5] RTSP streams this ensures the parameter sets if available are always returned on the first call to grab()/read() and has two purposes: 1) To ensure the parameter sets are available even if they are not transmitted in band. This is common for axis ip camera's. 2) To allow callers of VideoCapture::grab()[read()] to write to split the raw stream over multiple files by appending the parameter sets to the begining of any new files. For (1) there is no alternative, for (2) if the parameter sets were provided in band it would be possible to parse the raw bit stream and search for the parameter sets however that would be a lot of work when that information is already provided by FFMPEG. For MPEG4 files this information is only suplied in the header and is required for decoding. Two properties are also required to enable the raw encoded bitstream to be written to multiple files, these are; 1) an indicator as to whether the last frame was a key frame or not - each new file needs to start at a key frame to avoid storing unusable frame diffs, 2) the length in bytes of the paramater sets contained in the last frame - required to split the paramater sets from the frame without having to parse the stream. Any call to VideoCapture::get(CAP_PROP_LF_PARAM_SET_LEN) returning a number greater than zero indicates the presense of a parameter set at the begining of the raw bitstream. * Adjust test data to account for extraData * Address warning. * Change added property names and remove paramater set start code check. * Output extra data on calls to retrieve instead of appending to the first packet. * Reverted old test case and added new one to evaluate new functionality. * Add missing definition. * Remove flag from legacy api. Add property to determine if returning extra data is supported. Always allow extra data to be returned on calls to cap.retrieve() Update test case. * Update condition which indicates CAP_PROP_CODEC_EXTRADATA_INDEX is not supported in test case. * Include compatibility for windows dll if not updated. Enforce existing return status convention. * Fix return error and missing test constraints. --- modules/videoio/include/opencv2/videoio.hpp | 2 ++ modules/videoio/src/cap_ffmpeg.cpp | 12 ++++++-- modules/videoio/src/cap_ffmpeg_impl.hpp | 32 +++++++++++++++------ modules/videoio/test/test_ffmpeg.cpp | 21 ++++++++++++++ 4 files changed, 56 insertions(+), 11 deletions(-) diff --git a/modules/videoio/include/opencv2/videoio.hpp b/modules/videoio/include/opencv2/videoio.hpp index 4b5bc135bc..8d97c31282 100644 --- a/modules/videoio/include/opencv2/videoio.hpp +++ b/modules/videoio/include/opencv2/videoio.hpp @@ -200,6 +200,8 @@ enum VideoCaptureProperties { CAP_PROP_AUDIO_TOTAL_CHANNELS = 64, //!< (read-only) Number of audio channels in the selected audio stream (mono, stereo, etc) CAP_PROP_AUDIO_TOTAL_STREAMS = 65, //!< (read-only) Number of audio streams. CAP_PROP_AUDIO_SYNCHRONIZE = 66, //!< (open, read) Enables audio synchronization. + CAP_PROP_LRF_HAS_KEY_FRAME = 67, //!< FFmpeg back-end only - Indicates whether the Last Raw Frame (LRF), output from VideoCapture::read() when VideoCapture is initialized with VideoCapture::open(CAP_FFMPEG, {CAP_PROP_FORMAT, -1}) or VideoCapture::set(CAP_PROP_FORMAT,-1) is called before the first call to VideoCapture::read(), contains encoded data for a key frame. + CAP_PROP_CODEC_EXTRADATA_INDEX = 68, //!< Positive index indicates that returning extra data is supported by the video back end. This can be retrieved as cap.retrieve(data, ). E.g. When reading from a h264 encoded RTSP stream, the FFmpeg backend could return the SPS and/or PPS if available (if sent in reply to a DESCRIBE request), from calls to cap.retrieve(data, ). #ifndef CV_DOXYGEN CV__CAP_PROP_LATEST #endif diff --git a/modules/videoio/src/cap_ffmpeg.cpp b/modules/videoio/src/cap_ffmpeg.cpp index 474907bb48..7030d0e653 100644 --- a/modules/videoio/src/cap_ffmpeg.cpp +++ b/modules/videoio/src/cap_ffmpeg.cpp @@ -87,7 +87,7 @@ public: { return ffmpegCapture ? icvGrabFrame_FFMPEG_p(ffmpegCapture)!=0 : false; } - virtual bool retrieveFrame(int, cv::OutputArray frame) CV_OVERRIDE + virtual bool retrieveFrame(int flag, cv::OutputArray frame) CV_OVERRIDE { unsigned char* data = 0; int step=0, width=0, height=0, cn=0; @@ -102,8 +102,14 @@ public: } } - if (!icvRetrieveFrame_FFMPEG_p(ffmpegCapture, &data, &step, &width, &height, &cn)) - return false; + if (flag == 0) { + if (!icvRetrieveFrame_FFMPEG_p(ffmpegCapture, &data, &step, &width, &height, &cn)) + return false; + } + else { + if (!ffmpegCapture->retrieveFrame(flag, &data, &step, &width, &height, &cn)) + return false; + } cv::Mat tmp(height, width, CV_MAKETYPE(CV_8U, cn), data, step); this->rotateFrame(tmp); diff --git a/modules/videoio/src/cap_ffmpeg_impl.hpp b/modules/videoio/src/cap_ffmpeg_impl.hpp index 76e1b855b7..b5471baeea 100644 --- a/modules/videoio/src/cap_ffmpeg_impl.hpp +++ b/modules/videoio/src/cap_ffmpeg_impl.hpp @@ -481,7 +481,7 @@ struct CvCapture_FFMPEG double getProperty(int) const; bool setProperty(int, double); bool grabFrame(); - bool retrieveFrame(int, unsigned char** data, int* step, int* width, int* height, int* cn); + bool retrieveFrame(int flag, unsigned char** data, int* step, int* width, int* height, int* cn); bool retrieveHWFrame(cv::OutputArray output); void rotateFrame(cv::Mat &mat) const; @@ -547,6 +547,7 @@ struct CvCapture_FFMPEG VideoAccelerationType va_type; int hw_device; int use_opencl; + int extraDataIdx; }; void CvCapture_FFMPEG::init() @@ -590,6 +591,7 @@ void CvCapture_FFMPEG::init() va_type = cv::VIDEO_ACCELERATION_NONE; // TODO OpenCV 5.0: change to _ANY? hw_device = -1; use_opencl = 0; + extraDataIdx = 1; } @@ -1408,20 +1410,28 @@ bool CvCapture_FFMPEG::grabFrame() return valid; } -bool CvCapture_FFMPEG::retrieveFrame(int, unsigned char** data, int* step, int* width, int* height, int* cn) +bool CvCapture_FFMPEG::retrieveFrame(int flag, unsigned char** data, int* step, int* width, int* height, int* cn) { if (!video_st) return false; - if (rawMode) + if (rawMode || flag == extraDataIdx) { - AVPacket& p = bsfc ? packet_filtered : packet; - *data = p.data; - *step = p.size; - *width = p.size; + bool ret = true; + if (flag == 0) { + AVPacket& p = bsfc ? packet_filtered : packet; + *data = p.data; + *step = p.size; + ret = p.data != NULL; + } + else if (flag == extraDataIdx) { + *data = ic->streams[video_stream]->codec->extradata; + *step = ic->streams[video_stream]->codec->extradata_size; + } + *width = *step; *height = 1; *cn = 1; - return p.data != NULL; + return ret; } AVFrame* sw_picture = picture; @@ -1586,6 +1596,12 @@ double CvCapture_FFMPEG::getProperty( int property_id ) const if (rawMode) return -1; break; + case CAP_PROP_LRF_HAS_KEY_FRAME: { + const AVPacket& p = bsfc ? packet_filtered : packet; + return ((p.flags & AV_PKT_FLAG_KEY) != 0) ? 1 : 0; + } + case CAP_PROP_CODEC_EXTRADATA_INDEX: + return extraDataIdx; case CAP_PROP_BITRATE: return static_cast(get_bitrate()); case CAP_PROP_ORIENTATION_META: diff --git a/modules/videoio/test/test_ffmpeg.cpp b/modules/videoio/test/test_ffmpeg.cpp index 52ec4c751d..87e25bbd3d 100644 --- a/modules/videoio/test/test_ffmpeg.cpp +++ b/modules/videoio/test/test_ffmpeg.cpp @@ -476,6 +476,27 @@ static void ffmpeg_check_read_raw(VideoCapture& cap) EXPECT_EQ((size_t)37118, data.total()); } +TEST(videoio_ffmpeg, ffmpeg_check_extra_data) +{ + if (!videoio_registry::hasBackend(CAP_FFMPEG)) + throw SkipTestException("FFmpeg backend was not found"); + + string video_file = findDataFile("video/big_buck_bunny.mp4"); + VideoCapture cap; + EXPECT_NO_THROW(cap.open(video_file, CAP_FFMPEG)); + ASSERT_TRUE(cap.isOpened()) << "Can't open the video"; + const int codecExtradataIdx = (int)cap.get(CAP_PROP_CODEC_EXTRADATA_INDEX); +#ifdef _WIN32 // handle old FFmpeg backend + if (codecExtradataIdx <= 0) + throw SkipTestException("Codec extra data is not supported by backend or video stream"); +#endif + Mat data; + ASSERT_TRUE(cap.retrieve(data, codecExtradataIdx)); + EXPECT_EQ(CV_8UC1, data.type()) << "CV_8UC1 != " << typeToString(data.type()); + EXPECT_TRUE(data.rows == 1 || data.cols == 1) << data.size; + EXPECT_EQ((size_t)45, data.total()); +} + TEST(videoio_ffmpeg, open_with_property) { if (!videoio_registry::hasBackend(CAP_FFMPEG)) From c6ab32ffb9157cb3be37a5b3979c6bee7f78b5d5 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 24 Nov 2021 03:43:55 +0000 Subject: [PATCH 090/226] 3rdparty: libjpeg-turbo 2.1.0 => 2.1.2 https://github.com/libjpeg-turbo/libjpeg-turbo/releases/tag/2.1.2 --- 3rdparty/libjpeg-turbo/CMakeLists.txt | 4 ++-- 3rdparty/libjpeg-turbo/jconfigint.h.in | 10 ++++++++++ 3rdparty/libjpeg-turbo/src/jchuff.c | 5 +++-- 3rdparty/libjpeg-turbo/src/jcmaster.c | 2 +- 3rdparty/libjpeg-turbo/src/jcphuff.c | 10 ++++++---- 3rdparty/libjpeg-turbo/src/jdapimin.c | 3 ++- 3rdparty/libjpeg-turbo/src/jdhuff.c | 11 ++++++++++- 3rdparty/libjpeg-turbo/src/jdmainct.c | 5 +++-- 3rdparty/libjpeg-turbo/src/jmemmgr.c | 6 +++--- 3rdparty/libjpeg-turbo/src/jpegint.h | 15 ++++++++++++++- 10 files changed, 54 insertions(+), 17 deletions(-) diff --git a/3rdparty/libjpeg-turbo/CMakeLists.txt b/3rdparty/libjpeg-turbo/CMakeLists.txt index 3c7f29b08e..73e1eee141 100644 --- a/3rdparty/libjpeg-turbo/CMakeLists.txt +++ b/3rdparty/libjpeg-turbo/CMakeLists.txt @@ -4,9 +4,9 @@ ocv_warnings_disable(CMAKE_C_FLAGS -Wunused-parameter -Wsign-compare -Wshorten-6 set(VERSION_MAJOR 2) set(VERSION_MINOR 1) -set(VERSION_REVISION 0) +set(VERSION_REVISION 2) set(VERSION ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_REVISION}) -set(LIBJPEG_TURBO_VERSION_NUMBER 2001000) +set(LIBJPEG_TURBO_VERSION_NUMBER 2001002) string(TIMESTAMP BUILD "opencv-${OPENCV_VERSION}-libjpeg-turbo") if(CMAKE_BUILD_TYPE STREQUAL "Debug") diff --git a/3rdparty/libjpeg-turbo/jconfigint.h.in b/3rdparty/libjpeg-turbo/jconfigint.h.in index c94bffaa35..a46979df1e 100644 --- a/3rdparty/libjpeg-turbo/jconfigint.h.in +++ b/3rdparty/libjpeg-turbo/jconfigint.h.in @@ -40,3 +40,13 @@ #define HAVE_BITSCANFORWARD #endif #endif + +#if defined(__has_attribute) +#if __has_attribute(fallthrough) +#define FALLTHROUGH __attribute__((fallthrough)); +#else +#define FALLTHROUGH +#endif +#else +#define FALLTHROUGH +#endif diff --git a/3rdparty/libjpeg-turbo/src/jchuff.c b/3rdparty/libjpeg-turbo/src/jchuff.c index 2bce767ebd..8ff817b151 100644 --- a/3rdparty/libjpeg-turbo/src/jchuff.c +++ b/3rdparty/libjpeg-turbo/src/jchuff.c @@ -44,8 +44,9 @@ * flags (this defines __thumb__). */ -#if defined(__arm__) || defined(__aarch64__) || defined(_M_ARM) || \ - defined(_M_ARM64) +/* NOTE: Both GCC and Clang define __GNUC__ */ +#if (defined(__GNUC__) && (defined(__arm__) || defined(__aarch64__))) || \ + defined(_M_ARM) || defined(_M_ARM64) #if !defined(__thumb__) || defined(__thumb2__) #define USE_CLZ_INTRINSIC #endif diff --git a/3rdparty/libjpeg-turbo/src/jcmaster.c b/3rdparty/libjpeg-turbo/src/jcmaster.c index 998dc40a5c..c2b2600031 100644 --- a/3rdparty/libjpeg-turbo/src/jcmaster.c +++ b/3rdparty/libjpeg-turbo/src/jcmaster.c @@ -493,7 +493,7 @@ prepare_for_pass(j_compress_ptr cinfo) master->pass_type = output_pass; master->pass_number++; #endif - /*FALLTHROUGH*/ + FALLTHROUGH /*FALLTHROUGH*/ case output_pass: /* Do a data-output pass. */ /* We need not repeat per-scan setup if prior optimization pass did it. */ diff --git a/3rdparty/libjpeg-turbo/src/jcphuff.c b/3rdparty/libjpeg-turbo/src/jcphuff.c index bd14fc27d5..1101987180 100644 --- a/3rdparty/libjpeg-turbo/src/jcphuff.c +++ b/3rdparty/libjpeg-turbo/src/jcphuff.c @@ -7,6 +7,7 @@ * Copyright (C) 2011, 2015, 2018, 2021, D. R. Commander. * Copyright (C) 2016, 2018, Matthieu Darbois. * Copyright (C) 2020, Arm Limited. + * Copyright (C) 2021, Alex Richardson. * For conditions of distribution and use, see the accompanying README.ijg * file. * @@ -52,8 +53,9 @@ * flags (this defines __thumb__). */ -#if defined(__arm__) || defined(__aarch64__) || defined(_M_ARM) || \ - defined(_M_ARM64) +/* NOTE: Both GCC and Clang define __GNUC__ */ +#if (defined(__GNUC__) && (defined(__arm__) || defined(__aarch64__))) || \ + defined(_M_ARM) || defined(_M_ARM64) #if !defined(__thumb__) || defined(__thumb2__) #define USE_CLZ_INTRINSIC #endif @@ -679,7 +681,7 @@ encode_mcu_AC_first(j_compress_ptr cinfo, JBLOCKROW *MCU_data) emit_restart(entropy, entropy->next_restart_num); #ifdef WITH_SIMD - cvalue = values = (JCOEF *)PAD((size_t)values_unaligned, 16); + cvalue = values = (JCOEF *)PAD((JUINTPTR)values_unaligned, 16); #else /* Not using SIMD, so alignment is not needed */ cvalue = values = values_unaligned; @@ -944,7 +946,7 @@ encode_mcu_AC_refine(j_compress_ptr cinfo, JBLOCKROW *MCU_data) emit_restart(entropy, entropy->next_restart_num); #ifdef WITH_SIMD - cabsvalue = absvalues = (JCOEF *)PAD((size_t)absvalues_unaligned, 16); + cabsvalue = absvalues = (JCOEF *)PAD((JUINTPTR)absvalues_unaligned, 16); #else /* Not using SIMD, so alignment is not needed */ cabsvalue = absvalues = absvalues_unaligned; diff --git a/3rdparty/libjpeg-turbo/src/jdapimin.c b/3rdparty/libjpeg-turbo/src/jdapimin.c index 21a41d2e9f..4609b1322f 100644 --- a/3rdparty/libjpeg-turbo/src/jdapimin.c +++ b/3rdparty/libjpeg-turbo/src/jdapimin.c @@ -23,6 +23,7 @@ #include "jinclude.h" #include "jpeglib.h" #include "jdmaster.h" +#include "jconfigint.h" /* @@ -308,7 +309,7 @@ jpeg_consume_input(j_decompress_ptr cinfo) /* Initialize application's data source module */ (*cinfo->src->init_source) (cinfo); cinfo->global_state = DSTATE_INHEADER; - /*FALLTHROUGH*/ + FALLTHROUGH /*FALLTHROUGH*/ case DSTATE_INHEADER: retcode = (*cinfo->inputctl->consume_input) (cinfo); if (retcode == JPEG_REACHED_SOS) { /* Found SOS, prepare to decompress */ diff --git a/3rdparty/libjpeg-turbo/src/jdhuff.c b/3rdparty/libjpeg-turbo/src/jdhuff.c index f786c10547..679d221685 100644 --- a/3rdparty/libjpeg-turbo/src/jdhuff.c +++ b/3rdparty/libjpeg-turbo/src/jdhuff.c @@ -584,7 +584,7 @@ decode_mcu_slow(j_decompress_ptr cinfo, JBLOCKROW *MCU_data) * behavior is, to the best of our understanding, innocuous, and it is * unclear how to work around it without potentially affecting * performance. Thus, we (hopefully temporarily) suppress UBSan integer - * overflow errors for this function. + * overflow errors for this function and decode_mcu_fast(). */ s += state.last_dc_val[ci]; state.last_dc_val[ci] = s; @@ -651,6 +651,12 @@ decode_mcu_slow(j_decompress_ptr cinfo, JBLOCKROW *MCU_data) } +#if defined(__has_feature) +#if __has_feature(undefined_behavior_sanitizer) +__attribute__((no_sanitize("signed-integer-overflow"), + no_sanitize("unsigned-integer-overflow"))) +#endif +#endif LOCAL(boolean) decode_mcu_fast(j_decompress_ptr cinfo, JBLOCKROW *MCU_data) { @@ -681,6 +687,9 @@ decode_mcu_fast(j_decompress_ptr cinfo, JBLOCKROW *MCU_data) if (entropy->dc_needed[blkn]) { int ci = cinfo->MCU_membership[blkn]; + /* Refer to the comment in decode_mcu_slow() regarding the supression of + * a UBSan integer overflow error in this line of code. + */ s += state.last_dc_val[ci]; state.last_dc_val[ci] = s; if (block) diff --git a/3rdparty/libjpeg-turbo/src/jdmainct.c b/3rdparty/libjpeg-turbo/src/jdmainct.c index 50301d6b50..f466b259f0 100644 --- a/3rdparty/libjpeg-turbo/src/jdmainct.c +++ b/3rdparty/libjpeg-turbo/src/jdmainct.c @@ -18,6 +18,7 @@ #include "jinclude.h" #include "jdmainct.h" +#include "jconfigint.h" /* @@ -360,7 +361,7 @@ process_data_context_main(j_decompress_ptr cinfo, JSAMPARRAY output_buf, main_ptr->context_state = CTX_PREPARE_FOR_IMCU; if (*out_row_ctr >= out_rows_avail) return; /* Postprocessor exactly filled output buf */ - /*FALLTHROUGH*/ + FALLTHROUGH /*FALLTHROUGH*/ case CTX_PREPARE_FOR_IMCU: /* Prepare to process first M-1 row groups of this iMCU row */ main_ptr->rowgroup_ctr = 0; @@ -371,7 +372,7 @@ process_data_context_main(j_decompress_ptr cinfo, JSAMPARRAY output_buf, if (main_ptr->iMCU_row_ctr == cinfo->total_iMCU_rows) set_bottom_pointers(cinfo); main_ptr->context_state = CTX_PROCESS_IMCU; - /*FALLTHROUGH*/ + FALLTHROUGH /*FALLTHROUGH*/ case CTX_PROCESS_IMCU: /* Call postprocessor using previously set pointers */ (*cinfo->post->post_process_data) (cinfo, diff --git a/3rdparty/libjpeg-turbo/src/jmemmgr.c b/3rdparty/libjpeg-turbo/src/jmemmgr.c index 508ca7429c..70b8ec0c49 100644 --- a/3rdparty/libjpeg-turbo/src/jmemmgr.c +++ b/3rdparty/libjpeg-turbo/src/jmemmgr.c @@ -4,7 +4,7 @@ * This file was part of the Independent JPEG Group's software: * Copyright (C) 1991-1997, Thomas G. Lane. * libjpeg-turbo Modifications: - * Copyright (C) 2016, D. R. Commander. + * Copyright (C) 2016, 2021, D. R. Commander. * For conditions of distribution and use, see the accompanying README.ijg * file. * @@ -1032,7 +1032,7 @@ free_pool(j_common_ptr cinfo, int pool_id) large_pool_ptr next_lhdr_ptr = lhdr_ptr->next; space_freed = lhdr_ptr->bytes_used + lhdr_ptr->bytes_left + - sizeof(large_pool_hdr); + sizeof(large_pool_hdr) + ALIGN_SIZE - 1; jpeg_free_large(cinfo, (void *)lhdr_ptr, space_freed); mem->total_space_allocated -= space_freed; lhdr_ptr = next_lhdr_ptr; @@ -1045,7 +1045,7 @@ free_pool(j_common_ptr cinfo, int pool_id) while (shdr_ptr != NULL) { small_pool_ptr next_shdr_ptr = shdr_ptr->next; space_freed = shdr_ptr->bytes_used + shdr_ptr->bytes_left + - sizeof(small_pool_hdr); + sizeof(small_pool_hdr) + ALIGN_SIZE - 1; jpeg_free_small(cinfo, (void *)shdr_ptr, space_freed); mem->total_space_allocated -= space_freed; shdr_ptr = next_shdr_ptr; diff --git a/3rdparty/libjpeg-turbo/src/jpegint.h b/3rdparty/libjpeg-turbo/src/jpegint.h index 195fbcb9b6..8c8534793a 100644 --- a/3rdparty/libjpeg-turbo/src/jpegint.h +++ b/3rdparty/libjpeg-turbo/src/jpegint.h @@ -5,8 +5,9 @@ * Copyright (C) 1991-1997, Thomas G. Lane. * Modified 1997-2009 by Guido Vollbeding. * libjpeg-turbo Modifications: - * Copyright (C) 2015-2016, 2019, D. R. Commander. + * Copyright (C) 2015-2016, 2019, 2021, D. R. Commander. * Copyright (C) 2015, Google, Inc. + * Copyright (C) 2021, Alex Richardson. * For conditions of distribution and use, see the accompanying README.ijg * file. * @@ -47,6 +48,18 @@ typedef enum { /* Operating modes for buffer controllers */ /* JLONG must hold at least signed 32-bit values. */ typedef long JLONG; +/* JUINTPTR must hold pointer values. */ +#ifdef __UINTPTR_TYPE__ +/* + * __UINTPTR_TYPE__ is GNU-specific and available in GCC 4.6+ and Clang 3.0+. + * Fortunately, that is sufficient to support the few architectures for which + * sizeof(void *) != sizeof(size_t). The only other options would require C99 + * or Clang-specific builtins. + */ +typedef __UINTPTR_TYPE__ JUINTPTR; +#else +typedef size_t JUINTPTR; +#endif /* * Left shift macro that handles a negative operand without causing any From 9cc60c9dd32b5fa5f7bdd7991ed6610eb7c2a7ee Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Thu, 25 Nov 2021 12:44:21 +0100 Subject: [PATCH 091/226] Use ==/!= to compare constant literals (str, bytes, int, float, tuple) Avoid `SyntaxWarning` on Python >= 3.8 ``` >>> "convolutional" == "convolutional" True >>> "convolutional" is "convolutional" :1: SyntaxWarning: "is" with a literal. Did you mean "=="? True ``` Related to #21121 --- samples/dnn/tf_text_graph_ssd.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/dnn/tf_text_graph_ssd.py b/samples/dnn/tf_text_graph_ssd.py index dbdee6ea9f..d27fd0d384 100644 --- a/samples/dnn/tf_text_graph_ssd.py +++ b/samples/dnn/tf_text_graph_ssd.py @@ -270,12 +270,12 @@ def createSSDGraph(modelPath, configPath, outputPath): addConstNode('concat/axis_flatten', [-1], graph_def) addConstNode('PriorBox/concat/axis', [-2], graph_def) - for label in ['ClassPredictor', 'BoxEncodingPredictor' if box_predictor is 'convolutional' else 'BoxPredictor']: + for label in ['ClassPredictor', 'BoxEncodingPredictor' if box_predictor == 'convolutional' else 'BoxPredictor']: concatInputs = [] for i in range(num_layers): # Flatten predictions flatten = NodeDef() - if box_predictor is 'convolutional': + if box_predictor == 'convolutional': inpName = 'BoxPredictor_%d/%s/BiasAdd' % (i, label) else: if i == 0: @@ -308,7 +308,7 @@ def createSSDGraph(modelPath, configPath, outputPath): priorBox = NodeDef() priorBox.name = 'PriorBox_%d' % i priorBox.op = 'PriorBox' - if box_predictor is 'convolutional': + if box_predictor == 'convolutional': priorBox.input.append('BoxPredictor_%d/BoxEncodingPredictor/BiasAdd' % i) else: if i == 0: From 23bbe511fe4fd0143ffc7f0c0ccfe22caaba9a60 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 26 Nov 2021 11:07:14 +0100 Subject: [PATCH 092/226] CMakeLists.txt: Fix typo discovered by codespell https://pypi.org/project/codespell/ --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b660cdd33b..169c385fad 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -113,7 +113,7 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ${ENABLE_PIC}) ocv_cmake_hook(PRE_CMAKE_BOOTSTRAP) -# Bootstap CMake system: setup CMAKE_SYSTEM_NAME and other vars +# Bootstrap CMake system: setup CMAKE_SYSTEM_NAME and other vars enable_language(CXX C) ocv_cmake_hook(POST_CMAKE_BOOTSTRAP) From cdbb042ce44c9e7c79f11a24a232ce7a0475efe0 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 26 Nov 2021 11:57:54 +0100 Subject: [PATCH 093/226] Use print() function in both Python 2 and Python 3 --- modules/ts/misc/chart.py | 17 +++++++++-------- modules/ts/misc/report.py | 3 ++- modules/ts/misc/summary.py | 3 ++- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/modules/ts/misc/chart.py b/modules/ts/misc/chart.py index 0547eefdfc..6ae726abeb 100755 --- a/modules/ts/misc/chart.py +++ b/modules/ts/misc/chart.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +from __future__ import print_function import testlog_parser, sys, os, xml, re from table_formatter import * from optparse import OptionParser @@ -116,7 +117,7 @@ if __name__ == "__main__": (options, args) = parser.parse_args() if len(args) != 1: - print >> sys.stderr, "Usage:\n", os.path.basename(sys.argv[0]), ".xml" + print("Usage:\n", os.path.basename(sys.argv[0]), ".xml", file=sys.stderr) exit(1) options.generateHtml = detectHtmlOutputType(options.format) @@ -136,7 +137,7 @@ if __name__ == "__main__": args[0] = os.path.basename(args[0]) if not tests: - print >> sys.stderr, "Error - no tests matched" + print("Error - no tests matched", file=sys.stderr) exit(1) argsnum = len(tests[0][1]) @@ -156,26 +157,26 @@ if __name__ == "__main__": names1.add(sn) if sn == sname: if len(pair[1]) != argsnum: - print >> sys.stderr, "Error - unable to create chart tables for functions having different argument numbers" + print("Error - unable to create chart tables for functions having different argument numbers", file=sys.stderr) sys.exit(1) for i in range(argsnum): arglists[i][pair[1][i]] = 1 if names1 or len(names) != 1: - print >> sys.stderr, "Error - unable to create tables for functions from different test suits:" + print("Error - unable to create tables for functions from different test suits:", file=sys.stderr) i = 1 for name in sorted(names): - print >> sys.stderr, "%4s: %s" % (i, name) + print("%4s: %s" % (i, name), file=sys.stderr) i += 1 if names1: - print >> sys.stderr, "Other suits in this log (can not be chosen):" + print("Other suits in this log (can not be chosen):", file=sys.stderr) for name in sorted(names1): - print >> sys.stderr, "%4s: %s" % (i, name) + print("%4s: %s" % (i, name), file=sys.stderr) i += 1 sys.exit(1) if argsnum < 2: - print >> sys.stderr, "Error - tests from %s have less than 2 parameters" % sname + print("Error - tests from %s have less than 2 parameters" % sname, file=sys.stderr) exit(1) for i in range(argsnum): diff --git a/modules/ts/misc/report.py b/modules/ts/misc/report.py index f0a87f2a12..ec60e16a13 100755 --- a/modules/ts/misc/report.py +++ b/modules/ts/misc/report.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +from __future__ import print_function import testlog_parser, sys, os, xml, re, glob from table_formatter import * from optparse import OptionParser @@ -14,7 +15,7 @@ if __name__ == "__main__": (options, args) = parser.parse_args() if len(args) < 1: - print >> sys.stderr, "Usage:\n", os.path.basename(sys.argv[0]), ".xml" + print("Usage:\n", os.path.basename(sys.argv[0]), ".xml", file=sys.stderr) exit(0) options.generateHtml = detectHtmlOutputType(options.format) diff --git a/modules/ts/misc/summary.py b/modules/ts/misc/summary.py index 9da1fb60c6..bfb98358d2 100755 --- a/modules/ts/misc/summary.py +++ b/modules/ts/misc/summary.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +from __future__ import print_function import testlog_parser, sys, os, xml, glob, re from table_formatter import * from optparse import OptionParser @@ -26,7 +27,7 @@ def getSetName(tset, idx, columns, short = True): if __name__ == "__main__": if len(sys.argv) < 2: - print >> sys.stderr, "Usage:\n", os.path.basename(sys.argv[0]), ".xml [.xml ...]" + print("Usage:\n", os.path.basename(sys.argv[0]), ".xml [.xml ...]", file=sys.stderr) exit(0) parser = OptionParser() From ebe4ca6b6061684f4c6bbea8851b37a3229edbc6 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 26 Nov 2021 12:29:56 +0100 Subject: [PATCH 094/226] Fix typos discovered by codespell --- doc/js_tutorials/js_assets/js_camshift.html | 2 +- doc/js_tutorials/js_assets/js_meanshift.html | 2 +- modules/highgui/CMakeLists.txt | 2 +- modules/js/generator/embindgen.py | 4 ++-- modules/ts/misc/run.py | 2 +- platforms/winpack_dldt/build_package.py | 4 ++-- samples/dnn/dasiamrpn_tracker.py | 2 +- samples/dnn/siamrpnpp.py | 2 +- samples/python/tutorial_code/video/meanshift/camshift.py | 2 +- samples/python/tutorial_code/video/meanshift/meanshift.py | 2 +- 10 files changed, 12 insertions(+), 12 deletions(-) diff --git a/doc/js_tutorials/js_assets/js_camshift.html b/doc/js_tutorials/js_assets/js_camshift.html index 046ab20efd..b2d10751fa 100644 --- a/doc/js_tutorials/js_assets/js_camshift.html +++ b/doc/js_tutorials/js_assets/js_camshift.html @@ -77,7 +77,7 @@ cv.normalize(roiHist, roiHist, 0, 255, cv.NORM_MINMAX); // delete useless mats. roi.delete(); hsvRoi.delete(); mask.delete(); low.delete(); high.delete(); hsvRoiVec.delete(); -// Setup the termination criteria, either 10 iteration or move by atleast 1 pt +// Setup the termination criteria, either 10 iteration or move by at least 1 pt let termCrit = new cv.TermCriteria(cv.TERM_CRITERIA_EPS | cv.TERM_CRITERIA_COUNT, 10, 1); let hsv = new cv.Mat(video.height, video.width, cv.CV_8UC3); diff --git a/doc/js_tutorials/js_assets/js_meanshift.html b/doc/js_tutorials/js_assets/js_meanshift.html index 9e29002c8d..958539bede 100644 --- a/doc/js_tutorials/js_assets/js_meanshift.html +++ b/doc/js_tutorials/js_assets/js_meanshift.html @@ -77,7 +77,7 @@ cv.normalize(roiHist, roiHist, 0, 255, cv.NORM_MINMAX); // delete useless mats. roi.delete(); hsvRoi.delete(); mask.delete(); low.delete(); high.delete(); hsvRoiVec.delete(); -// Setup the termination criteria, either 10 iteration or move by atleast 1 pt +// Setup the termination criteria, either 10 iteration or move by at least 1 pt let termCrit = new cv.TermCriteria(cv.TERM_CRITERIA_EPS | cv.TERM_CRITERIA_COUNT, 10, 1); let hsv = new cv.Mat(video.height, video.width, cv.CV_8UC3); diff --git a/modules/highgui/CMakeLists.txt b/modules/highgui/CMakeLists.txt index e48e2470ce..8339a1d7d2 100644 --- a/modules/highgui/CMakeLists.txt +++ b/modules/highgui/CMakeLists.txt @@ -55,7 +55,7 @@ if(HAVE_QT) QT5_ADD_RESOURCES(_RCC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.qrc) QT5_WRAP_CPP(_MOC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.h) else() - message(FATAL_ERROR "Unsuported QT version: ${QT_VERSION_MAJOR}") + message(FATAL_ERROR "Unsupported QT version: ${QT_VERSION_MAJOR}") endif() list(APPEND highgui_srcs diff --git a/modules/js/generator/embindgen.py b/modules/js/generator/embindgen.py index 7eb20c9861..784c463f34 100644 --- a/modules/js/generator/embindgen.py +++ b/modules/js/generator/embindgen.py @@ -484,7 +484,7 @@ class JSWrapperGenerator(object): arg_types.append(arg_type) unwrapped_arg_types.append(arg_type) - # Function attribure + # Function attribute func_attribs = '' if '*' in ''.join(arg_types): func_attribs += ', allow_raw_pointers()' @@ -679,7 +679,7 @@ class JSWrapperGenerator(object): def_args.append(arg.defval) arg_types.append(orig_arg_types[-1]) - # Function attribure + # Function attribute func_attribs = '' if '*' in ''.join(orig_arg_types): func_attribs += ', allow_raw_pointers()' diff --git a/modules/ts/misc/run.py b/modules/ts/misc/run.py index c2e4d6532b..bb17aa884f 100755 --- a/modules/ts/misc/run.py +++ b/modules/ts/misc/run.py @@ -105,7 +105,7 @@ if __name__ == "__main__": path = args.build_path try: if not os.path.isdir(path): - raise Err("Not a directory (should contain CMakeCache.txt ot test executables)") + raise Err("Not a directory (should contain CMakeCache.txt to test executables)") cache = CMakeCache(args.configuration) fname = os.path.join(path, "CMakeCache.txt") diff --git a/platforms/winpack_dldt/build_package.py b/platforms/winpack_dldt/build_package.py index 0194323930..0292b028b8 100644 --- a/platforms/winpack_dldt/build_package.py +++ b/platforms/winpack_dldt/build_package.py @@ -163,7 +163,7 @@ class BuilderDLDT: self.config = config cpath = self.config.dldt_config - log.info('DLDT build configration: %s', cpath) + log.info('DLDT build configuration: %s', cpath) if not os.path.exists(cpath): cpath = os.path.join(SCRIPT_DIR, cpath) if not os.path.exists(cpath): @@ -573,5 +573,5 @@ if __name__ == "__main__": try: main() except: - log.info('FATAL: Error occured. To investigate problem try to change logging level using LOGLEVEL=DEBUG environment variable.') + log.info('FATAL: Error occurred. To investigate problem try to change logging level using LOGLEVEL=DEBUG environment variable.') raise diff --git a/samples/dnn/dasiamrpn_tracker.py b/samples/dnn/dasiamrpn_tracker.py index 03e99d6dbf..1f329423a3 100644 --- a/samples/dnn/dasiamrpn_tracker.py +++ b/samples/dnn/dasiamrpn_tracker.py @@ -226,7 +226,7 @@ def main(): mark = True drawing = False cx, cy, w, h = 0.0, 0.0, 0, 0 - # Fucntion for drawing during videostream + # Function for drawing during videostream def get_bb(event, x, y, flag, param): nonlocal point1, point2, cx, cy, w, h, drawing, mark diff --git a/samples/dnn/siamrpnpp.py b/samples/dnn/siamrpnpp.py index c7c49b1b85..d2eae9f7e2 100644 --- a/samples/dnn/siamrpnpp.py +++ b/samples/dnn/siamrpnpp.py @@ -300,7 +300,7 @@ class SiamRPNTracker: # clip boundary cx, cy, width, height = self._bbox_clip(cx, cy, width, height, img.shape[:2]) - # udpate state + # update state self.center_pos = np.array([cx, cy]) self.w = width self.h = height diff --git a/samples/python/tutorial_code/video/meanshift/camshift.py b/samples/python/tutorial_code/video/meanshift/camshift.py index d115bdb4d3..774e8b6c0e 100644 --- a/samples/python/tutorial_code/video/meanshift/camshift.py +++ b/samples/python/tutorial_code/video/meanshift/camshift.py @@ -24,7 +24,7 @@ mask = cv.inRange(hsv_roi, np.array((0., 60.,32.)), np.array((180.,255.,255.))) roi_hist = cv.calcHist([hsv_roi],[0],mask,[180],[0,180]) cv.normalize(roi_hist,roi_hist,0,255,cv.NORM_MINMAX) -# Setup the termination criteria, either 10 iteration or move by atleast 1 pt +# Setup the termination criteria, either 10 iteration or move by at least 1 pt term_crit = ( cv.TERM_CRITERIA_EPS | cv.TERM_CRITERIA_COUNT, 10, 1 ) while(1): diff --git a/samples/python/tutorial_code/video/meanshift/meanshift.py b/samples/python/tutorial_code/video/meanshift/meanshift.py index f765b023e9..ecdd573cb1 100644 --- a/samples/python/tutorial_code/video/meanshift/meanshift.py +++ b/samples/python/tutorial_code/video/meanshift/meanshift.py @@ -24,7 +24,7 @@ mask = cv.inRange(hsv_roi, np.array((0., 60.,32.)), np.array((180.,255.,255.))) roi_hist = cv.calcHist([hsv_roi],[0],mask,[180],[0,180]) cv.normalize(roi_hist,roi_hist,0,255,cv.NORM_MINMAX) -# Setup the termination criteria, either 10 iteration or move by atleast 1 pt +# Setup the termination criteria, either 10 iteration or move by at least 1 pt term_crit = ( cv.TERM_CRITERIA_EPS | cv.TERM_CRITERIA_COUNT, 10, 1 ) while(1): From b95d71af2bb86ac9b4699edf81d2332f2c52ea60 Mon Sep 17 00:00:00 2001 From: Maxim Pashchenkov Date: Fri, 26 Nov 2021 14:31:15 +0300 Subject: [PATCH 095/226] Merge pull request #21106 from mpashchenkov:mp/ocv-gapi-clean-samples G-API: Cleaning samples * parseSSD + removed render details from gcpukernel * self-rev * Applying comment * Added operators * warnings --- .../include/opencv2/gapi/cpu/gcpukernel.hpp | 9 -- modules/gapi/samples/gaze_estimation.cpp | 85 +---------------- modules/gapi/samples/infer_ie_onnx_hybrid.cpp | 8 +- modules/gapi/samples/infer_single_roi.cpp | 94 +++---------------- modules/gapi/samples/infer_ssd_onnx.cpp | 78 +++------------ .../gapi/samples/onevpl_infer_single_roi.cpp | 74 ++------------- .../gapi/samples/privacy_masking_camera.cpp | 63 ++----------- .../gapi/samples/semantic_segmentation.cpp | 18 +++- 8 files changed, 61 insertions(+), 368 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp b/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp index 009c19688b..48909a84fc 100644 --- a/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp +++ b/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp @@ -28,14 +28,6 @@ namespace gimpl { // Forward-declare an internal class class GCPUExecutable; - - namespace render - { - namespace ocv - { - class GRenderExecutable; - } - } } // namespace gimpl namespace gapi @@ -133,7 +125,6 @@ protected: std::unordered_map m_results; friend class gimpl::GCPUExecutable; - friend class gimpl::render::ocv::GRenderExecutable; }; class GAPI_EXPORTS GCPUKernel diff --git a/modules/gapi/samples/gaze_estimation.cpp b/modules/gapi/samples/gaze_estimation.cpp index 96b2ebf163..8ccaccd2c3 100644 --- a/modules/gapi/samples/gaze_estimation.cpp +++ b/modules/gapi/samples/gaze_estimation.cpp @@ -9,6 +9,7 @@ #include #include #include // CommandLineParser +#include const std::string about = "This is an OpenCV-based version of Gaze Estimation example"; @@ -58,16 +59,6 @@ G_API_OP(Size, , "custom.gapi.size") { } }; -G_API_OP(ParseSSD, - , - "custom.gaze_estimation.parseSSD") { - static cv::GArrayDesc outMeta( const cv::GMatDesc & - , const cv::GOpaqueDesc & - , bool) { - return cv::empty_array_desc(); - } -}; - // Left/Right eye per every face G_API_OP(ParseEyes, (GMats, GRects, GSize)>, @@ -91,27 +82,6 @@ G_API_OP(ProcessPoses, } }; -void adjustBoundingBox(cv::Rect& boundingBox) { - auto w = boundingBox.width; - auto h = boundingBox.height; - - boundingBox.x -= static_cast(0.067 * w); - boundingBox.y -= static_cast(0.028 * h); - - boundingBox.width += static_cast(0.15 * w); - boundingBox.height += static_cast(0.13 * h); - - if (boundingBox.width < boundingBox.height) { - auto dx = (boundingBox.height - boundingBox.width); - boundingBox.x -= dx / 2; - boundingBox.width += dx; - } else { - auto dy = (boundingBox.width - boundingBox.height); - boundingBox.y -= dy / 2; - boundingBox.height += dy; - } -} - void gazeVectorToGazeAngles(const cv::Point3f& gazeVector, cv::Point2f& gazeAngles) { auto r = cv::norm(gazeVector); @@ -130,55 +100,6 @@ GAPI_OCV_KERNEL(OCVSize, Size) { } }; -GAPI_OCV_KERNEL(OCVParseSSD, ParseSSD) { - static void run(const cv::Mat &in_ssd_result, - const cv::Size &upscale, - const bool filter_out_of_bounds, - std::vector &out_objects) { - const auto &in_ssd_dims = in_ssd_result.size; - CV_Assert(in_ssd_dims.dims() == 4u); - - const int MAX_PROPOSALS = in_ssd_dims[2]; - const int OBJECT_SIZE = in_ssd_dims[3]; - CV_Assert(OBJECT_SIZE == 7); // fixed SSD object size - - const cv::Rect surface({0,0}, upscale); - out_objects.clear(); - - const float *data = in_ssd_result.ptr(); - for (int i = 0; i < MAX_PROPOSALS; i++) { - const float image_id = data[i * OBJECT_SIZE + 0]; - const float label = data[i * OBJECT_SIZE + 1]; - const float confidence = data[i * OBJECT_SIZE + 2]; - const float rc_left = data[i * OBJECT_SIZE + 3]; - const float rc_top = data[i * OBJECT_SIZE + 4]; - const float rc_right = data[i * OBJECT_SIZE + 5]; - const float rc_bottom = data[i * OBJECT_SIZE + 6]; - (void) label; - if (image_id < 0.f) { - break; // marks end-of-detections - } - if (confidence < 0.5f) { - continue; // skip objects with low confidence - } - cv::Rect rc; // map relative coordinates to the original image scale - rc.x = static_cast(rc_left * upscale.width); - rc.y = static_cast(rc_top * upscale.height); - rc.width = static_cast(rc_right * upscale.width) - rc.x; - rc.height = static_cast(rc_bottom * upscale.height) - rc.y; - adjustBoundingBox(rc); // TODO: new option? - - const auto clipped_rc = rc & surface; // TODO: new option? - if (filter_out_of_bounds) { - if (clipped_rc.area() != rc.area()) { - continue; - } - } - out_objects.emplace_back(clipped_rc); - } - } -}; - cv::Rect eyeBox(const cv::Rect &face_rc, float p1_x, float p1_y, float p2_x, float p2_y, float scale = 1.8f) { @@ -335,11 +256,10 @@ int main(int argc, char *argv[]) cmd.printMessage(); return 0; } - cv::GMat in; cv::GMat faces = cv::gapi::infer(in); cv::GOpaque sz = cv::gapi::streaming::size(in); - cv::GArray faces_rc = custom::ParseSSD::on(faces, sz, true); + cv::GArray faces_rc = cv::gapi::parseSSD(faces, sz, 0.5f, true, true); cv::GArray angles_y, angles_p, angles_r; std::tie(angles_y, angles_p, angles_r) = cv::gapi::infer(faces_rc, in); cv::GArray heads_pos = custom::ProcessPoses::on(angles_y, angles_p, angles_r); @@ -386,7 +306,6 @@ int main(int argc, char *argv[]) }.cfgInputLayers({"left_eye_image", "right_eye_image", "head_pose_angles"}); auto kernels = cv::gapi::kernels< custom::OCVSize - , custom::OCVParseSSD , custom::OCVParseEyes , custom::OCVProcessPoses>(); auto networks = cv::gapi::networks(face_net, head_net, landmarks_net, gaze_net); diff --git a/modules/gapi/samples/infer_ie_onnx_hybrid.cpp b/modules/gapi/samples/infer_ie_onnx_hybrid.cpp index b8612a25ca..594c592573 100644 --- a/modules/gapi/samples/infer_ie_onnx_hybrid.cpp +++ b/modules/gapi/samples/infer_ie_onnx_hybrid.cpp @@ -156,7 +156,6 @@ int main(int argc, char *argv[]) auto in_src = cv::gapi::wip::make_src(input); pipeline.setSource(cv::gin(in_src)); - pipeline.start(); cv::util::optional out_frame; cv::util::optional> out_faces; @@ -167,8 +166,13 @@ int main(int argc, char *argv[]) std::vector last_emotions; cv::VideoWriter writer; + cv::TickMeter tm; + std::size_t frames = 0u; + tm.start(); + pipeline.start(); while (pipeline.pull(cv::gout(out_frame, out_faces, out_emotions))) { + ++frames; if (out_faces && out_emotions) { last_faces = *out_faces; last_emotions = *out_emotions; @@ -191,5 +195,7 @@ int main(int argc, char *argv[]) cv::waitKey(1); } } + tm.stop(); + std::cout << "Processed " << frames << " frames" << " (" << frames / tm.getTimeSec() << " FPS)" << std::endl; return 0; } diff --git a/modules/gapi/samples/infer_single_roi.cpp b/modules/gapi/samples/infer_single_roi.cpp index 6f02770a66..e9c26a9b63 100644 --- a/modules/gapi/samples/infer_single_roi.cpp +++ b/modules/gapi/samples/infer_single_roi.cpp @@ -13,6 +13,7 @@ #include #include #include +#include const std::string keys = "{ h help | | Print this help message }" @@ -69,36 +70,18 @@ using GRect = cv::GOpaque; using GSize = cv::GOpaque; using GPrims = cv::GArray; -G_API_OP(GetSize, , "sample.custom.get-size") { - static cv::GOpaqueDesc outMeta(const cv::GMatDesc &) { - return cv::empty_gopaque_desc(); - } -}; - G_API_OP(LocateROI, , "sample.custom.locate-roi") { static cv::GOpaqueDesc outMeta(const cv::GMatDesc &) { return cv::empty_gopaque_desc(); } }; -G_API_OP(ParseSSD, , "sample.custom.parse-ssd") { - static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GOpaqueDesc &, const cv::GOpaqueDesc &) { - return cv::empty_array_desc(); - } -}; - G_API_OP(BBoxes, , "sample.custom.b-boxes") { static cv::GArrayDesc outMeta(const cv::GArrayDesc &, const cv::GOpaqueDesc &) { return cv::empty_array_desc(); } }; -GAPI_OCV_KERNEL(OCVGetSize, GetSize) { - static void run(const cv::Mat &in, cv::Size &out) { - out = {in.cols, in.rows}; - } -}; - GAPI_OCV_KERNEL(OCVLocateROI, LocateROI) { // This is the place where we can run extra analytics // on the input image frame and select the ROI (region @@ -124,55 +107,6 @@ GAPI_OCV_KERNEL(OCVLocateROI, LocateROI) { } }; -GAPI_OCV_KERNEL(OCVParseSSD, ParseSSD) { - static void run(const cv::Mat &in_ssd_result, - const cv::Rect &in_roi, - const cv::Size &in_parent_size, - std::vector &out_objects) { - const auto &in_ssd_dims = in_ssd_result.size; - CV_Assert(in_ssd_dims.dims() == 4u); - - const int MAX_PROPOSALS = in_ssd_dims[2]; - const int OBJECT_SIZE = in_ssd_dims[3]; - CV_Assert(OBJECT_SIZE == 7); // fixed SSD object size - - const cv::Size up_roi = in_roi.size(); - const cv::Rect surface({0,0}, in_parent_size); - - out_objects.clear(); - - const float *data = in_ssd_result.ptr(); - for (int i = 0; i < MAX_PROPOSALS; i++) { - const float image_id = data[i * OBJECT_SIZE + 0]; - const float label = data[i * OBJECT_SIZE + 1]; - const float confidence = data[i * OBJECT_SIZE + 2]; - const float rc_left = data[i * OBJECT_SIZE + 3]; - const float rc_top = data[i * OBJECT_SIZE + 4]; - const float rc_right = data[i * OBJECT_SIZE + 5]; - const float rc_bottom = data[i * OBJECT_SIZE + 6]; - (void) label; // unused - - if (image_id < 0.f) { - break; // marks end-of-detections - } - if (confidence < 0.5f) { - continue; // skip objects with low confidence - } - - // map relative coordinates to the original image scale - // taking the ROI into account - cv::Rect rc; - rc.x = static_cast(rc_left * up_roi.width); - rc.y = static_cast(rc_top * up_roi.height); - rc.width = static_cast(rc_right * up_roi.width) - rc.x; - rc.height = static_cast(rc_bottom * up_roi.height) - rc.y; - rc.x += in_roi.x; - rc.y += in_roi.y; - out_objects.emplace_back(rc & surface); - } - } -}; - GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) { // This kernel converts the rectangles into G-API's // rendering primitives @@ -211,9 +145,7 @@ int main(int argc, char *argv[]) cmd.get("faced"), // device specifier }; auto kernels = cv::gapi::kernels - < custom::OCVGetSize - , custom::OCVLocateROI - , custom::OCVParseSSD + (); auto networks = cv::gapi::networks(face_net); @@ -222,16 +154,17 @@ int main(int argc, char *argv[]) cv::GStreamingCompiled pipeline; auto inputs = cv::gin(cv::gapi::wip::make_src(input)); + cv::GMat in; + cv::GOpaque sz = cv::gapi::streaming::size(in); if (opt_roi.has_value()) { // Use the value provided by user std::cout << "Will run inference for static region " << opt_roi.value() << " only" << std::endl; - cv::GMat in; cv::GOpaque in_roi; auto blob = cv::gapi::infer(in_roi, in); - auto rcs = custom::ParseSSD::on(blob, in_roi, custom::GetSize::on(in)); + cv::GArray rcs = cv::gapi::parseSSD(blob, sz, 0.5f, true, true); auto out = cv::gapi::wip::draw::render3ch(in, custom::BBoxes::on(rcs, in_roi)); pipeline = cv::GComputation(cv::GIn(in, in_roi), cv::GOut(out)) .compileStreaming(cv::compile_args(kernels, networks)); @@ -242,10 +175,9 @@ int main(int argc, char *argv[]) // Automatically detect ROI to infer. Make it output parameter std::cout << "ROI is not set or invalid. Locating it automatically" << std::endl; - cv::GMat in; cv::GOpaque roi = custom::LocateROI::on(in); auto blob = cv::gapi::infer(roi, in); - auto rcs = custom::ParseSSD::on(blob, roi, custom::GetSize::on(in)); + cv::GArray rcs = cv::gapi::parseSSD(blob, sz, 0.5f, true, true); auto out = cv::gapi::wip::draw::render3ch(in, custom::BBoxes::on(rcs, roi)); pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)) .compileStreaming(cv::compile_args(kernels, networks)); @@ -256,17 +188,15 @@ int main(int argc, char *argv[]) pipeline.start(); cv::Mat out; - int framesCount = 0; - cv::TickMeter t; - t.start(); + size_t frames = 0u; + cv::TickMeter tm; + tm.start(); while (pipeline.pull(cv::gout(out))) { cv::imshow("Out", out); cv::waitKey(1); - framesCount++; + ++frames; } - t.stop(); - std::cout << "Elapsed time: " << t.getTimeSec() << std::endl; - std::cout << "FPS: " << framesCount / (t.getTimeSec() ? t.getTimeSec() : 1) << std::endl; - std::cout << "framesCount: " << framesCount << std::endl; + tm.stop(); + std::cout << "Processed " << frames << " frames" << " (" << frames / tm.getTimeSec() << " FPS)" << std::endl; return 0; } diff --git a/modules/gapi/samples/infer_ssd_onnx.cpp b/modules/gapi/samples/infer_ssd_onnx.cpp index fc26ca1e36..c6073a2f7b 100644 --- a/modules/gapi/samples/infer_ssd_onnx.cpp +++ b/modules/gapi/samples/infer_ssd_onnx.cpp @@ -14,6 +14,7 @@ #include #include #include +#include namespace custom { @@ -23,71 +24,12 @@ using GDetections = cv::GArray; using GSize = cv::GOpaque; using GPrims = cv::GArray; -G_API_OP(GetSize, , "sample.custom.get-size") { - static cv::GOpaqueDesc outMeta(const cv::GMatDesc &) { - return cv::empty_gopaque_desc(); - } -}; -G_API_OP(ParseSSD, , "sample.custom.parse-ssd") { - static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GOpaqueDesc &) { - return cv::empty_array_desc(); - } -}; G_API_OP(BBoxes, , "sample.custom.b-boxes") { static cv::GArrayDesc outMeta(const cv::GArrayDesc &) { return cv::empty_array_desc(); } }; -GAPI_OCV_KERNEL(OCVGetSize, GetSize) { - static void run(const cv::Mat &in, cv::Size &out) { - out = {in.cols, in.rows}; - } -}; -GAPI_OCV_KERNEL(OCVParseSSD, ParseSSD) { - static void run(const cv::Mat &in_ssd_result, - const cv::Size &in_parent_size, - std::vector &out_objects) { - const auto &in_ssd_dims = in_ssd_result.size; - CV_Assert(in_ssd_dims.dims() == 4u); - - const int MAX_PROPOSALS = in_ssd_dims[2]; - const int OBJECT_SIZE = in_ssd_dims[3]; - - CV_Assert(OBJECT_SIZE == 7); // fixed SSD object size - - const cv::Rect surface({0,0}, in_parent_size); - - out_objects.clear(); - - const float *data = in_ssd_result.ptr(); - for (int i = 0; i < MAX_PROPOSALS; i++) { - const float image_id = data[i * OBJECT_SIZE + 0]; - const float label = data[i * OBJECT_SIZE + 1]; - const float confidence = data[i * OBJECT_SIZE + 2]; - const float rc_left = data[i * OBJECT_SIZE + 3]; - const float rc_top = data[i * OBJECT_SIZE + 4]; - const float rc_right = data[i * OBJECT_SIZE + 5]; - const float rc_bottom = data[i * OBJECT_SIZE + 6]; - (void) label; // unused - - if (image_id < 0.f) { - break; // marks end-of-detections - } - if (confidence < 0.5f) { - continue; // skip objects with low confidence - } - - // map relative coordinates to the original image scale - cv::Rect rc; - rc.x = static_cast(rc_left * in_parent_size.width); - rc.y = static_cast(rc_top * in_parent_size.height); - rc.width = static_cast(rc_right * in_parent_size.width) - rc.x; - rc.height = static_cast(rc_bottom * in_parent_size.height) - rc.y; - out_objects.emplace_back(rc & surface); - } - } -}; GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) { // This kernel converts the rectangles into G-API's // rendering primitives @@ -151,7 +93,6 @@ void remap_ssd_ports(const std::unordered_map &onnx, } } // anonymous namespace - const std::string keys = "{ h help | | Print this help message }" "{ input | | Path to the input video file }" @@ -175,15 +116,14 @@ int main(int argc, char *argv[]) auto obj_net = cv::gapi::onnx::Params{obj_model_path} .cfgOutputLayers({"detection_output"}) .cfgPostProc({cv::GMatDesc{CV_32F, {1,1,200,7}}}, remap_ssd_ports); - auto kernels = cv::gapi::kernels< custom::OCVGetSize - , custom::OCVParseSSD - , custom::OCVBBoxes>(); + auto kernels = cv::gapi::kernels(); auto networks = cv::gapi::networks(obj_net); // Now build the graph cv::GMat in; auto blob = cv::gapi::infer(in); - auto rcs = custom::ParseSSD::on(blob, custom::GetSize::on(in)); + cv::GArray rcs = + cv::gapi::parseSSD(blob, cv::gapi::streaming::size(in), 0.5f, true, true); auto out = cv::gapi::wip::draw::render3ch(in, custom::BBoxes::on(rcs)); cv::GStreamingCompiled pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)) .compileStreaming(cv::compile_args(kernels, networks)); @@ -192,12 +132,16 @@ int main(int argc, char *argv[]) // The execution part pipeline.setSource(std::move(inputs)); - pipeline.start(); + cv::TickMeter tm; cv::VideoWriter writer; - + size_t frames = 0u; cv::Mat outMat; + + tm.start(); + pipeline.start(); while (pipeline.pull(cv::gout(outMat))) { + ++frames; cv::imshow("Out", outMat); cv::waitKey(1); if (!output.empty()) { @@ -209,5 +153,7 @@ int main(int argc, char *argv[]) writer << outMat; } } + tm.stop(); + std::cout << "Processed " << frames << " frames" << " (" << frames / tm.getTimeSec() << " FPS)" << std::endl; return 0; } diff --git a/modules/gapi/samples/onevpl_infer_single_roi.cpp b/modules/gapi/samples/onevpl_infer_single_roi.cpp index 9da64c99d3..fec0f0d043 100644 --- a/modules/gapi/samples/onevpl_infer_single_roi.cpp +++ b/modules/gapi/samples/onevpl_infer_single_roi.cpp @@ -13,6 +13,7 @@ #include #include #include // CommandLineParser +#include #ifdef HAVE_INF_ENGINE #include // ParamMap @@ -126,12 +127,6 @@ G_API_OP(LocateROI, , "sample.custom.locate-roi") { } }; -G_API_OP(ParseSSD, , "sample.custom.parse-ssd") { - static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GOpaqueDesc &, const cv::GOpaqueDesc &) { - return cv::empty_array_desc(); - } -}; - G_API_OP(BBoxes, , "sample.custom.b-boxes") { static cv::GArrayDesc outMeta(const cv::GArrayDesc &, const cv::GOpaqueDesc &) { return cv::empty_array_desc(); @@ -163,55 +158,6 @@ GAPI_OCV_KERNEL(OCVLocateROI, LocateROI) { } }; -GAPI_OCV_KERNEL(OCVParseSSD, ParseSSD) { - static void run(const cv::Mat &in_ssd_result, - const cv::Rect &in_roi, - const cv::Size &in_parent_size, - std::vector &out_objects) { - const auto &in_ssd_dims = in_ssd_result.size; - CV_Assert(in_ssd_dims.dims() == 4u); - - const int MAX_PROPOSALS = in_ssd_dims[2]; - const int OBJECT_SIZE = in_ssd_dims[3]; - CV_Assert(OBJECT_SIZE == 7); // fixed SSD object size - - const cv::Size up_roi = in_roi.size(); - const cv::Rect surface({0,0}, in_parent_size); - - out_objects.clear(); - - const float *data = in_ssd_result.ptr(); - for (int i = 0; i < MAX_PROPOSALS; i++) { - const float image_id = data[i * OBJECT_SIZE + 0]; - const float label = data[i * OBJECT_SIZE + 1]; - const float confidence = data[i * OBJECT_SIZE + 2]; - const float rc_left = data[i * OBJECT_SIZE + 3]; - const float rc_top = data[i * OBJECT_SIZE + 4]; - const float rc_right = data[i * OBJECT_SIZE + 5]; - const float rc_bottom = data[i * OBJECT_SIZE + 6]; - (void) label; // unused - - if (image_id < 0.f) { - break; // marks end-of-detections - } - if (confidence < 0.5f) { - continue; // skip objects with low confidence - } - - // map relative coordinates to the original image scale - // taking the ROI into account - cv::Rect rc; - rc.x = static_cast(rc_left * up_roi.width); - rc.y = static_cast(rc_top * up_roi.height); - rc.width = static_cast(rc_right * up_roi.width) - rc.x; - rc.height = static_cast(rc_bottom * up_roi.height) - rc.y; - rc.x += in_roi.x; - rc.y += in_roi.y; - out_objects.emplace_back(rc & surface); - } - } -}; - GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) { // This kernel converts the rectangles into G-API's // rendering primitives @@ -350,7 +296,6 @@ int main(int argc, char *argv[]) { auto kernels = cv::gapi::kernels < custom::OCVLocateROI - , custom::OCVParseSSD , custom::OCVBBoxes>(); auto networks = cv::gapi::networks(face_net); @@ -379,7 +324,7 @@ int main(int argc, char *argv[]) { auto size = cv::gapi::streaming::size(in); auto roi = custom::LocateROI::on(size); auto blob = cv::gapi::infer(roi, in); - auto rcs = custom::ParseSSD::on(blob, roi, size); + cv::GArray rcs = cv::gapi::parseSSD(blob, size, 0.5f, true, true); auto out_frame = cv::gapi::wip::draw::renderFrame(in, custom::BBoxes::on(rcs, roi)); auto out = cv::gapi::streaming::BGR(out_frame); @@ -398,8 +343,8 @@ int main(int argc, char *argv[]) { pipeline.setSource(std::move(cap)); pipeline.start(); - int framesCount = 0; - cv::TickMeter t; + size_t frames = 0u; + cv::TickMeter tm; cv::VideoWriter writer; if (!output.empty() && !writer.isOpened()) { const auto sz = cv::Size{frame_descr.size.width, frame_descr.size.height}; @@ -408,20 +353,17 @@ int main(int argc, char *argv[]) { } cv::Mat outMat; - t.start(); + tm.start(); while (pipeline.pull(cv::gout(outMat))) { cv::imshow("Out", outMat); cv::waitKey(1); if (!output.empty()) { writer << outMat; } - framesCount++; + ++frames; } - t.stop(); - std::cout << "Elapsed time: " << t.getTimeSec() << std::endl; - std::cout << "FPS: " << framesCount / t.getTimeSec() << std::endl; - std::cout << "framesCount: " << framesCount << std::endl; - + tm.stop(); + std::cout << "Processed " << frames << " frames" << " (" << frames / tm.getTimeSec() << " FPS)" << std::endl; return 0; } diff --git a/modules/gapi/samples/privacy_masking_camera.cpp b/modules/gapi/samples/privacy_masking_camera.cpp index 6e8411c674..e7c0a531f2 100644 --- a/modules/gapi/samples/privacy_masking_camera.cpp +++ b/modules/gapi/samples/privacy_masking_camera.cpp @@ -13,6 +13,7 @@ #include #include #include +#include const std::string about = "This is an OpenCV-based version of Privacy Masking Camera example"; @@ -49,12 +50,6 @@ G_API_NET(FaceDetector, , "face-detector" using GDetections = cv::GArray; -G_API_OP(ParseSSD, , "custom.privacy_masking.postproc") { - static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GMatDesc &, int) { - return cv::empty_array_desc(); - } -}; - using GPrims = cv::GArray; G_API_OP(ToMosaic, , "custom.privacy_masking.to_mosaic") { @@ -63,53 +58,6 @@ G_API_OP(ToMosaic, , "custom.privacy_masking.t } }; -GAPI_OCV_KERNEL(OCVParseSSD, ParseSSD) { - static void run(const cv::Mat &in_ssd_result, - const cv::Mat &in_frame, - const int filter_label, - std::vector &out_objects) { - const auto &in_ssd_dims = in_ssd_result.size; - CV_Assert(in_ssd_dims.dims() == 4u); - - const int MAX_PROPOSALS = in_ssd_dims[2]; - const int OBJECT_SIZE = in_ssd_dims[3]; - CV_Assert(OBJECT_SIZE == 7); // fixed SSD object size - - const cv::Size upscale = in_frame.size(); - const cv::Rect surface({0,0}, upscale); - - out_objects.clear(); - - const float *data = in_ssd_result.ptr(); - for (int i = 0; i < MAX_PROPOSALS; i++) { - const float image_id = data[i * OBJECT_SIZE + 0]; - const float label = data[i * OBJECT_SIZE + 1]; - const float confidence = data[i * OBJECT_SIZE + 2]; - const float rc_left = data[i * OBJECT_SIZE + 3]; - const float rc_top = data[i * OBJECT_SIZE + 4]; - const float rc_right = data[i * OBJECT_SIZE + 5]; - const float rc_bottom = data[i * OBJECT_SIZE + 6]; - - if (image_id < 0.f) { - break; // marks end-of-detections - } - if (confidence < 0.5f) { - continue; // skip objects with low confidence - } - if (filter_label != -1 && static_cast(label) != filter_label) { - continue; // filter out object classes if filter is specified - } - - cv::Rect rc; // map relative coordinates to the original image scale - rc.x = static_cast(rc_left * upscale.width); - rc.y = static_cast(rc_top * upscale.height); - rc.width = static_cast(rc_right * upscale.width) - rc.x; - rc.height = static_cast(rc_bottom * upscale.height) - rc.y; - out_objects.emplace_back(rc & surface); - } - } -}; - GAPI_OCV_KERNEL(OCVToMosaic, ToMosaic) { static void run(const std::vector &in_plate_rcs, const std::vector &in_face_rcs, @@ -150,10 +98,13 @@ int main(int argc, char *argv[]) cv::GMat blob_faces = cv::gapi::infer(in); // VehLicDetector from Open Model Zoo marks vehicles with label "1" and // license plates with label "2", filter out license plates only. - cv::GArray rc_plates = custom::ParseSSD::on(blob_plates, in, 2); + cv::GOpaque sz = cv::gapi::streaming::size(in); + cv::GArray rc_plates, rc_faces; + cv::GArray labels; + std::tie(rc_plates, labels) = cv::gapi::parseSSD(blob_plates, sz, 0.5f, 2); // Face detector produces faces only so there's no need to filter by label, // pass "-1". - cv::GArray rc_faces = custom::ParseSSD::on(blob_faces, in, -1); + std::tie(rc_faces, labels) = cv::gapi::parseSSD(blob_faces, sz, 0.5f, -1); cv::GMat out = cv::gapi::wip::draw::render3ch(in, custom::ToMosaic::on(rc_plates, rc_faces)); cv::GComputation graph(in, out); @@ -169,7 +120,7 @@ int main(int argc, char *argv[]) weights_path(face_model_path), // path to weights cmd.get("faced"), // device specifier }; - auto kernels = cv::gapi::kernels(); + auto kernels = cv::gapi::kernels(); auto networks = cv::gapi::networks(plate_net, face_net); cv::TickMeter tm; diff --git a/modules/gapi/samples/semantic_segmentation.cpp b/modules/gapi/samples/semantic_segmentation.cpp index 4cdb14cc5c..fd3ec27750 100644 --- a/modules/gapi/samples/semantic_segmentation.cpp +++ b/modules/gapi/samples/semantic_segmentation.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include const std::string keys = @@ -117,10 +118,7 @@ GAPI_OCV_KERNEL(OCVPostProcessing, PostProcessing) { cv::Mat mask_img; classesToColors(classes, mask_img); - cv::resize(mask_img, out, in.size()); - const float blending = 0.3f; - out = in * blending + out * (1 - blending); } }; } // namespace custom @@ -148,7 +146,10 @@ int main(int argc, char *argv[]) { // Now build the graph cv::GMat in; cv::GMat out_blob = cv::gapi::infer(in); - cv::GMat out = custom::PostProcessing::on(in, out_blob); + cv::GMat post_proc_out = custom::PostProcessing::on(in, out_blob); + cv::GMat blending_in = in * 0.3f; + cv::GMat blending_out = post_proc_out * 0.7f; + cv::GMat out = blending_in + blending_out; cv::GStreamingCompiled pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)) .compileStreaming(cv::compile_args(kernels, networks)); @@ -156,11 +157,16 @@ int main(int argc, char *argv[]) { // The execution part pipeline.setSource(std::move(inputs)); - pipeline.start(); cv::VideoWriter writer; + cv::TickMeter tm; cv::Mat outMat; + + std::size_t frames = 0u; + tm.start(); + pipeline.start(); while (pipeline.pull(cv::gout(outMat))) { + ++frames; cv::imshow("Out", outMat); cv::waitKey(1); if (!output.empty()) { @@ -172,5 +178,7 @@ int main(int argc, char *argv[]) { writer << outMat; } } + tm.stop(); + std::cout << "Processed " << frames << " frames" << " (" << frames / tm.getTimeSec() << " FPS)" << std::endl; return 0; } From 2deb38d61580ac65e3cfe037a2e39952b0e652b3 Mon Sep 17 00:00:00 2001 From: Orest Chura Date: Fri, 26 Nov 2021 19:40:36 +0300 Subject: [PATCH 096/226] Merge pull request #21083 from OrestChura:oc/fix_coverity_vino_issues [G-API] Fixed Coverity issues * Fixed Coverity issues - VectorRef&OpaqueRef m_kind = CV_UNKNOWN - added same-type overload for saturate() - sanitized resize value in ByteMemoryInStream::operator>> (std::string& str) - handled throws from ~GStreamingExecutor() * Catching exception by const ref * Addressing Sergey's comments * Applied enable_if semanitcs to saturate(x, round) too * Removed uncaught_exception, made destructor noexcept back * Split Fluid ConvertTo to multiple functions to avoid ifs; added CV_ALWAYS_INLINE * Added FIXME to address throwings from stop() * Fix standalone * Addressing comments * Guarded SIMD optimizations properly * Removed excess parameter from simd_impl functions --- modules/gapi/include/opencv2/gapi/garray.hpp | 2 +- modules/gapi/include/opencv2/gapi/gopaque.hpp | 2 +- .../gapi/include/opencv2/gapi/own/cvdefs.hpp | 10 + .../include/opencv2/gapi/own/saturate.hpp | 67 +++---- .../src/backends/common/serialization.cpp | 2 +- .../gapi/src/backends/fluid/gfluidcore.cpp | 175 ++++++++++-------- .../gapi/src/executor/gstreamingexecutor.cpp | 14 +- 7 files changed, 157 insertions(+), 115 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/garray.hpp b/modules/gapi/include/opencv2/gapi/garray.hpp index d7b365f7f9..17b03332e0 100644 --- a/modules/gapi/include/opencv2/gapi/garray.hpp +++ b/modules/gapi/include/opencv2/gapi/garray.hpp @@ -236,7 +236,7 @@ namespace detail class VectorRef { std::shared_ptr m_ref; - cv::detail::OpaqueKind m_kind; + cv::detail::OpaqueKind m_kind = cv::detail::OpaqueKind::CV_UNKNOWN; template inline void check() const { diff --git a/modules/gapi/include/opencv2/gapi/gopaque.hpp b/modules/gapi/include/opencv2/gapi/gopaque.hpp index 979a9db8c4..f77795c506 100644 --- a/modules/gapi/include/opencv2/gapi/gopaque.hpp +++ b/modules/gapi/include/opencv2/gapi/gopaque.hpp @@ -232,7 +232,7 @@ namespace detail class OpaqueRef { std::shared_ptr m_ref; - cv::detail::OpaqueKind m_kind; + cv::detail::OpaqueKind m_kind = cv::detail::OpaqueKind::CV_UNKNOWN; template inline void check() const { diff --git a/modules/gapi/include/opencv2/gapi/own/cvdefs.hpp b/modules/gapi/include/opencv2/gapi/own/cvdefs.hpp index 9ec0f89ed8..b0c91d3530 100644 --- a/modules/gapi/include/opencv2/gapi/own/cvdefs.hpp +++ b/modules/gapi/include/opencv2/gapi/own/cvdefs.hpp @@ -84,6 +84,16 @@ typedef unsigned short ushort; // cvdef.h: +#ifndef CV_ALWAYS_INLINE +# if defined(__GNUC__) && (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1)) +# define CV_ALWAYS_INLINE inline __attribute__((always_inline)) +# elif defined(_MSC_VER) +# define CV_ALWAYS_INLINE __forceinline +# else +# define CV_ALWAYS_INLINE inline +# endif +#endif + #define CV_MAT_CN_MASK ((CV_CN_MAX - 1) << CV_CN_SHIFT) #define CV_MAT_CN(flags) ((((flags) & CV_MAT_CN_MASK) >> CV_CN_SHIFT) + 1) #define CV_MAT_TYPE_MASK (CV_DEPTH_MAX*CV_CN_MAX - 1) diff --git a/modules/gapi/include/opencv2/gapi/own/saturate.hpp b/modules/gapi/include/opencv2/gapi/own/saturate.hpp index 5b232479ea..74eaecf57e 100644 --- a/modules/gapi/include/opencv2/gapi/own/saturate.hpp +++ b/modules/gapi/include/opencv2/gapi/own/saturate.hpp @@ -11,9 +11,9 @@ #include #include -#include #include +#include namespace cv { namespace gapi { namespace own { //----------------------------- @@ -22,16 +22,12 @@ namespace cv { namespace gapi { namespace own { // //----------------------------- -template -static inline DST saturate(SRC x) +template::value && + std::is_integral::value && + std::is_integral::value> > +static CV_ALWAYS_INLINE DST saturate(SRC x) { - // only integral types please! - GAPI_DbgAssert(std::is_integral::value && - std::is_integral::value); - - if (std::is_same::value) - return static_cast(x); - if (sizeof(DST) > sizeof(SRC)) return static_cast(x); @@ -44,38 +40,35 @@ static inline DST saturate(SRC x) std::numeric_limits::max(): static_cast(x); } +template +static CV_ALWAYS_INLINE T saturate(T x) +{ + return x; +} +template::value, bool> = true > +static CV_ALWAYS_INLINE DST saturate(SRC x, R) +{ + return static_cast(x); +} +template::value && + std::is_integral::value , bool> = true > +static CV_ALWAYS_INLINE DST saturate(SRC x, R) +{ + return saturate(x); +} // Note, that OpenCV rounds differently: // - like std::round() for add, subtract // - like std::rint() for multiply, divide -template -static inline DST saturate(SRC x, R round) +template::value && + std::is_floating_point::value, bool> = true > +static CV_ALWAYS_INLINE DST saturate(SRC x, R round) { - if (std::is_floating_point::value) - { - return static_cast(x); - } - else if (std::is_integral::value) - { - GAPI_DbgAssert(std::is_integral::value && - std::is_integral::value); - return saturate(x); - } - else - { - GAPI_DbgAssert(std::is_integral::value && - std::is_floating_point::value); -#ifdef _WIN32 -// Suppress warning about converting x to floating-point -// Note that x is already floating-point at this point -#pragma warning(disable: 4244) -#endif - int ix = static_cast(round(x)); -#ifdef _WIN32 -#pragma warning(default: 4244) -#endif - return saturate(ix); - } + int ix = static_cast(round(x)); + return saturate(ix); } // explicit suffix 'd' for double type diff --git a/modules/gapi/src/backends/common/serialization.cpp b/modules/gapi/src/backends/common/serialization.cpp index 619b2feb74..638ce2f025 100644 --- a/modules/gapi/src/backends/common/serialization.cpp +++ b/modules/gapi/src/backends/common/serialization.cpp @@ -928,7 +928,7 @@ IIStream& ByteMemoryInStream::operator>> (std::string& str) { if (sz == 0u) { str.clear(); } else { - str.resize(sz); + str.resize(static_cast(sz)); for (auto &&i : ade::util::iota(sz)) { *this >> str[i]; } } return *this; diff --git a/modules/gapi/src/backends/fluid/gfluidcore.cpp b/modules/gapi/src/backends/fluid/gfluidcore.cpp index d68ae733dd..7a3d90acc7 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore.cpp +++ b/modules/gapi/src/backends/fluid/gfluidcore.cpp @@ -1955,93 +1955,122 @@ GAPI_FLUID_KERNEL(GFluidLUT, cv::gapi::core::GLUT, false) // //------------------------- +#if CV_SIMD128 +template +CV_ALWAYS_INLINE int run_convertto_simd(DST*, const SRC*, int) +{ + return 0; +} +CV_ALWAYS_INLINE int run_convertto_simd(uchar *out, const float *in, const int length) +{ + int l = 0; + for (; l <= length - 16; l += 16) + { + v_int32x4 i0, i1, i2, i3; + i0 = v_round( v_load( (float*)& in[l ] ) ); + i1 = v_round( v_load( (float*)& in[l + 4] ) ); + i2 = v_round( v_load( (float*)& in[l + 8] ) ); + i3 = v_round( v_load( (float*)& in[l + 12] ) ); + + v_uint16x8 us0, us1; + us0 = v_pack_u(i0, i1); + us1 = v_pack_u(i2, i3); + + v_uint8x16 uc; + uc = v_pack(us0, us1); + v_store((uchar*)& out[l], uc); + } + return l; +} +CV_ALWAYS_INLINE int run_convertto_simd(ushort *out, const float *in, const int length) +{ + int l = 0; + for (; l <= length - 8; l += 8) + { + v_int32x4 i0, i1; + i0 = v_round( v_load( (float*)& in[l ] ) ); + i1 = v_round( v_load( (float*)& in[l + 4] ) ); + + v_uint16x8 us; + us = v_pack_u(i0, i1); + v_store((ushort*)& out[l], us); + } + return l; +} +#endif + +template::value && + std::is_floating_point::value, bool> = true > +CV_ALWAYS_INLINE void run_convertto(DST *out, const SRC *in, const int length) +{ + // manual SIMD if need rounding + static_assert(std::is_same::value, "64-bit floating-point source is not supported"); + int l = 0; // cycle index +#if CV_SIMD128 + l = run_convertto_simd(out, in, length); +#endif + // tail of SIMD cycle + for (; l < length; l++) + { + out[l] = saturate(in[l], rintf); + } +} +template::value && + std::is_integral::value , bool> = true > +CV_ALWAYS_INLINE void run_convertto(DST *out, const SRC *in, const int length) +{ + for (int l = 0; l < length; l++) + { + out[l] = saturate(in[l]); + } +} +template::value, bool> = true > +CV_ALWAYS_INLINE void run_convertto(DST *out, const SRC *in, const int length) +{ + static_assert(!std::is_same::value, "64-bit floating-point source is not supported"); + for (int l = 0; l < length; l++) + { + out[l] = static_cast(in[l]); + } +} + +template +CV_ALWAYS_INLINE void run_convertto(DST *out, const SRC *in, const float alpha, const float beta, + const int length) +{ + static_assert(!std::is_same::value, "64-bit floating-point source is not supported"); + // TODO: optimize if alpha and beta and data are integral + for (int l = 0; l < length; l++) + { + out[l] = saturate(in[l] * alpha + beta, rintf); + } +} + template static void run_convertto(Buffer &dst, const View &src, double _alpha, double _beta) { const auto *in = src.InLine(0); auto *out = dst.OutLine(); - int width = dst.length(); - int chan = dst.meta().chan; - int length = width * chan; + const int width = dst.length(); + const int chan = dst.meta().chan; + const int length = width * chan; // NB: don't do this if SRC or DST is 64-bit - auto alpha = static_cast( _alpha ); - auto beta = static_cast( _beta ); + const auto alpha = static_cast( _alpha ); + const auto beta = static_cast( _beta ); // compute faster if no alpha no beta - if (alpha == 1 && beta == 0) + if (1.f == alpha && 0.f == beta) { - // manual SIMD if need rounding - if (std::is_integral::value && std::is_floating_point::value) - { - GAPI_Assert(( std::is_same::value )); - - int l = 0; // cycle index - - #if CV_SIMD128 - if (std::is_same::value) - { - for (; l <= length-16; l+=16) - { - v_int32x4 i0, i1, i2, i3; - i0 = v_round( v_load( (float*)& in[l ] ) ); - i1 = v_round( v_load( (float*)& in[l + 4] ) ); - i2 = v_round( v_load( (float*)& in[l + 8] ) ); - i3 = v_round( v_load( (float*)& in[l + 12] ) ); - - v_uint16x8 us0, us1; - us0 = v_pack_u(i0, i1); - us1 = v_pack_u(i2, i3); - - v_uint8x16 uc; - uc = v_pack(us0, us1); - v_store((uchar*)& out[l], uc); - } - } - if (std::is_same::value) - { - for (; l <= length-8; l+=8) - { - v_int32x4 i0, i1; - i0 = v_round( v_load( (float*)& in[l ] ) ); - i1 = v_round( v_load( (float*)& in[l + 4] ) ); - - v_uint16x8 us; - us = v_pack_u(i0, i1); - v_store((ushort*)& out[l], us); - } - } - #endif - - // tail of SIMD cycle - for (; l < length; l++) - { - out[l] = saturate(in[l], rintf); - } - } - else if (std::is_integral::value) // here SRC is integral - { - for (int l=0; l < length; l++) - { - out[l] = saturate(in[l]); - } - } - else // DST is floating-point, SRC is any - { - for (int l=0; l < length; l++) - { - out[l] = static_cast(in[l]); - } - } + run_convertto(out, in, length); } else // if alpha or beta is non-trivial { - // TODO: optimize if alpha and beta and data are integral - for (int l=0; l < length; l++) - { - out[l] = saturate(in[l]*alpha + beta, rintf); - } + run_convertto(out, in, alpha, beta, length); } } diff --git a/modules/gapi/src/executor/gstreamingexecutor.cpp b/modules/gapi/src/executor/gstreamingexecutor.cpp index ef93833143..d15e17ea28 100644 --- a/modules/gapi/src/executor/gstreamingexecutor.cpp +++ b/modules/gapi/src/executor/gstreamingexecutor.cpp @@ -24,6 +24,8 @@ #include "backends/streaming/gstreamingbackend.hpp" // GCopy #include "compiler/gcompiler.hpp" // for compileIslands +#include + #include "executor/gstreamingexecutor.hpp" #include @@ -1382,8 +1384,16 @@ cv::gimpl::GStreamingExecutor::GStreamingExecutor(std::unique_ptr && cv::gimpl::GStreamingExecutor::~GStreamingExecutor() { - if (state == State::READY || state == State::RUNNING) - stop(); + // FIXME: this is a temporary try-catch exception hadling. + // Need to eliminate throwings from stop() + try { + if (state == State::READY || state == State::RUNNING) + stop(); + } catch (const std::exception& e) { + std::stringstream message; + message << "~GStreamingExecutor() threw exception with message '" << e.what() << "'\n"; + GAPI_LOG_WARNING(NULL, message.str()); + } } void cv::gimpl::GStreamingExecutor::setSource(GRunArgs &&ins) From e20fe421e7c3a40c48d90db30ebd4752671492a0 Mon Sep 17 00:00:00 2001 From: Orest Chura Date: Fri, 26 Nov 2021 19:42:12 +0300 Subject: [PATCH 097/226] Merge pull request #21103 from OrestChura:oc/fix_1D_Mat_RMat_View_issue [G-API] Fix issue of getting 1D Mat out of RMat::View * Fix issue of getting 1D Mat out of RMat::View - added test - fixed for standalone too (removed Assert(dims.empty())) * Fixed asVeiw() function for standalone * Put more detailed comment --- modules/gapi/src/backends/common/gbackend.hpp | 18 +++++++++++++++--- modules/gapi/test/rmat/rmat_view_tests.cpp | 9 +++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/modules/gapi/src/backends/common/gbackend.hpp b/modules/gapi/src/backends/common/gbackend.hpp index 7532486dbd..99b8f5dd37 100644 --- a/modules/gapi/src/backends/common/gbackend.hpp +++ b/modules/gapi/src/backends/common/gbackend.hpp @@ -24,8 +24,17 @@ namespace gimpl { inline cv::Mat asMat(RMat::View& v) { #if !defined(GAPI_STANDALONE) - return v.dims().empty() ? cv::Mat(v.rows(), v.cols(), v.type(), v.ptr(), v.step()) - : cv::Mat(v.dims(), v.type(), v.ptr(), v.steps().data()); + if (v.dims().empty()) { + return cv::Mat(v.rows(), v.cols(), v.type(), v.ptr(), v.step()); + } else { + cv::Mat m(v.dims(), v.type(), v.ptr(), v.steps().data()); + if (v.dims().size() == 1) { + // FIXME: cv::Mat() constructor will set m.dims to 2; + // To obtain 1D Mat, we have to set m.dims back to 1 manually + m.dims = 1; + } + return m; + } #else // FIXME: add a check that steps are default return v.dims().empty() ? cv::Mat(v.rows(), v.cols(), v.type(), v.ptr(), v.step()) @@ -41,7 +50,10 @@ namespace gimpl { } return RMat::View(cv::descr_of(m), m.data, steps, std::move(cb)); #else - return RMat::View(cv::descr_of(m), m.data, m.step, std::move(cb)); + return m.dims.empty() + ? RMat::View(cv::descr_of(m), m.data, m.step, std::move(cb)) + // Own Mat doesn't support n-dimensional steps so default ones are used in this case + : RMat::View(cv::descr_of(m), m.data, RMat::View::stepsT{}, std::move(cb)); #endif } diff --git a/modules/gapi/test/rmat/rmat_view_tests.cpp b/modules/gapi/test/rmat/rmat_view_tests.cpp index 14025231a7..d829b6c655 100644 --- a/modules/gapi/test/rmat/rmat_view_tests.cpp +++ b/modules/gapi/test/rmat/rmat_view_tests.cpp @@ -268,4 +268,13 @@ TEST_F(RMatViewCallbackTest, MagazineInteraction) { mag.slot().erase(rc); EXPECT_EQ(1, callbackCalls); } + +TEST(RMatView, Access1DMat) { + cv::Mat m({1}, CV_32FC1); + m.dims = 1; + auto rmat = cv::make_rmat(m); + auto view = rmat.access(cv::RMat::Access::R); + auto out = cv::gimpl::asMat(view); + EXPECT_EQ(1, out.dims); +} } // namespace opencv_test From d668aa7c2446e07902dc9f980568f52ecf9796f8 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 26 Nov 2021 19:18:48 +0100 Subject: [PATCH 098/226] Merge pull request #21121 from cclauss:patch-2 * GitHub Action to lint Python code * Move from automatic events to manual ones * flake8: Do not look for undefined names --- .github/workflows/lint_python.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/lint_python.yml diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml new file mode 100644 index 0000000000..cbc27489db --- /dev/null +++ b/.github/workflows/lint_python.yml @@ -0,0 +1,25 @@ +name: lint_python +on: workflow_dispatch +jobs: + lint_python: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - run: pip install --upgrade pip wheel + - run: pip install bandit black codespell flake8 flake8-2020 flake8-bugbear + flake8-comprehensions isort mypy pytest pyupgrade safety + - run: bandit --recursive --skip B101 . || true # B101 is assert statements + - run: black --check . || true + - run: codespell || true # --ignore-words-list="" --skip="*.css,*.js,*.lock" + - run: flake8 . --count --select=E9,F63,F7 --show-source --statistics + - run: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=88 + --show-source --statistics + - run: isort --check-only --profile black . || true + - run: pip install -r requirements.txt || pip install --editable . || true + - run: mkdir --parents --verbose .mypy_cache + - run: mypy --ignore-missing-imports --install-types --non-interactive . || true + - run: pytest . || true + - run: pytest --doctest-modules . || true + - run: shopt -s globstar && pyupgrade --py36-plus **/*.py || true + - run: safety check From 985aa0423d151e5ba502b0f588c6a93460ab53ac Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 25 Nov 2021 19:56:27 +0000 Subject: [PATCH 099/226] dnn(test): update InferenceEngine tests --- modules/dnn/test/test_backends.cpp | 27 ++- modules/dnn/test/test_caffe_importer.cpp | 16 +- modules/dnn/test/test_common.hpp | 5 +- modules/dnn/test/test_darknet_importer.cpp | 64 ++++- modules/dnn/test/test_halide_layers.cpp | 21 +- modules/dnn/test/test_layers.cpp | 10 +- modules/dnn/test/test_onnx_importer.cpp | 47 +++- modules/dnn/test/test_tf_importer.cpp | 262 ++++++++++++++++++--- modules/dnn/test/test_torch_importer.cpp | 8 +- 9 files changed, 402 insertions(+), 58 deletions(-) diff --git a/modules/dnn/test/test_backends.cpp b/modules/dnn/test/test_backends.cpp index 5426a11a3f..19e0729abb 100644 --- a/modules/dnn/test/test_backends.cpp +++ b/modules/dnn/test/test_backends.cpp @@ -209,8 +209,16 @@ TEST_P(DNNTestNetwork, MobileNet_SSD_Caffe_Different_Width_Height) #if defined(INF_ENGINE_RELEASE) if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); #endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // IE exception: Ngraph operation Transpose with name conv15_2_mbox_conf_perm has dynamic output shape on 0 port, but CPU plug-in supports only static shape + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); +#endif + Mat sample = imread(findDataFile("dnn/street.png")); Mat inp = blobFromImage(sample, 1.0f / 127.5, Size(300, 560), Scalar(127.5, 127.5, 127.5), false); float diffScores = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.029 : 0.0; @@ -280,12 +288,23 @@ TEST_P(DNNTestNetwork, SSD_VGG16) CV_TEST_TAG_DEBUG_VERYLONG); if (backend == DNN_BACKEND_HALIDE && target == DNN_TARGET_CPU) applyTestTag(CV_TEST_TAG_DNN_SKIP_HALIDE); // TODO HALIDE_CPU - double scoreThreshold = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0325 : 0.0; - const float lInf = (target == DNN_TARGET_MYRIAD) ? 0.032 : 0.0; + Mat sample = imread(findDataFile("dnn/street.png")); Mat inp = blobFromImage(sample, 1.0f, Size(300, 300), Scalar(), false); + + float scoreDiff = 0.0, iouDiff = 0.0; + if (target == DNN_TARGET_OPENCL_FP16) + { + scoreDiff = 0.04; + } + else if (target == DNN_TARGET_MYRIAD) + { + scoreDiff = 0.0325; + iouDiff = 0.032; + } + processNet("dnn/VGG_ILSVRC2016_SSD_300x300_iter_440000.caffemodel", - "dnn/ssd_vgg16.prototxt", inp, "detection_out", "", scoreThreshold, lInf); + "dnn/ssd_vgg16.prototxt", inp, "detection_out", "", scoreDiff, iouDiff); expectNoFallbacksFromIE(net); } diff --git a/modules/dnn/test/test_caffe_importer.cpp b/modules/dnn/test/test_caffe_importer.cpp index 1f809f5b95..18b8d5ad82 100644 --- a/modules/dnn/test/test_caffe_importer.cpp +++ b/modules/dnn/test/test_caffe_importer.cpp @@ -489,10 +489,12 @@ TEST_P(Test_Caffe_nets, Colorization) { l1 = 0.5; lInf = 11; } +#if defined(INF_ENGINE_RELEASE) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) { - l1 = 0.26; lInf = 6.5; + l1 = 0.3; lInf = 10; } +#endif normAssert(out, ref, "", l1, lInf); expectNoFallbacksFromIE(net); @@ -682,6 +684,13 @@ TEST_P(Test_Caffe_nets, FasterRCNN_zf) #endif CV_TEST_TAG_DEBUG_LONG ); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // IE exception: Ngraph operation Reshape with name rpn_cls_score_reshape has dynamic output shape on 0 port, but CPU plug-in supports only static shape + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); +#endif if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); @@ -701,6 +710,11 @@ TEST_P(Test_Caffe_nets, RFCN) CV_TEST_TAG_LONG, CV_TEST_TAG_DEBUG_VERYLONG ); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // Exception: Function contains several inputs and outputs with one friendly name! (HETERO bug?) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target != DNN_TARGET_CPU) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); diff --git a/modules/dnn/test/test_common.hpp b/modules/dnn/test/test_common.hpp index 5fc7447705..02a676da36 100644 --- a/modules/dnn/test/test_common.hpp +++ b/modules/dnn/test/test_common.hpp @@ -155,16 +155,19 @@ public: static void checkBackend(int backend, int target, Mat* inp = 0, Mat* ref = 0) { + CV_UNUSED(backend); CV_UNUSED(target); CV_UNUSED(inp); CV_UNUSED(ref); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021000000) if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_MYRIAD) { if (inp && ref && inp->dims == 4 && ref->dims == 4 && inp->size[0] != 1 && inp->size[0] != ref->size[0]) { + std::cout << "Inconsistent batch size of input and output blobs for Myriad plugin" << std::endl; applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); - throw SkipTestException("Inconsistent batch size of input and output blobs for Myriad plugin"); } } +#endif } void expectNoFallbacks(Net& net, bool raiseError = true) diff --git a/modules/dnn/test/test_darknet_importer.cpp b/modules/dnn/test/test_darknet_importer.cpp index ea700573e6..885bd6eb6b 100644 --- a/modules/dnn/test/test_darknet_importer.cpp +++ b/modules/dnn/test/test_darknet_importer.cpp @@ -245,13 +245,13 @@ public: nms_boxes.push_back(box); nms_confidences.push_back(conf); nms_classIds.push_back(class_id); -#if 0 // use to update test reference data - std::cout << b << ", " << class_id << ", " << conf << "f, " - << box.x << "f, " << box.y << "f, " - << box.x + box.width << "f, " << box.y + box.height << "f," - << std::endl; -#endif - + if (cvtest::debugLevel > 0) + { + std::cout << b << ", " << class_id << ", " << conf << "f, " + << box.x << "f, " << box.y << "f, " + << box.x + box.width << "f, " << box.y + box.height << "f," + << std::endl; + } } if (cvIsNaN(iouDiff)) @@ -347,10 +347,22 @@ TEST_P(Test_Darknet_nets, YoloVoc) 1, 6, 0.667770f, 0.446555f, 0.453578f, 0.499986f, 0.519167f, // a car 1, 6, 0.844947f, 0.637058f, 0.460398f, 0.828508f, 0.66427f); // a car - double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 1e-2 : 8e-5; - double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.018 : 3e-4; double nmsThreshold = (target == DNN_TARGET_MYRIAD) ? 0.397 : 0.4; + double scoreDiff = 8e-5, iouDiff = 3e-4; + if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) + { + scoreDiff = 1e-2; + iouDiff = 0.018; + } +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // accuracy + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + { + iouDiff = std::numeric_limits::quiet_NaN(); + } +#endif + std::string config_file = "yolo-voc.cfg"; std::string weights_file = "yolo-voc.weights"; @@ -363,6 +375,12 @@ TEST_P(Test_Darknet_nets, YoloVoc) SCOPED_TRACE("batch size 2"); testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff, 0.24, nmsThreshold); } + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // accuracy + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif } TEST_P(Test_Darknet_nets, TinyYoloVoc) @@ -584,6 +602,14 @@ TEST_P(Test_Darknet_nets, YOLOv4) std::string config_file = "yolov4.cfg"; std::string weights_file = "yolov4.weights"; + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // accuracy (batch 1) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + { + iouDiff = std::numeric_limits::quiet_NaN(); + } +#endif #if defined(INF_ENGINE_RELEASE) if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_MYRIAD && @@ -602,6 +628,13 @@ TEST_P(Test_Darknet_nets, YOLOv4) { SCOPED_TRACE("batch size 2"); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // accuracy (batch 1) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + { + iouDiff = 0.45f; + } +#endif #if defined(INF_ENGINE_RELEASE) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) { @@ -617,6 +650,12 @@ TEST_P(Test_Darknet_nets, YOLOv4) testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff); } + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // accuracy + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif } TEST_P(Test_Darknet_nets, YOLOv4_tiny) @@ -685,6 +724,13 @@ TEST_P(Test_Darknet_nets, YOLOv4x_mish) { applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB)); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // IE exception: Ngraph operation Transpose with name permute_168 has dynamic output shape on 0 port, but CPU plug-in supports only static shape + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); +#endif #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); diff --git a/modules/dnn/test/test_halide_layers.cpp b/modules/dnn/test/test_halide_layers.cpp index c9c0bc1ced..b24968b8e8 100644 --- a/modules/dnn/test/test_halide_layers.cpp +++ b/modules/dnn/test/test_halide_layers.cpp @@ -39,12 +39,13 @@ static void test(Mat& input, Net& net, Backend backendId, Target targetId, bool l1 = default_l1; if (lInf == 0.0) lInf = default_lInf; -#if 0 - std::cout << "l1=" << l1 << " lInf=" << lInf << std::endl; - std::cout << outputDefault.reshape(1, outputDefault.total()).t() << std::endl; - std::cout << outputHalide.reshape(1, outputDefault.total()).t() << std::endl; -#endif normAssert(outputDefault, outputHalide, "", l1, lInf); + if (cvtest::debugLevel > 0 || testing::Test::HasFailure()) + { + std::cout << "l1=" << l1 << " lInf=" << lInf << std::endl; + std::cout << outputDefault.reshape(1, outputDefault.total()).t() << std::endl; + std::cout << outputHalide.reshape(1, outputDefault.total()).t() << std::endl; + } } static void test(LayerParams& params, Mat& input, Backend backendId, Target targetId, bool skipCheck = false, double l1 = 0.0, double lInf = 0.0) @@ -795,6 +796,16 @@ TEST_P(Eltwise, Accuracy) Backend backendId = get<0>(get<4>(GetParam())); Target targetId = get<1>(get<4>(GetParam())); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // accuracy + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && targetId == DNN_TARGET_OPENCL && + inSize == Vec3i(1, 4, 5) && op == "sum" && numConv == 1 && !weighted) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && targetId == DNN_TARGET_OPENCL && + inSize == Vec3i(2, 8, 6) && op == "sum" && numConv == 1 && !weighted) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2018050000) if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && targetId == DNN_TARGET_MYRIAD && inSize == Vec3i(1, 4, 5)) diff --git a/modules/dnn/test/test_layers.cpp b/modules/dnn/test/test_layers.cpp index fbe9605e7f..836b0aab9a 100644 --- a/modules/dnn/test/test_layers.cpp +++ b/modules/dnn/test/test_layers.cpp @@ -373,7 +373,7 @@ TEST_P(Test_Caffe_layers, layer_prelu_fc) // Reference output values are in range [-0.0001, 10.3906] double l1 = (target == DNN_TARGET_MYRIAD) ? 0.005 : 0.0; double lInf = (target == DNN_TARGET_MYRIAD) ? 0.021 : 0.0; -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2020040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) { l1 = 0.006f; lInf = 0.05f; @@ -1416,6 +1416,14 @@ TEST_P(Test_DLDT_two_inputs, as_backend) double l1 = (targetId == DNN_TARGET_OPENCL_FP16 || targetId == DNN_TARGET_MYRIAD) ? 0.06 : 1e-6; double lInf = (targetId == DNN_TARGET_OPENCL_FP16 || targetId == DNN_TARGET_MYRIAD) ? 0.3 : 1e-5; normAssert(out, ref, "", l1, lInf); + if (cvtest::debugLevel > 0 || HasFailure()) + { + std::cout << "input1 scale=" << kScale << " input2 scale=" << kScaleInv << std::endl; + std::cout << "input1: " << firstInp.size << " " << firstInp.reshape(1, 1) << std::endl; + std::cout << "input2: " << secondInp.size << " " << secondInp.reshape(1, 1) << std::endl; + std::cout << "ref: " << ref.reshape(1, 1) << std::endl; + std::cout << "out: " << out.reshape(1, 1) << std::endl; + } } INSTANTIATE_TEST_CASE_P(/*nothing*/, Test_DLDT_two_inputs, Combine( diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index daba29e547..0fd5d08258 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -686,6 +686,14 @@ TEST_P(Test_ONNX_layers, Split_EltwiseMax) TEST_P(Test_ONNX_layers, LSTM_Activations) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // IE Exception: Ngraph operation Reshape with name Block1237_Output_0_before_reshape has dynamic output shape on 0 port, but CPU plug-in supports only static shape + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); +#endif + testONNXModels("lstm_cntk_tanh", pb, 0, 0, false, false); } @@ -810,6 +818,13 @@ TEST_P(Test_ONNX_layers, Conv1d_variable_weight_bias) TEST_P(Test_ONNX_layers, GatherMultiOutput) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // IE Exception: Ngraph operation Reshape with name 6 has dynamic output shape on 0 port, but CPU plug-in supports only static shape + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); +#endif #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021030000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); // exception @@ -817,7 +832,7 @@ TEST_P(Test_ONNX_layers, GatherMultiOutput) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); // exception #endif -#if defined(INF_ENGINE_RELEASE) +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2021030000) if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE); #endif @@ -827,14 +842,25 @@ TEST_P(Test_ONNX_layers, GatherMultiOutput) TEST_P(Test_ONNX_layers, DynamicAxes) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // accuracy + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); +#endif +#if defined(INF_ENGINE_RELEASE) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) { if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); } +#if INF_ENGINE_VER_MAJOR_LT(2021000000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) { if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); } +#endif +#endif testONNXModels("squeeze_and_conv_dynamic_axes"); testONNXModels("unsqueeze_and_conv_dynamic_axes"); testONNXModels("gather_dynamic_axes"); @@ -914,6 +940,13 @@ TEST_P(Test_ONNX_layers, PoolConv1d) TEST_P(Test_ONNX_layers, ConvResizePool1d) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // IE Exception: Ngraph operation Reshape with name 15 has dynamic output shape on 0 port, but CPU plug-in supports only static shape + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); +#endif #if defined(INF_ENGINE_RELEASE) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) { @@ -1116,8 +1149,12 @@ TEST_P(Test_ONNX_nets, TinyYolov2) #endif // output range: [-11; 8] - double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.017 : default_l1; - double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.14 : default_lInf; + double l1 = default_l1, lInf = default_lInf; + if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) + { + l1 = 0.02; + lInf = 0.2; + } #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) { @@ -1202,10 +1239,10 @@ TEST_P(Test_ONNX_nets, Emotion_ferplus) l1 = 2.4e-4; lInf = 6e-4; } -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2020040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) { - l1 = 0.012f; lInf = 0.035f; + l1 = 0.013f; lInf = 0.035f; } #endif diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index b688c31383..d02b7136d6 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -83,6 +83,10 @@ public: void runTensorFlowNet(const std::string& prefix, bool hasText = false, double l1 = 0.0, double lInf = 0.0, bool memoryLoad = false, const std::string& groupPrefix = "") { + if (cvtest::debugLevel > 0) + { + std::cout << prefix << groupPrefix << std::endl; + } std::string netPath = path(prefix + groupPrefix + "_net.pb"); std::string netConfig = (hasText ? path(prefix + groupPrefix + "_net.pbtxt") : ""); std::string inpPath = path(prefix + "_in.npy"); @@ -118,6 +122,16 @@ public: net.setInput(input); cv::Mat output = net.forward(); normAssert(ref, output, "", l1 ? l1 : default_l1, lInf ? lInf : default_lInf); + + if (cvtest::debugLevel > 0 || HasFailure()) + { + std::cout << "input: " << input.size << std::endl; + std::cout << input.reshape(1, 1) << std::endl; + std::cout << "ref " << ref.size << std::endl; + std::cout << ref.reshape(1, 1) << std::endl; + std::cout << "output: " << output.size << std::endl; + std::cout << output.reshape(1, 1) << std::endl; + } } }; @@ -132,7 +146,7 @@ TEST_P(Test_TensorFlow_layers, reduce_max) { if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); - runTensorFlowNet("max_pool_by_axis"); + runTensorFlowNet("max_pool_by_axis", false, 0.0f, 0.0f); } TEST_P(Test_TensorFlow_layers, reduce_sum) @@ -144,7 +158,11 @@ TEST_P(Test_TensorFlow_layers, reduce_sum) TEST_P(Test_TensorFlow_layers, reduce_max_channel) { - runTensorFlowNet("reduce_max_channel"); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2020040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) // incorrect result + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + runTensorFlowNet("reduce_max_channel", false, 0.0f, 0.0f); } TEST_P(Test_TensorFlow_layers, reduce_sum_channel) @@ -154,6 +172,10 @@ TEST_P(Test_TensorFlow_layers, reduce_sum_channel) TEST_P(Test_TensorFlow_layers, reduce_max_channel_keep_dims) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2020040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) // incorrect result + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif runTensorFlowNet("reduce_max_channel", false, 0.0, 0.0, false, "_keep_dims"); } @@ -220,13 +242,49 @@ TEST_P(Test_TensorFlow_layers, padding) runTensorFlowNet("keras_pad_concat"); } -TEST_P(Test_TensorFlow_layers, padding_asymmetric) +TEST_P(Test_TensorFlow_layers, padding_asymmetric_1) { runTensorFlowNet("conv2d_asymmetric_pads_nchw"); +} + +TEST_P(Test_TensorFlow_layers, padding_asymmetric_2) +{ runTensorFlowNet("conv2d_asymmetric_pads_nhwc"); +} + +TEST_P(Test_TensorFlow_layers, padding_asymmetric_3) +{ +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_CPU) // Exception: Unsupported pad value + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_CPU, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2020020000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) // Exception: Unsupported pad value + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif runTensorFlowNet("max_pool2d_asymmetric_pads_nchw"); +} + +TEST_P(Test_TensorFlow_layers, padding_asymmetric_4) +{ +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_CPU) // Exception: Unsupported pad value + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_CPU, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2020020000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) // Exception: Unsupported pad value + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif runTensorFlowNet("max_pool2d_asymmetric_pads_nhwc"); +} + +TEST_P(Test_TensorFlow_layers, padding_asymmetric_5) +{ runTensorFlowNet("conv2d_backprop_input_asymmetric_pads_nchw"); +} + +TEST_P(Test_TensorFlow_layers, padding_asymmetric_6) +{ runTensorFlowNet("conv2d_backprop_input_asymmetric_pads_nhwc"); } @@ -267,6 +325,13 @@ TEST_P(Test_TensorFlow_layers, pad_and_concat) TEST_P(Test_TensorFlow_layers, concat_axis_1) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // IE Exception: Ngraph operation Transpose with name Flatten_1/flatten/Reshape/nhwc has dynamic output shape on 0 port, but CPU plug-in supports only static shape + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); +#endif #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021030000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); // exception @@ -413,19 +478,77 @@ TEST_P(Test_TensorFlow_layers, pooling_reduce_sum) runTensorFlowNet("reduce_sum"); // a SUM pooling over all spatial dimensions. } -TEST_P(Test_TensorFlow_layers, pooling_reduce_sum2) +TEST_P(Test_TensorFlow_layers, pooling_reduce_sum_0_false) { - int axises[] = {0, 1, 2, 3}; - for (int keepdims = 0; keepdims <= 1; ++keepdims) - { - for (int i = 0; i < sizeof(axises)/sizeof(axises[0]); ++i) - { - runTensorFlowNet(cv::format("reduce_sum_%d_%s", axises[i], (keepdims ? "True" : "False"))); - } - runTensorFlowNet(cv::format("reduce_sum_1_2_%s", keepdims ? "True" : "False")); - } + runTensorFlowNet("reduce_sum_0_False"); } +TEST_P(Test_TensorFlow_layers, pooling_reduce_sum_1_false) +{ + runTensorFlowNet("reduce_sum_1_False"); +} + +TEST_P(Test_TensorFlow_layers, pooling_reduce_sum_2_false) +{ + runTensorFlowNet("reduce_sum_2_False"); +} + +TEST_P(Test_TensorFlow_layers, pooling_reduce_sum_3_false) +{ + runTensorFlowNet("reduce_sum_3_False"); +} + +TEST_P(Test_TensorFlow_layers, pooling_reduce_sum_1_2_false) +{ +#if defined(INF_ENGINE_RELEASE) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) + { + default_l1 = 0.01f; + } + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + { + default_l1 = 0.01f; + } +#endif + runTensorFlowNet("reduce_sum_1_2_False"); +} + +TEST_P(Test_TensorFlow_layers, pooling_reduce_sum_0_true) +{ + runTensorFlowNet("reduce_sum_0_True"); +} + +TEST_P(Test_TensorFlow_layers, pooling_reduce_sum_1_true) +{ + runTensorFlowNet("reduce_sum_1_True"); +} + +TEST_P(Test_TensorFlow_layers, pooling_reduce_sum_2_true) +{ + runTensorFlowNet("reduce_sum_2_True"); +} + +TEST_P(Test_TensorFlow_layers, pooling_reduce_sum_3_true) +{ + runTensorFlowNet("reduce_sum_3_True"); +} + +TEST_P(Test_TensorFlow_layers, pooling_reduce_sum_1_2_true) +{ +#if defined(INF_ENGINE_RELEASE) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) + { + default_l1 = 0.01f; + } + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + { + default_l1 = 0.01f; + } +#endif + runTensorFlowNet("reduce_sum_1_2_True"); +} + + TEST_P(Test_TensorFlow_layers, max_pool_grad) { if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) @@ -642,8 +765,13 @@ TEST_P(Test_TensorFlow_nets, MobileNet_SSD) net.setInput(inp); Mat out = net.forward(); - double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0043 : default_l1; - double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.037 : default_lInf; + double scoreDiff = default_l1, iouDiff = default_lInf; + if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) + { + scoreDiff = 0.01; + iouDiff = 0.1; + } + normAssertDetections(ref, out, "", 0.2, scoreDiff, iouDiff); #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_RELEASE >= 2019010000 expectNoFallbacksFromIE(net); @@ -720,16 +848,13 @@ TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD) expectNoFallbacksFromIE(net); } -TEST_P(Test_TensorFlow_nets, Faster_RCNN) +TEST_P(Test_TensorFlow_nets, Faster_RCNN_inception_v2_coco_2018_01_28) { - // FIXIT split test applyTestTag( (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB), CV_TEST_TAG_LONG, CV_TEST_TAG_DEBUG_VERYLONG ); - static std::string names[] = {"faster_rcnn_inception_v2_coco_2018_01_28", - "faster_rcnn_resnet50_coco_2018_01_28"}; #ifdef INF_ENGINE_RELEASE if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && @@ -740,21 +865,28 @@ TEST_P(Test_TensorFlow_nets, Faster_RCNN) backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); #endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) // segfault: inference-engine/thirdparty/clDNN/src/gpu/detection_output_cpu.cpp:111: // Assertion `prior_height > 0' failed. if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); - - if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); +#endif checkBackend(); - double scoresDiff = backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 ? 2.9e-5 : 1e-5; - for (int i = 0; i < 2; ++i) + double scoresDiff = 1e-5; + double iouDiff = 1e-4; + + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) { - std::string proto = findDataFile("dnn/" + names[i] + ".pbtxt"); - std::string model = findDataFile("dnn/" + names[i] + ".pb", false); + scoresDiff = 0.02; + iouDiff = 0.1; + } + + std::string name = "faster_rcnn_inception_v2_coco_2018_01_28"; + { + std::string proto = findDataFile("dnn/" + name + ".pbtxt"); + std::string model = findDataFile("dnn/" + name + ".pb", false); Net net = readNetFromTensorflow(model, proto); net.setPreferableBackend(backend); @@ -765,8 +897,74 @@ TEST_P(Test_TensorFlow_nets, Faster_RCNN) net.setInput(blob); Mat out = net.forward(); - Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/" + names[i] + ".detection_out.npy")); - normAssertDetections(ref, out, names[i].c_str(), 0.3, scoresDiff); + Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/" + name + ".detection_out.npy")); + + // accuracy (both OpenCV & IE) + if (target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + + normAssertDetections(ref, out, name.c_str(), 0.3, scoresDiff, iouDiff); + } +} + +TEST_P(Test_TensorFlow_nets, Faster_RCNN_resnet50_coco_2018_01_28) +{ + applyTestTag( + (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB), + CV_TEST_TAG_LONG, + CV_TEST_TAG_DEBUG_VERYLONG + ); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // IE exception: Ngraph operation Transpose with name FirstStageBoxPredictor/ClassPredictor/reshape_1/nhwc has dynamic output shape on 0 port, but CPU plug-in supports only static shape + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); +#endif + +#ifdef INF_ENGINE_RELEASE + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && + (INF_ENGINE_VER_MAJOR_LT(2019020000) || target != DNN_TARGET_CPU)) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + + if (INF_ENGINE_VER_MAJOR_GT(2019030000) && + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) + // segfault: inference-engine/thirdparty/clDNN/src/gpu/detection_output_cpu.cpp:111: + // Assertion `prior_height > 0' failed. + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif + + checkBackend(); + + double scoresDiff = backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 ? 2.9e-5 : 1e-5; + double iouDiff = 1e-4; + + std::string name = "faster_rcnn_resnet50_coco_2018_01_28"; + { + std::string proto = findDataFile("dnn/" + name + ".pbtxt"); + std::string model = findDataFile("dnn/" + name + ".pb", false); + + Net net = readNetFromTensorflow(model, proto); + net.setPreferableBackend(backend); + net.setPreferableTarget(target); + Mat img = imread(findDataFile("dnn/dog416.png")); + Mat blob = blobFromImage(img, 1.0f, Size(800, 600), Scalar(), true, false); + + net.setInput(blob); + Mat out = net.forward(); + + Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/" + name + ".detection_out.npy")); + + // accuracy + if (target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + + normAssertDetections(ref, out, name.c_str(), 0.3, scoresDiff, iouDiff); } } @@ -1152,6 +1350,10 @@ TEST_P(Test_TensorFlow_layers, resize_bilinear_down) TEST_P(Test_TensorFlow_layers, resize_concat_optimization) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target != DNN_TARGET_CPU) // Exception: Function contains several inputs and outputs with one friendly name! (HETERO bug?) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif runTensorFlowNet("resize_concat_optimization"); } @@ -1271,7 +1473,7 @@ TEST_P(Test_TensorFlow_nets, Mask_RCNN) Mat outDetections = outs[0]; Mat outMasks = outs[1]; - double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.019 : 2e-5; + double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.2 : 2e-5; double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.018 : default_lInf; normAssertDetections(refDetections, outDetections, "", /*threshold for zero confidence*/1e-5, scoreDiff, iouDiff); @@ -1305,7 +1507,7 @@ TEST_P(Test_TensorFlow_nets, Mask_RCNN) double inter = cv::countNonZero(masks & refMasks); double area = cv::countNonZero(masks | refMasks); - EXPECT_GE(inter / area, 0.99); + EXPECT_GE(inter / area, (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.98 : 0.99); if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) expectNoFallbacks(net); diff --git a/modules/dnn/test/test_torch_importer.cpp b/modules/dnn/test/test_torch_importer.cpp index 7316a06856..96e8ac85a8 100644 --- a/modules/dnn/test/test_torch_importer.cpp +++ b/modules/dnn/test/test_torch_importer.cpp @@ -164,8 +164,12 @@ TEST_P(Test_Torch_layers, run_concat) TEST_P(Test_Torch_layers, run_depth_concat) { - runTorchNet("net_depth_concat", "", false, true, true, 0.0, - target == DNN_TARGET_OPENCL_FP16 ? 0.032 : 0.0); + double lInf = 0.0; + if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) + { + lInf = 0.032; + } + runTorchNet("net_depth_concat", "", false, true, true, 0.0, lInf); } TEST_P(Test_Torch_layers, run_deconv) From c15218e37a706ee07d4cb920aaeef834c55e87e8 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 26 Nov 2021 19:34:52 +0000 Subject: [PATCH 100/226] gapi: fix build with MSVS2015 (with IE) --- modules/gapi/test/infer/gapi_infer_ie_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/gapi/test/infer/gapi_infer_ie_test.cpp b/modules/gapi/test/infer/gapi_infer_ie_test.cpp index d15b41979e..69ed80054c 100644 --- a/modules/gapi/test/infer/gapi_infer_ie_test.cpp +++ b/modules/gapi/test/infer/gapi_infer_ie_test.cpp @@ -2865,11 +2865,11 @@ TEST(Infer, ModelWith2DInputs) // NB: Define model with 2D inputs. auto in1 = std::make_shared( ngraph::element::Type_t::u8, - ngraph::Shape(std::vector{{H, W}}) + ngraph::Shape(std::vector{(size_t)H, (size_t)W}) ); auto in2 = std::make_shared( ngraph::element::Type_t::u8, - ngraph::Shape(std::vector{{H, W}}) + ngraph::Shape(std::vector{(size_t)H, (size_t)W}) ); auto result = std::make_shared(in1, in2); auto func = std::make_shared( From 31b2d6be75c8f271f83b9118f9d950f73b22b49d Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 25 Nov 2021 19:56:27 +0000 Subject: [PATCH 101/226] dnn(test): update InferenceEngine tests (4.x) --- modules/dnn/test/test_model.cpp | 46 +++++++++++++++++++++++---- modules/dnn/test/test_tf_importer.cpp | 5 ++- 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/modules/dnn/test/test_model.cpp b/modules/dnn/test/test_model.cpp index 4c08cc30b9..4f5922182a 100644 --- a/modules/dnn/test/test_model.cpp +++ b/modules/dnn/test/test_model.cpp @@ -287,6 +287,12 @@ TEST_P(Test_Model, DetectRegion) CV_TEST_TAG_MEMORY_2GB ); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // accuracy + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); @@ -295,6 +301,7 @@ TEST_P(Test_Model, DetectRegion) #endif #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000) + // FIXIT DNN_BACKEND_INFERENCE_ENGINE is misused if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); #endif @@ -340,6 +347,12 @@ TEST_P(Test_Model, DetectRegionWithNmsAcrossClasses) CV_TEST_TAG_MEMORY_2GB ); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // accuracy + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); @@ -390,7 +403,14 @@ TEST_P(Test_Model, DetectRegionWithNmsAcrossClasses) TEST_P(Test_Model, DetectionOutput) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // Exception: Function contains several inputs and outputs with one friendly name! (HETERO bug?) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target != DNN_TARGET_CPU) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + #if defined(INF_ENGINE_RELEASE) + // FIXIT DNN_BACKEND_INFERENCE_ENGINE is misused if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); @@ -468,9 +488,9 @@ TEST_P(Test_Model, DetectionMobilenetSSD) } else if (target == DNN_TARGET_MYRIAD) { - scoreDiff = 1.7e-2; + scoreDiff = 0.017; if (getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) - iouDiff = 6.91e-2; + iouDiff = 0.1; } else if (target == DNN_TARGET_CUDA_FP16) { @@ -579,7 +599,8 @@ TEST_P(Test_Model, Detection_normalized) #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2020040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) { - iouDiff = 0.095f; + scoreDiff = 0.02; + iouDiff = 0.1f; } #endif testDetectModel(weights_file, config_file, img_path, refClassIds, refConfidences, refBoxes, @@ -608,8 +629,13 @@ TEST_P(Test_Model, Segmentation) TEST_P(Test_Model, TextRecognition) { - if (target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // IE Exception: Ngraph operation Reshape with name 71 has dynamic output shape on 0 port, but CPU plug-in supports only static shape + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); +#endif std::string imgPath = _tf("text_rec_test.png"); std::string weightPath = _tf("onnx/models/crnn.onnx", false); @@ -627,8 +653,14 @@ TEST_P(Test_Model, TextRecognition) TEST_P(Test_Model, TextRecognitionWithCTCPrefixBeamSearch) { - if (target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // IE Exception: Ngraph operation Reshape with name 71 has dynamic output shape on 0 port, but CPU plug-in supports only static shape + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); +#endif + std::string imgPath = _tf("text_rec_test.png"); std::string weightPath = _tf("onnx/models/crnn.onnx", false); diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index ed8e8648e0..cbd7401232 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -731,6 +731,10 @@ TEST_P(Test_TensorFlow_layers, BiasAdd) TEST_P(Test_TensorFlow_layers, ExpandDims) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_CPU) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_CPU, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); // Layout::ANY is broken on CPU +#endif #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X @@ -1042,7 +1046,6 @@ TEST_P(Test_TensorFlow_nets, Faster_RCNN_resnet50_coco_2018_01_28) double iouDiff = 1e-4; if (target == DNN_TARGET_CUDA) { - // for faster_rcnn_resnet50_coco_2018_01_28 scoresDiff = 0.06; iouDiff = 0.08; } From a6277370cab46e4394b35cc93cf9aada69cb2a70 Mon Sep 17 00:00:00 2001 From: yuki takehara Date: Sun, 28 Nov 2021 03:34:52 +0900 Subject: [PATCH 102/226] Merge pull request #21107 from take1014:remove_assert_21038 resolves #21038 * remove C assert * revert C header * fix several points in review * fix test_ds.cpp --- modules/calib3d/src/calibration.cpp | 8 +-- modules/calib3d/src/compat_ptsetreg.cpp | 6 +- .../calib3d/test/test_cameracalibration.cpp | 22 +++---- modules/calib3d/test/test_fundam.cpp | 8 +-- modules/calib3d/test/test_stereomatching.cpp | 34 +++++------ modules/core/include/opencv2/core/wimage.hpp | 8 +-- modules/core/src/array.cpp | 20 +++---- modules/core/src/datastructs.cpp | 60 +++++++++---------- modules/core/src/dxt.cpp | 26 ++++---- modules/core/src/lapack.cpp | 4 +- modules/core/src/matmul.simd.hpp | 4 +- modules/core/src/matrix_sparse.cpp | 2 +- .../src/opencl/runtime/opencl_clamdblas.cpp | 2 +- .../src/opencl/runtime/opencl_clamdfft.cpp | 2 +- .../core/src/opencl/runtime/opencl_core.cpp | 2 +- modules/core/src/persistence.cpp | 4 +- modules/core/src/persistence_json.cpp | 2 +- modules/core/src/persistence_types.cpp | 18 +++--- modules/core/src/persistence_xml.cpp | 6 +- modules/core/src/persistence_yml.cpp | 2 +- modules/core/test/test_ds.cpp | 46 +++++++------- modules/core/test/test_dxt.cpp | 2 +- modules/core/test/test_mat.cpp | 6 +- modules/core/test/test_math.cpp | 14 ++--- .../src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp | 1 - .../test/test_descriptors_regression.cpp | 4 +- .../test/test_detectors_regression.cpp | 4 +- .../features2d/test/test_nearestneighbors.cpp | 2 +- .../flann/include/opencv2/flann/nn_index.h | 4 +- modules/highgui/src/precomp.hpp | 1 - modules/highgui/src/window_carbon.cpp | 2 +- modules/highgui/src/window_gtk.cpp | 2 +- modules/highgui/src/window_w32.cpp | 7 +-- modules/highgui/src/window_winrt.cpp | 1 - modules/imgproc/src/approx.cpp | 20 +++---- modules/imgproc/src/bilateral_filter.simd.hpp | 2 +- modules/imgproc/src/contours.cpp | 4 +- modules/imgproc/src/convhull.cpp | 2 +- modules/imgproc/src/drawing.cpp | 12 ++-- modules/imgproc/src/emd.cpp | 6 +- modules/imgproc/src/hough.cpp | 8 +-- modules/imgproc/src/median_blur.simd.hpp | 2 +- modules/imgproc/src/moments.cpp | 2 +- modules/imgproc/src/precomp.hpp | 1 - modules/imgproc/src/resize.cpp | 8 +-- modules/imgproc/src/samplers.cpp | 2 +- modules/imgproc/src/segmentation.cpp | 6 +- modules/imgproc/src/subdivision2d.cpp | 2 +- modules/imgproc/test/test_approxpoly.cpp | 4 +- .../imgproc/test/test_bilateral_filter.cpp | 2 +- modules/imgproc/test/test_canny.cpp | 2 +- modules/imgproc/test/test_color.cpp | 12 ++-- modules/imgproc/test/test_contours.cpp | 2 +- modules/imgproc/test/test_convhull.cpp | 16 ++--- .../imgproc/test/test_distancetransform.cpp | 2 +- modules/imgproc/test/test_filter.cpp | 8 +-- modules/imgproc/test/test_imgwarp.cpp | 4 +- modules/imgproc/test/test_imgwarp_strict.cpp | 2 +- modules/imgproc/test/test_templmatch.cpp | 2 +- modules/imgproc/test/test_thresh.cpp | 4 +- modules/ml/src/precomp.hpp | 1 - modules/ml/src/tree.cpp | 2 +- modules/objdetect/src/cascadedetect.cpp | 8 +-- .../objdetect/src/detection_based_tracker.cpp | 1 - modules/objdetect/src/haar.cpp | 4 +- modules/objdetect/src/hog.cpp | 4 +- modules/objdetect/test/test_cascadeandhog.cpp | 12 ++-- modules/ts/src/ocl_perf.cpp | 2 +- modules/ts/src/ts.cpp | 2 +- modules/ts/src/ts_arrtest.cpp | 6 +- modules/ts/src/ts_func.cpp | 8 +-- modules/ts/src/ts_perf.cpp | 8 +-- modules/videoio/src/cap_cmu.cpp | 4 +- modules/videoio/src/cap_dc1394.cpp | 2 +- modules/videoio/src/cap_dc1394_v2.cpp | 1 - modules/videoio/src/cap_ffmpeg_impl.hpp | 9 ++- modules/videoio/src/cap_giganetix.cpp | 2 +- modules/videoio/src/cap_libv4l.cpp | 3 +- modules/videoio/src/cap_qt.cpp | 5 +- modules/videoio/src/cap_v4l.cpp | 5 +- modules/videoio/src/precomp.hpp | 1 - 81 files changed, 277 insertions(+), 286 deletions(-) diff --git a/modules/calib3d/src/calibration.cpp b/modules/calib3d/src/calibration.cpp index 650735f035..ee1a751530 100644 --- a/modules/calib3d/src/calibration.cpp +++ b/modules/calib3d/src/calibration.cpp @@ -2125,7 +2125,7 @@ static double cvStereoCalibrateImpl( const CvMat* _objectPoints, const CvMat* _i if( solver.state == CvLevMarq::CALC_J ) { int iofs = (nimages+1)*6 + k*NINTRINSIC, eofs = (i+1)*6; - assert( JtJ && JtErr ); + CV_Assert( JtJ && JtErr ); Mat _JtJ(cvarrToMat(JtJ)), _JtErr(cvarrToMat(JtErr)); @@ -2929,7 +2929,7 @@ cvRQDecomp3x3( const CvMat *matrixM, CvMat *matrixR, CvMat *matrixQ, CvMat Qx = cvMat(3, 3, CV_64F, _Qx); cvMatMul(&M, &Qx, &R); - assert(fabs(matR[2][1]) < FLT_EPSILON); + CV_DbgAssert(fabs(matR[2][1]) < FLT_EPSILON); matR[2][1] = 0; /* Find Givens rotation for y axis. */ @@ -2948,7 +2948,7 @@ cvRQDecomp3x3( const CvMat *matrixM, CvMat *matrixR, CvMat *matrixQ, CvMat Qy = cvMat(3, 3, CV_64F, _Qy); cvMatMul(&R, &Qy, &M); - assert(fabs(matM[2][0]) < FLT_EPSILON); + CV_DbgAssert(fabs(matM[2][0]) < FLT_EPSILON); matM[2][0] = 0; /* Find Givens rotation for z axis. */ @@ -2968,7 +2968,7 @@ cvRQDecomp3x3( const CvMat *matrixM, CvMat *matrixR, CvMat *matrixQ, CvMat Qz = cvMat(3, 3, CV_64F, _Qz); cvMatMul(&M, &Qz, &R); - assert(fabs(matR[1][0]) < FLT_EPSILON); + CV_DbgAssert(fabs(matR[1][0]) < FLT_EPSILON); matR[1][0] = 0; // Solve the decomposition ambiguity. diff --git a/modules/calib3d/src/compat_ptsetreg.cpp b/modules/calib3d/src/compat_ptsetreg.cpp index 6e67000b3b..6199f26bb2 100644 --- a/modules/calib3d/src/compat_ptsetreg.cpp +++ b/modules/calib3d/src/compat_ptsetreg.cpp @@ -121,7 +121,7 @@ bool CvLevMarq::update( const CvMat*& _param, CvMat*& matJ, CvMat*& _err ) { matJ = _err = 0; - assert( !err.empty() ); + CV_Assert( !err.empty() ); if( state == DONE ) { _param = param; @@ -154,7 +154,7 @@ bool CvLevMarq::update( const CvMat*& _param, CvMat*& matJ, CvMat*& _err ) return true; } - assert( state == CHECK_ERR ); + CV_Assert( state == CHECK_ERR ); errNorm = cvNorm( err, 0, CV_L2 ); if( errNorm > prevErrNorm ) { @@ -222,7 +222,7 @@ bool CvLevMarq::updateAlt( const CvMat*& _param, CvMat*& _JtJ, CvMat*& _JtErr, d return true; } - assert( state == CHECK_ERR ); + CV_Assert( state == CHECK_ERR ); if( errNorm > prevErrNorm ) { if( ++lambdaLg10 <= 16 ) diff --git a/modules/calib3d/test/test_cameracalibration.cpp b/modules/calib3d/test/test_cameracalibration.cpp index 366eee341a..a63c131a0f 100644 --- a/modules/calib3d/test/test_cameracalibration.cpp +++ b/modules/calib3d/test/test_cameracalibration.cpp @@ -831,30 +831,30 @@ void CV_CameraCalibrationTest_CPP::calibrate(int imageCount, int* pointCounts, perViewErrorsMat, flags ); - assert( stdDevsMatInt.type() == CV_64F ); - assert( stdDevsMatInt.total() == static_cast(CV_CALIB_NINTRINSIC) ); + CV_Assert( stdDevsMatInt.type() == CV_64F ); + CV_Assert( stdDevsMatInt.total() == static_cast(CV_CALIB_NINTRINSIC) ); memcpy( stdDevs, stdDevsMatInt.ptr(), CV_CALIB_NINTRINSIC*sizeof(double) ); - assert( stdDevsMatExt.type() == CV_64F ); - assert( stdDevsMatExt.total() == static_cast(6*imageCount) ); + CV_Assert( stdDevsMatExt.type() == CV_64F ); + CV_Assert( stdDevsMatExt.total() == static_cast(6*imageCount) ); memcpy( stdDevs + CV_CALIB_NINTRINSIC, stdDevsMatExt.ptr(), 6*imageCount*sizeof(double) ); - assert( perViewErrorsMat.type() == CV_64F); - assert( perViewErrorsMat.total() == static_cast(imageCount) ); + CV_Assert( perViewErrorsMat.type() == CV_64F); + CV_Assert( perViewErrorsMat.total() == static_cast(imageCount) ); memcpy( perViewErrors, perViewErrorsMat.ptr(), imageCount*sizeof(double) ); - assert( cameraMatrix.type() == CV_64FC1 ); + CV_Assert( cameraMatrix.type() == CV_64FC1 ); memcpy( _cameraMatrix, cameraMatrix.ptr(), 9*sizeof(double) ); - assert( cameraMatrix.type() == CV_64FC1 ); + CV_Assert( cameraMatrix.type() == CV_64FC1 ); memcpy( _distortionCoeffs, distCoeffs.ptr(), 4*sizeof(double) ); vector::iterator rvecsIt = rvecs.begin(); vector::iterator tvecsIt = tvecs.begin(); double *rm = rotationMatrices, *tm = translationVectors; - assert( rvecsIt->type() == CV_64FC1 ); - assert( tvecsIt->type() == CV_64FC1 ); + CV_Assert( rvecsIt->type() == CV_64FC1 ); + CV_Assert( tvecsIt->type() == CV_64FC1 ); for( int i = 0; i < imageCount; ++rvecsIt, ++tvecsIt, i++, rm+=9, tm+=3 ) { Mat r9( 3, 3, CV_64FC1 ); @@ -1141,7 +1141,7 @@ void CV_ProjectPointsTest::run(int) imgPoints, dpdrot, dpdt, dpdf, dpdc, dpddist, 0 ); // calculate and check image points - assert( (int)imgPoints.size() == pointCount ); + CV_Assert( (int)imgPoints.size() == pointCount ); vector::const_iterator it = imgPoints.begin(); for( int i = 0; i < pointCount; i++, ++it ) { diff --git a/modules/calib3d/test/test_fundam.cpp b/modules/calib3d/test/test_fundam.cpp index 236db6ec4d..853695ef15 100644 --- a/modules/calib3d/test/test_fundam.cpp +++ b/modules/calib3d/test/test_fundam.cpp @@ -56,7 +56,7 @@ static int cvTsRodrigues( const CvMat* src, CvMat* dst, CvMat* jacobian ) if( jacobian ) { - assert( (jacobian->rows == 9 && jacobian->cols == 3) || + CV_Assert( (jacobian->rows == 9 && jacobian->cols == 3) || (jacobian->rows == 3 && jacobian->cols == 9) ); } @@ -65,7 +65,7 @@ static int cvTsRodrigues( const CvMat* src, CvMat* dst, CvMat* jacobian ) double r[3], theta; CvMat _r = cvMat( src->rows, src->cols, CV_MAKETYPE(CV_64F,CV_MAT_CN(src->type)), r); - assert( dst->rows == 3 && dst->cols == 3 ); + CV_Assert( dst->rows == 3 && dst->cols == 3 ); cvConvert( src, &_r ); @@ -320,7 +320,7 @@ static int cvTsRodrigues( const CvMat* src, CvMat* dst, CvMat* jacobian ) } else { - assert(0); + CV_Assert(0); return 0; } @@ -404,7 +404,7 @@ static void test_convertHomogeneous( const Mat& _src, Mat& _dst ) } else { - assert( count == dst.cols ); + CV_Assert( count == dst.cols ); ddims = dst.channels()*dst.rows; if( dst.rows == 1 ) { diff --git a/modules/calib3d/test/test_stereomatching.cpp b/modules/calib3d/test/test_stereomatching.cpp index e92c170c00..4ea23ebff3 100644 --- a/modules/calib3d/test/test_stereomatching.cpp +++ b/modules/calib3d/test/test_stereomatching.cpp @@ -406,7 +406,7 @@ void CV_StereoMatchingTest::run(int) { string dataPath = ts->get_data_path() + "cv/"; string algorithmName = name; - assert( !algorithmName.empty() ); + CV_Assert( !algorithmName.empty() ); if( dataPath.empty() ) { ts->printf( cvtest::TS::LOG, "dataPath is empty" ); @@ -553,22 +553,22 @@ int CV_StereoMatchingTest::processStereoMatchingResults( FileStorage& fs, int ca { // rightDisp is not used in current test virsion int code = cvtest::TS::OK; - assert( fs.isOpened() ); - assert( trueLeftDisp.type() == CV_32FC1 ); - assert( trueRightDisp.empty() || trueRightDisp.type() == CV_32FC1 ); - assert( leftDisp.type() == CV_32FC1 && (rightDisp.empty() || rightDisp.type() == CV_32FC1) ); + CV_Assert( fs.isOpened() ); + CV_Assert( trueLeftDisp.type() == CV_32FC1 ); + CV_Assert( trueRightDisp.empty() || trueRightDisp.type() == CV_32FC1 ); + CV_Assert( leftDisp.type() == CV_32FC1 && (rightDisp.empty() || rightDisp.type() == CV_32FC1) ); // get masks for unknown ground truth disparity values Mat leftUnknMask, rightUnknMask; DatasetParams params = datasetsParams[caseDatasets[caseIdx]]; absdiff( trueLeftDisp, Scalar(params.dispUnknVal), leftUnknMask ); leftUnknMask = leftUnknMask < std::numeric_limits::epsilon(); - assert(leftUnknMask.type() == CV_8UC1); + CV_Assert(leftUnknMask.type() == CV_8UC1); if( !trueRightDisp.empty() ) { absdiff( trueRightDisp, Scalar(params.dispUnknVal), rightUnknMask ); rightUnknMask = rightUnknMask < std::numeric_limits::epsilon(); - assert(rightUnknMask.type() == CV_8UC1); + CV_Assert(rightUnknMask.type() == CV_8UC1); } // calculate errors @@ -623,7 +623,7 @@ int CV_StereoMatchingTest::readDatasetsParams( FileStorage& fs ) } datasetsParams.clear(); FileNode fn = fs.getFirstTopLevelNode(); - assert(fn.isSeq()); + CV_Assert(fn.isSeq()); for( int i = 0; i < (int)fn.size(); i+=3 ) { String _name = fn[i]; @@ -649,7 +649,7 @@ int CV_StereoMatchingTest::readRunParams( FileStorage& fs ) void CV_StereoMatchingTest::writeErrors( const string& errName, const vector& errors, FileStorage* fs ) { - assert( (int)errors.size() == ERROR_KINDS_COUNT ); + CV_Assert( (int)errors.size() == ERROR_KINDS_COUNT ); vector::const_iterator it = errors.begin(); if( fs ) for( int i = 0; i < ERROR_KINDS_COUNT; i++, ++it ) @@ -696,9 +696,9 @@ void CV_StereoMatchingTest::readROI( FileNode& fn, Rect& validROI ) int CV_StereoMatchingTest::compareErrors( const vector& calcErrors, const vector& validErrors, const vector& eps, const string& errName ) { - assert( (int)calcErrors.size() == ERROR_KINDS_COUNT ); - assert( (int)validErrors.size() == ERROR_KINDS_COUNT ); - assert( (int)eps.size() == ERROR_KINDS_COUNT ); + CV_Assert( (int)calcErrors.size() == ERROR_KINDS_COUNT ); + CV_Assert( (int)validErrors.size() == ERROR_KINDS_COUNT ); + CV_Assert( (int)eps.size() == ERROR_KINDS_COUNT ); vector::const_iterator calcIt = calcErrors.begin(), validIt = validErrors.begin(), epsIt = eps.begin(); @@ -757,7 +757,7 @@ protected: { int code = CV_StereoMatchingTest::readRunParams( fs ); FileNode fn = fs.getFirstTopLevelNode(); - assert(fn.isSeq()); + CV_Assert(fn.isSeq()); for( int i = 0; i < (int)fn.size(); i+=5 ) { String caseName = fn[i], datasetName = fn[i+1]; @@ -776,8 +776,8 @@ protected: Rect& calcROI, Mat& leftDisp, Mat& /*rightDisp*/, int caseIdx ) { RunParams params = caseRunParams[caseIdx]; - assert( params.ndisp%16 == 0 ); - assert( _leftImg.type() == CV_8UC3 && _rightImg.type() == CV_8UC3 ); + CV_Assert( params.ndisp%16 == 0 ); + CV_Assert( _leftImg.type() == CV_8UC3 && _rightImg.type() == CV_8UC3 ); Mat leftImg; cvtColor( _leftImg, leftImg, COLOR_BGR2GRAY ); Mat rightImg; cvtColor( _rightImg, rightImg, COLOR_BGR2GRAY ); @@ -883,7 +883,7 @@ protected: { int code = CV_StereoMatchingTest::readRunParams(fs); FileNode fn = fs.getFirstTopLevelNode(); - assert(fn.isSeq()); + CV_Assert(fn.isSeq()); for( int i = 0; i < (int)fn.size(); i+=5 ) { String caseName = fn[i], datasetName = fn[i+1]; @@ -902,7 +902,7 @@ protected: Rect& calcROI, Mat& leftDisp, Mat& /*rightDisp*/, int caseIdx ) { RunParams params = caseRunParams[caseIdx]; - assert( params.ndisp%16 == 0 ); + CV_Assert( params.ndisp%16 == 0 ); Ptr sgbm = StereoSGBM::create( 0, params.ndisp, params.winSize, 10*params.winSize*params.winSize, 40*params.winSize*params.winSize, diff --git a/modules/core/include/opencv2/core/wimage.hpp b/modules/core/include/opencv2/core/wimage.hpp index c7b6efa06a..da3a7ba0b3 100644 --- a/modules/core/include/opencv2/core/wimage.hpp +++ b/modules/core/include/opencv2/core/wimage.hpp @@ -236,11 +236,11 @@ protected: void operator=(const WImage&); explicit WImage(IplImage* img) : image_(img) { - assert(!img || img->depth == Depth()); + CV_Assert(!img || img->depth == Depth()); } void SetIpl(IplImage* image) { - assert(!image || image->depth == Depth()); + CV_Assert(!image || image->depth == Depth()); image_ = image; } @@ -260,7 +260,7 @@ public: enum { kChannels = C }; explicit WImageC(IplImage* img) : WImage(img) { - assert(!img || img->nChannels == Channels()); + CV_Assert(!img || img->nChannels == Channels()); } // Construct a view into a region of this image @@ -283,7 +283,7 @@ protected: void operator=(const WImageC&); void SetIpl(IplImage* image) { - assert(!image || image->depth == WImage::Depth()); + CV_Assert(!image || image->depth == WImage::Depth()); WImage::SetIpl(image); } }; diff --git a/modules/core/src/array.cpp b/modules/core/src/array.cpp index 1a5ea0100f..2d87a8d9f2 100644 --- a/modules/core/src/array.cpp +++ b/modules/core/src/array.cpp @@ -497,7 +497,7 @@ cvInitNArrayIterator( int count, CvArr** arrs, // returns zero value if iteration is finished, non-zero otherwise CV_IMPL int cvNextNArraySlice( CvNArrayIterator* iterator ) { - assert( iterator != 0 ); + CV_Assert( iterator != 0 ); int i, dims; for( dims = iterator->dims; dims > 0; dims-- ) @@ -648,7 +648,7 @@ icvGetNodePtr( CvSparseMat* mat, const int* idx, int* _type, int i, tabidx; unsigned hashval = 0; CvSparseNode *node; - assert( CV_IS_SPARSE_MAT( mat )); + CV_Assert( CV_IS_SPARSE_MAT( mat )); if( !precalc_hashval ) { @@ -697,7 +697,7 @@ icvGetNodePtr( CvSparseMat* mat, const int* idx, int* _type, int newrawsize = newsize*sizeof(newtable[0]); CvSparseMatIterator iterator; - assert( (newsize & (newsize - 1)) == 0 ); + CV_Assert( (newsize & (newsize - 1)) == 0 ); // resize hash table newtable = (void**)cvAlloc( newrawsize ); @@ -742,7 +742,7 @@ icvDeleteNode( CvSparseMat* mat, const int* idx, unsigned* precalc_hashval ) int i, tabidx; unsigned hashval = 0; CvSparseNode *node, *prev = 0; - assert( CV_IS_SPARSE_MAT( mat )); + CV_Assert( CV_IS_SPARSE_MAT( mat )); if( !precalc_hashval ) { @@ -1462,7 +1462,7 @@ cvScalarToRawData( const CvScalar* scalar, void* data, int type, int extend_to_1 int cn = CV_MAT_CN( type ); int depth = type & CV_MAT_DEPTH_MASK; - assert( scalar && data ); + CV_Assert( scalar && data ); if( (unsigned)(cn - 1) >= 4 ) CV_Error( CV_StsOutOfRange, "The number of channels must be 1, 2, 3 or 4" ); @@ -1509,7 +1509,7 @@ cvScalarToRawData( const CvScalar* scalar, void* data, int type, int extend_to_1 ((double*)data)[cn] = (double)(scalar->val[cn]); break; default: - assert(0); + CV_Assert(0); CV_Error( CV_BadDepth, "" ); } @@ -1534,7 +1534,7 @@ cvRawDataToScalar( const void* data, int flags, CvScalar* scalar ) { int cn = CV_MAT_CN( flags ); - assert( scalar && data ); + CV_Assert( scalar && data ); if( (unsigned)(cn - 1) >= 4 ) CV_Error( CV_StsOutOfRange, "The number of channels must be 1, 2, 3 or 4" ); @@ -1572,7 +1572,7 @@ cvRawDataToScalar( const void* data, int flags, CvScalar* scalar ) scalar->val[cn] = ((double*)data)[cn]; break; default: - assert(0); + CV_Assert(0); CV_Error( CV_BadDepth, "" ); } } @@ -2623,7 +2623,7 @@ cvReshapeMatND( const CvArr* arr, { CvMatND* mat = (CvMatND*)arr; - assert( new_cn > 0 ); + CV_Assert( new_cn > 0 ); int last_dim_size = mat->dim[mat->dims-1].size*CV_MAT_CN(mat->type); int new_size = last_dim_size/new_cn; @@ -2901,7 +2901,7 @@ CV_IMPL IplImage * cvCreateImage( CvSize size, int depth, int channels ) { IplImage *img = cvCreateImageHeader( size, depth, channels ); - assert( img ); + CV_Assert( img ); cvCreateData( img ); return img; diff --git a/modules/core/src/datastructs.cpp b/modules/core/src/datastructs.cpp index cd9196a130..898fce4cc2 100644 --- a/modules/core/src/datastructs.cpp +++ b/modules/core/src/datastructs.cpp @@ -97,7 +97,7 @@ icvInitMemStorage( CvMemStorage* storage, int block_size ) block_size = CV_STORAGE_BLOCK_SIZE; block_size = cvAlign( block_size, CV_STRUCT_ALIGN ); - assert( sizeof(CvMemBlock) % CV_STRUCT_ALIGN == 0 ); + CV_Assert( sizeof(CvMemBlock) % CV_STRUCT_ALIGN == 0 ); memset( storage, 0, sizeof( *storage )); storage->signature = CV_STORAGE_MAGIC_VAL; @@ -240,7 +240,7 @@ icvGoNextMemBlock( CvMemStorage * storage ) if( block == parent->top ) /* the single allocated block */ { - assert( parent->bottom == block ); + CV_Assert( parent->bottom == block ); parent->top = parent->bottom = 0; parent->free_space = 0; } @@ -266,7 +266,7 @@ icvGoNextMemBlock( CvMemStorage * storage ) if( storage->top->next ) storage->top = storage->top->next; storage->free_space = storage->block_size - sizeof(CvMemBlock); - assert( storage->free_space % CV_STRUCT_ALIGN == 0 ); + CV_Assert( storage->free_space % CV_STRUCT_ALIGN == 0 ); } @@ -331,7 +331,7 @@ cvMemStorageAlloc( CvMemStorage* storage, size_t size ) if( size > INT_MAX ) CV_Error( CV_StsOutOfRange, "Too large memory block is requested" ); - assert( storage->free_space % CV_STRUCT_ALIGN == 0 ); + CV_Assert( storage->free_space % CV_STRUCT_ALIGN == 0 ); if( (size_t)storage->free_space < size ) { @@ -343,7 +343,7 @@ cvMemStorageAlloc( CvMemStorage* storage, size_t size ) } ptr = ICV_FREE_PTR(storage); - assert( (size_t)ptr % CV_STRUCT_ALIGN == 0 ); + CV_Assert( (size_t)ptr % CV_STRUCT_ALIGN == 0 ); storage->free_space = cvAlignLeft(storage->free_space - (int)size, CV_STRUCT_ALIGN ); return ptr; @@ -683,7 +683,7 @@ icvGrowSeq( CvSeq *seq, int in_front_of ) else { icvGoNextMemBlock( storage ); - assert( storage->free_space >= delta ); + CV_Assert( storage->free_space >= delta ); } } @@ -716,7 +716,7 @@ icvGrowSeq( CvSeq *seq, int in_front_of ) * For used blocks it means current number * of sequence elements in the block: */ - assert( block->count % seq->elem_size == 0 && block->count > 0 ); + CV_Assert( block->count % seq->elem_size == 0 && block->count > 0 ); if( !in_front_of ) { @@ -732,7 +732,7 @@ icvGrowSeq( CvSeq *seq, int in_front_of ) if( block != block->prev ) { - assert( seq->first->start_index == 0 ); + CV_Assert( seq->first->start_index == 0 ); seq->first = block; } else @@ -760,7 +760,7 @@ icvFreeSeqBlock( CvSeq *seq, int in_front_of ) { CvSeqBlock *block = seq->first; - assert( (in_front_of ? block : block->prev)->count == 0 ); + CV_Assert( (in_front_of ? block : block->prev)->count == 0 ); if( block == block->prev ) /* single block case */ { @@ -775,7 +775,7 @@ icvFreeSeqBlock( CvSeq *seq, int in_front_of ) if( !in_front_of ) { block = block->prev; - assert( seq->ptr == block->data ); + CV_Assert( seq->ptr == block->data ); block->count = (int)(seq->block_max - seq->ptr); seq->block_max = seq->ptr = block->prev->data + @@ -804,7 +804,7 @@ icvFreeSeqBlock( CvSeq *seq, int in_front_of ) block->next->prev = block->prev; } - assert( block->count > 0 && block->count % seq->elem_size == 0 ); + CV_Assert( block->count > 0 && block->count % seq->elem_size == 0 ); block->next = seq->free_blocks; seq->free_blocks = block; } @@ -861,7 +861,7 @@ cvFlushSeqWriter( CvSeqWriter * writer ) CvSeqBlock *block = first_block; writer->block->count = (int)((writer->ptr - writer->block->data) / seq->elem_size); - assert( writer->block->count > 0 ); + CV_Assert( writer->block->count > 0 ); do { @@ -891,7 +891,7 @@ cvEndWriteSeq( CvSeqWriter * writer ) CvMemStorage *storage = seq->storage; schar *storage_block_max = (schar *) storage->top + storage->block_size; - assert( writer->block->count > 0 ); + CV_Assert( writer->block->count > 0 ); if( (unsigned)((storage_block_max - storage->free_space) - seq->block_max) < CV_STRUCT_ALIGN ) @@ -1147,7 +1147,7 @@ cvSeqPush( CvSeq *seq, const void *element ) icvGrowSeq( seq, 0 ); ptr = seq->ptr; - assert( ptr + elem_size <= seq->block_max /*&& ptr == seq->block_min */ ); + CV_Assert( ptr + elem_size <= seq->block_max /*&& ptr == seq->block_min */ ); } if( element ) @@ -1183,7 +1183,7 @@ cvSeqPop( CvSeq *seq, void *element ) if( --(seq->first->prev->count) == 0 ) { icvFreeSeqBlock( seq, 0 ); - assert( seq->ptr == seq->block_max ); + CV_Assert( seq->ptr == seq->block_max ); } } @@ -1207,7 +1207,7 @@ cvSeqPushFront( CvSeq *seq, const void *element ) icvGrowSeq( seq, 1 ); block = seq->first; - assert( block->start_index > 0 ); + CV_Assert( block->start_index > 0 ); } ptr = block->data -= elem_size; @@ -1289,7 +1289,7 @@ cvSeqInsert( CvSeq *seq, int before_index, const void *element ) icvGrowSeq( seq, 0 ); ptr = seq->ptr + elem_size; - assert( ptr <= seq->block_max ); + CV_Assert( ptr <= seq->block_max ); } delta_index = seq->first->start_index; @@ -1307,7 +1307,7 @@ cvSeqInsert( CvSeq *seq, int before_index, const void *element ) block = prev_block; /* Check that we don't fall into an infinite loop: */ - assert( block != seq->first->prev ); + CV_Assert( block != seq->first->prev ); } before_index = (before_index - block->start_index + delta_index) * elem_size; @@ -1346,7 +1346,7 @@ cvSeqInsert( CvSeq *seq, int before_index, const void *element ) block = next_block; /* Check that we don't fall into an infinite loop: */ - assert( block != seq->first ); + CV_Assert( block != seq->first ); } before_index = (before_index - block->start_index + delta_index) * elem_size; @@ -1502,7 +1502,7 @@ cvSeqPushMulti( CvSeq *seq, const void *_elements, int count, int front ) icvGrowSeq( seq, 1 ); block = seq->first; - assert( block->start_index > 0 ); + CV_Assert( block->start_index > 0 ); } delta = MIN( block->start_index, count ); @@ -1543,7 +1543,7 @@ cvSeqPopMulti( CvSeq *seq, void *_elements, int count, int front ) int delta = seq->first->prev->count; delta = MIN( delta, count ); - assert( delta > 0 ); + CV_Assert( delta > 0 ); seq->first->prev->count -= delta; seq->total -= delta; @@ -1568,7 +1568,7 @@ cvSeqPopMulti( CvSeq *seq, void *_elements, int count, int front ) int delta = seq->first->count; delta = MIN( delta, count ); - assert( delta > 0 ); + CV_Assert( delta > 0 ); seq->first->count -= delta; seq->total -= delta; @@ -2418,7 +2418,7 @@ cvSeqPartition( const CvSeq* seq, CvMemStorage* storage, CvSeq** labels, root2->rank += root->rank == root2->rank; root = root2; } - assert( root->parent == 0 ); + CV_Assert( root->parent == 0 ); // Compress path from node2 to the root: while( node2->parent ) @@ -2521,7 +2521,7 @@ cvSetAdd( CvSet* set, CvSetElem* element, CvSetElem** inserted_element ) ((CvSetElem*)ptr)->flags = count | CV_SET_ELEM_FREE_FLAG; ((CvSetElem*)ptr)->next_free = (CvSetElem*)(ptr + elem_size); } - assert( count <= CV_SET_ELEM_IDX_MASK+1 ); + CV_Assert( count <= CV_SET_ELEM_IDX_MASK+1 ); ((CvSetElem*)(ptr - elem_size))->next_free = 0; set->first->prev->count += count - set->total; set->total = count; @@ -2720,7 +2720,7 @@ cvFindGraphEdgeByPtr( const CvGraph* graph, for( ; edge; edge = edge->next[ofs] ) { ofs = start_vtx == edge->vtx[1]; - assert( ofs == 1 || start_vtx == edge->vtx[0] ); + CV_Assert( ofs == 1 || start_vtx == edge->vtx[0] ); if( edge->vtx[1] == end_vtx ) break; } @@ -2784,7 +2784,7 @@ cvGraphAddEdgeByPtr( CvGraph* graph, "vertex pointers coincide (or set to NULL)" ); edge = (CvGraphEdge*)cvSetNew( (CvSet*)(graph->edges) ); - assert( edge->flags >= 0 ); + CV_Assert( edge->flags >= 0 ); edge->vtx[0] = start_vtx; edge->vtx[1] = end_vtx; @@ -2861,7 +2861,7 @@ cvGraphRemoveEdgeByPtr( CvGraph* graph, CvGraphVtx* start_vtx, CvGraphVtx* end_v prev_ofs = ofs, prev_edge = edge, edge = edge->next[ofs] ) { ofs = start_vtx == edge->vtx[1]; - assert( ofs == 1 || start_vtx == edge->vtx[0] ); + CV_Assert( ofs == 1 || start_vtx == edge->vtx[0] ); if( edge->vtx[1] == end_vtx ) break; } @@ -2879,7 +2879,7 @@ cvGraphRemoveEdgeByPtr( CvGraph* graph, CvGraphVtx* start_vtx, CvGraphVtx* end_v prev_ofs = ofs, prev_edge = edge, edge = edge->next[ofs] ) { ofs = end_vtx == edge->vtx[1]; - assert( ofs == 1 || end_vtx == edge->vtx[0] ); + CV_Assert( ofs == 1 || end_vtx == edge->vtx[0] ); if( edge->vtx[0] == start_vtx ) break; } @@ -3396,7 +3396,7 @@ cvInsertNodeIntoTree( void* _node, void* _parent, void* _frame ) node->v_prev = _parent != _frame ? parent : 0; node->h_next = parent->v_next; - assert( parent->v_next != node ); + CV_Assert( parent->v_next != node ); if( parent->v_next ) parent->v_next->h_prev = node; @@ -3430,7 +3430,7 @@ cvRemoveNodeFromTree( void* _node, void* _frame ) if( parent ) { - assert( parent->v_next == node ); + CV_Assert( parent->v_next == node ); parent->v_next = node->h_next; } } diff --git a/modules/core/src/dxt.cpp b/modules/core/src/dxt.cpp index e378f31e66..a15f21b47f 100644 --- a/modules/core/src/dxt.cpp +++ b/modules/core/src/dxt.cpp @@ -238,7 +238,7 @@ DFTInit( int n0, int nf, const int* factors, int* itab, int elem_size, void* _wa else { // radix[] is initialized from index 'nf' down to zero - assert (nf < 34); + CV_Assert (nf < 34); radix[nf] = 1; digits[nf] = 0; for( i = 0; i < nf; i++ ) @@ -374,7 +374,7 @@ DFTInit( int n0, int nf, const int* factors, int* itab, int elem_size, void* _wa else { Complex* wave = (Complex*)_wave; - assert( elem_size == sizeof(Complex) ); + CV_Assert( elem_size == sizeof(Complex) ); wave[0].re = 1.f; wave[0].im = 0.f; @@ -874,13 +874,13 @@ DFT(const OcvDftOptions & c, const Complex* src, Complex* dst) // 0. shuffle data if( dst != src ) { - assert( !c.noPermute ); + CV_Assert( !c.noPermute ); if( !inv ) { for( i = 0; i <= n - 2; i += 2, itab += 2*tab_step ) { int k0 = itab[0], k1 = itab[tab_step]; - assert( (unsigned)k0 < (unsigned)n && (unsigned)k1 < (unsigned)n ); + CV_Assert( (unsigned)k0 < (unsigned)n && (unsigned)k1 < (unsigned)n ); dst[i] = src[k0]; dst[i+1] = src[k1]; } @@ -892,7 +892,7 @@ DFT(const OcvDftOptions & c, const Complex* src, Complex* dst) for( i = 0; i <= n - 2; i += 2, itab += 2*tab_step ) { int k0 = itab[0], k1 = itab[tab_step]; - assert( (unsigned)k0 < (unsigned)n && (unsigned)k1 < (unsigned)n ); + CV_Assert( (unsigned)k0 < (unsigned)n && (unsigned)k1 < (unsigned)n ); t.re = src[k0].re; t.im = -src[k0].im; dst[i] = t; t.re = src[k1].re; t.im = -src[k1].im; @@ -921,7 +921,7 @@ DFT(const OcvDftOptions & c, const Complex* src, Complex* dst) for( i = 0; i < n2; i += 2, itab += tab_step*2 ) { j = itab[0]; - assert( (unsigned)j < (unsigned)n2 ); + CV_Assert( (unsigned)j < (unsigned)n2 ); CV_SWAP(dst[i+1], dsth[j], t); if( j > i ) @@ -938,7 +938,7 @@ DFT(const OcvDftOptions & c, const Complex* src, Complex* dst) for( i = 0; i < n; i++, itab += tab_step ) { j = itab[0]; - assert( (unsigned)j < (unsigned)n ); + CV_Assert( (unsigned)j < (unsigned)n ); if( j > i ) CV_SWAP(dst[i], dst[j], t); } @@ -1218,7 +1218,7 @@ RealDFT(const OcvDftOptions & c, const T* src, T* dst) setIppErrorStatus(); #endif } - assert( c.tab_size == n ); + CV_Assert( c.tab_size == n ); if( n == 1 ) { @@ -1338,11 +1338,11 @@ CCSIDFT(const OcvDftOptions & c, const T* src, T* dst) T save_s1 = 0.; T t0, t1, t2, t3, t; - assert( c.tab_size == n ); + CV_Assert( c.tab_size == n ); if( complex_input ) { - assert( src != dst ); + CV_Assert( src != dst ); save_s1 = src[1]; ((T*)src)[1] = src[0]; src++; @@ -3175,7 +3175,7 @@ protected: } else { - assert( !inv ); + CV_Assert( !inv ); CopyColumn( dbuf0, complex_elem_size, dptr0, dst_step, len, complex_elem_size ); if( even ) @@ -3872,7 +3872,7 @@ DCTInit( int n, int elem_size, void* _wave, int inv ) if( n == 1 ) return; - assert( (n&1) == 0 ); + CV_Assert( (n&1) == 0 ); if( (n & (n - 1)) == 0 ) { @@ -3910,7 +3910,7 @@ DCTInit( int n, int elem_size, void* _wave, int inv ) else { Complex* wave = (Complex*)_wave; - assert( elem_size == sizeof(Complex) ); + CV_Assert( elem_size == sizeof(Complex) ); w.re = (float)scale; w.im = 0.f; diff --git a/modules/core/src/lapack.cpp b/modules/core/src/lapack.cpp index 9bca6a8211..a644fe15a7 100644 --- a/modules/core/src/lapack.cpp +++ b/modules/core/src/lapack.cpp @@ -1020,7 +1020,7 @@ double invert( InputArray _src, OutputArray _dst, int method ) } else { - assert( n == 1 ); + CV_Assert( n == 1 ); if( type == CV_32FC1 ) { @@ -1208,7 +1208,7 @@ bool solve( InputArray _src, InputArray _src2arg, OutputArray _dst, int method ) } else { - assert( src.rows == 1 ); + CV_Assert( src.rows == 1 ); if( type == CV_32FC1 ) { diff --git a/modules/core/src/matmul.simd.hpp b/modules/core/src/matmul.simd.hpp index c828e2906d..5a7f36d12b 100644 --- a/modules/core/src/matmul.simd.hpp +++ b/modules/core/src/matmul.simd.hpp @@ -169,7 +169,7 @@ GEMM_TransposeBlock( const uchar* src, size_t src_step, } break; default: - assert(0); + CV_Assert(0); return; } } @@ -2062,7 +2062,7 @@ MulTransposedR(const Mat& srcmat, const Mat& dstmat, const Mat& deltamat, double if( delta && delta_cols < size.width ) { - assert( delta_cols == 1 ); + CV_Assert( delta_cols == 1 ); buf_size *= 5; } buf.allocate(buf_size); diff --git a/modules/core/src/matrix_sparse.cpp b/modules/core/src/matrix_sparse.cpp index 21e7e91151..173f9ea8f6 100644 --- a/modules/core/src/matrix_sparse.cpp +++ b/modules/core/src/matrix_sparse.cpp @@ -638,7 +638,7 @@ void SparseMat::resizeHashTab(size_t newsize) uchar* SparseMat::newNode(const int* idx, size_t hashval) { const int HASH_MAX_FILL_FACTOR=3; - assert(hdr); + CV_Assert(hdr); size_t hsize = hdr->hashtab.size(); if( ++hdr->nodeCount > hsize*HASH_MAX_FILL_FACTOR ) { diff --git a/modules/core/src/opencl/runtime/opencl_clamdblas.cpp b/modules/core/src/opencl/runtime/opencl_clamdblas.cpp index 379929993f..9b7aaa4c61 100644 --- a/modules/core/src/opencl/runtime/opencl_clamdblas.cpp +++ b/modules/core/src/opencl/runtime/opencl_clamdblas.cpp @@ -113,7 +113,7 @@ static void* openclamdblas_check_fn(int ID); static void* openclamdblas_check_fn(int ID) { - assert(ID >= 0 && ID < (int)(sizeof(openclamdblas_fn)/sizeof(openclamdblas_fn[0]))); + CV_Assert(ID >= 0 && ID < (int)(sizeof(openclamdblas_fn)/sizeof(openclamdblas_fn[0]))); const struct DynamicFnEntry* e = openclamdblas_fn[ID]; void* func = CV_CL_GET_PROC_ADDRESS(e->fnName); if (!func) diff --git a/modules/core/src/opencl/runtime/opencl_clamdfft.cpp b/modules/core/src/opencl/runtime/opencl_clamdfft.cpp index 255bcd826a..eb5d6f0169 100644 --- a/modules/core/src/opencl/runtime/opencl_clamdfft.cpp +++ b/modules/core/src/opencl/runtime/opencl_clamdfft.cpp @@ -113,7 +113,7 @@ static void* openclamdfft_check_fn(int ID); static void* openclamdfft_check_fn(int ID) { - assert(ID >= 0 && ID < (int)(sizeof(openclamdfft_fn)/sizeof(openclamdfft_fn[0]))); + CV_Assert(ID >= 0 && ID < (int)(sizeof(openclamdfft_fn)/sizeof(openclamdfft_fn[0]))); const struct DynamicFnEntry* e = openclamdfft_fn[ID]; void* func = CV_CL_GET_PROC_ADDRESS(e->fnName); if (!func) diff --git a/modules/core/src/opencl/runtime/opencl_core.cpp b/modules/core/src/opencl/runtime/opencl_core.cpp index db2385a258..464777f864 100644 --- a/modules/core/src/opencl/runtime/opencl_core.cpp +++ b/modules/core/src/opencl/runtime/opencl_core.cpp @@ -360,7 +360,7 @@ static void* opencl_gl_check_fn(int ID); static void* opencl_gl_check_fn(int ID) { const struct DynamicFnEntry* e = NULL; - assert(ID >= 0 && ID < (int)(sizeof(opencl_gl_fn_list)/sizeof(opencl_gl_fn_list[0]))); + CV_Assert(ID >= 0 && ID < (int)(sizeof(opencl_gl_fn_list)/sizeof(opencl_gl_fn_list[0]))); e = opencl_gl_fn_list[ID]; void* func = CV_CL_GET_PROC_ADDRESS(e->fnName); if (!func) diff --git a/modules/core/src/persistence.cpp b/modules/core/src/persistence.cpp index 7e9d107c35..ff24b105a6 100644 --- a/modules/core/src/persistence.cpp +++ b/modules/core/src/persistence.cpp @@ -160,7 +160,7 @@ void icvFSCreateCollection( CvFileStorage* fs, int tag, CvFileNode* collection ) { if( collection->tag != CV_NODE_NONE ) { - assert( fs->fmt == CV_STORAGE_FORMAT_XML ); + CV_Assert( fs->fmt == CV_STORAGE_FORMAT_XML ); CV_PARSE_ERROR( "Sequence element should not have name (use <_>)" ); } @@ -551,7 +551,7 @@ int icvDecodeFormat( const char* dt, int* fmt_pairs, int max_len ) if( !dt || !len ) return 0; - assert( fmt_pairs != 0 && max_len > 0 ); + CV_Assert( fmt_pairs != 0 && max_len > 0 ); fmt_pairs[0] = 0; max_len *= 2; diff --git a/modules/core/src/persistence_json.cpp b/modules/core/src/persistence_json.cpp index 1ed6321b78..763a3fbe76 100644 --- a/modules/core/src/persistence_json.cpp +++ b/modules/core/src/persistence_json.cpp @@ -756,7 +756,7 @@ void icvJSONEndWriteStruct( CvFileStorage* fs ) cvSeqPop( fs->write_stack, &parent_flags ); fs->struct_indent -= 4; fs->struct_flags = parent_flags & ~CV_NODE_EMPTY; - assert( fs->struct_indent >= 0 ); + CV_Assert( fs->struct_indent >= 0 ); if ( CV_NODE_IS_COLLECTION(struct_flags) ) { diff --git a/modules/core/src/persistence_types.cpp b/modules/core/src/persistence_types.cpp index a446645734..ed1e1bd82d 100644 --- a/modules/core/src/persistence_types.cpp +++ b/modules/core/src/persistence_types.cpp @@ -34,7 +34,7 @@ static void icvWriteMat( CvFileStorage* fs, const char* name, const void* struct cv::Size size; int y; - assert( CV_IS_MAT_HDR_Z(mat) ); + CV_Assert( CV_IS_MAT_HDR_Z(mat) ); cvStartWriteStruct( fs, name, CV_NODE_MAP, CV_TYPE_NAME_MAT ); cvWriteInt( fs, "rows", mat->rows ); @@ -121,7 +121,7 @@ static void icvWriteMatND( CvFileStorage* fs, const char* name, const void* stru int dims, sizes[CV_MAX_DIM]; char dt[16]; - assert( CV_IS_MATND_HDR(mat) ); + CV_Assert( CV_IS_MATND_HDR(mat) ); cvStartWriteStruct( fs, name, CV_NODE_MAP, CV_TYPE_NAME_MATND ); dims = cvGetDims( mat, sizes ); @@ -237,7 +237,7 @@ static void icvWriteSparseMat( CvFileStorage* fs, const char* name, const void* int *prev_idx = 0; char dt[16]; - assert( CV_IS_SPARSE_MAT(mat) ); + CV_Assert( CV_IS_SPARSE_MAT(mat) ); memstorage = cvCreateMemStorage(); @@ -273,7 +273,7 @@ static void icvWriteSparseMat( CvFileStorage* fs, const char* name, const void* if( i > 0 ) { for( ; idx[k] == prev_idx[k]; k++ ) - assert( k < dims ); + CV_Assert( k < dims ); if( k < dims - 1 ) fs->write_int( fs, 0, k - dims + 1 ); } @@ -383,7 +383,7 @@ static void icvWriteImage( CvFileStorage* fs, const char* name, const void* stru cv::Size size; int y, depth; - assert( CV_IS_IMAGE(image) ); + CV_Assert( CV_IS_IMAGE(image) ); if( image->dataOrder == IPL_DATA_ORDER_PLANE ) CV_Error( CV_StsUnsupportedFormat, @@ -623,7 +623,7 @@ static void icvWriteSeq( CvFileStorage* fs, const char* name, const void* struct char buf[128]; char dt_buf[128], *dt; - assert( CV_IS_SEQ( seq )); + CV_Assert( CV_IS_SEQ( seq )); cvStartWriteStruct( fs, name, CV_NODE_MAP, CV_TYPE_NAME_SEQ ); if( level >= 0 ) @@ -671,7 +671,7 @@ static void icvWriteSeqTree( CvFileStorage* fs, const char* name, const void* st strcmp(recursive_value,"False") != 0 && strcmp(recursive_value,"FALSE") != 0; - assert( CV_IS_SEQ( seq )); + CV_Assert( CV_IS_SEQ( seq )); if( !is_recursive ) { @@ -873,7 +873,7 @@ static void* icvReadSeqTree( CvFileStorage* fs, CvFileNode* node ) root = seq; if( level > prev_level ) { - assert( level == prev_level + 1 ); + CV_Assert( level == prev_level + 1 ); parent = prev_seq; prev_seq = 0; if( parent ) @@ -933,7 +933,7 @@ static void icvWriteGraph( CvFileStorage* fs, const char* name, const void* stru char edge_dt_buf[128], *edge_dt; int write_buf_size; - assert( CV_IS_GRAPH(graph) ); + CV_Assert( CV_IS_GRAPH(graph) ); vtx_count = cvGraphGetVtxCount( graph ); edge_count = cvGraphGetEdgeCount( graph ); flag_buf = (int*)cvAlloc( vtx_count*sizeof(flag_buf[0])); diff --git a/modules/core/src/persistence_xml.cpp b/modules/core/src/persistence_xml.cpp index aaaa613bc2..a22faaa358 100644 --- a/modules/core/src/persistence_xml.cpp +++ b/modules/core/src/persistence_xml.cpp @@ -37,7 +37,7 @@ icvXMLSkipSpaces( CvFileStorage* fs, char* ptr, int mode ) if( c == '-' ) { - assert( ptr[1] == '-' && ptr[2] == '>' ); + CV_Assert( ptr[1] == '-' && ptr[2] == '>' ); mode = 0; ptr += 3; } @@ -484,7 +484,7 @@ icvXMLParseTag( CvFileStorage* fs, char* ptr, CvStringHashNode** _tag, else if( *ptr == '!' ) { tag_type = CV_XML_DIRECTIVE_TAG; - assert( ptr[1] != '-' || ptr[2] != '-' ); + CV_Assert( ptr[1] != '-' || ptr[2] != '-' ); ptr++; } else @@ -549,7 +549,7 @@ icvXMLParseTag( CvFileStorage* fs, char* ptr, CvStringHashNode** _tag, } ptr = icvXMLParseValue( fs, ptr, &stub, CV_NODE_STRING ); - assert( stub.tag == CV_NODE_STRING ); + CV_Assert( stub.tag == CV_NODE_STRING ); last->attr[count*2+1] = stub.data.str.ptr; count++; } diff --git a/modules/core/src/persistence_yml.cpp b/modules/core/src/persistence_yml.cpp index 713da33c50..885139fe76 100644 --- a/modules/core/src/persistence_yml.cpp +++ b/modules/core/src/persistence_yml.cpp @@ -797,7 +797,7 @@ void icvYMLEndWriteStruct( CvFileStorage* fs ) if( !CV_NODE_IS_FLOW(parent_flags) ) fs->struct_indent -= CV_YML_INDENT + CV_NODE_IS_FLOW(struct_flags); - assert( fs->struct_indent >= 0 ); + CV_Assert( fs->struct_indent >= 0 ); fs->struct_flags = parent_flags; } diff --git a/modules/core/test/test_ds.cpp b/modules/core/test/test_ds.cpp index 64d54c5966..a215b58d77 100644 --- a/modules/core/test/test_ds.cpp +++ b/modules/core/test/test_ds.cpp @@ -33,7 +33,7 @@ static void cvTsReleaseSimpleSeq( CvTsSimpleSeq** seq ) static schar* cvTsSimpleSeqElem( CvTsSimpleSeq* seq, int index ) { - assert( 0 <= index && index < seq->count ); + CV_Assert( 0 <= index && index < seq->count ); return seq->array + index * seq->elem_size; } @@ -50,7 +50,11 @@ static void cvTsSimpleSeqShiftAndCopy( CvTsSimpleSeq* seq, int from_idx, int to_ if( from_idx == to_idx ) return; - assert( (from_idx > to_idx && !elem) || (from_idx < to_idx && elem) ); + + if (elem) + CV_Assert(from_idx < to_idx); + else + CV_Assert(from_idx > to_idx); if( from_idx < seq->count ) { @@ -128,7 +132,7 @@ static void cvTsReleaseSimpleSet( CvTsSimpleSet** set_header ) static schar* cvTsSimpleSetFind( CvTsSimpleSet* set_header, int index ) { int idx = index * set_header->elem_size; - assert( 0 <= index && index < set_header->max_count ); + CV_Assert( 0 <= index && index < set_header->max_count ); return set_header->array[idx] ? set_header->array + idx + 1 : 0; } @@ -136,11 +140,11 @@ static schar* cvTsSimpleSetFind( CvTsSimpleSet* set_header, int index ) static int cvTsSimpleSetAdd( CvTsSimpleSet* set_header, void* elem ) { int idx, idx2; - assert( set_header->free_count > 0 ); + CV_Assert( set_header->free_count > 0 ); idx = set_header->free_stack[--set_header->free_count]; idx2 = idx * set_header->elem_size; - assert( set_header->array[idx2] == 0 ); + CV_Assert( set_header->array[idx2] == 0 ); set_header->array[idx2] = 1; if( set_header->elem_size > 1 ) memcpy( set_header->array + idx2 + 1, elem, set_header->elem_size - 1 ); @@ -152,9 +156,9 @@ static int cvTsSimpleSetAdd( CvTsSimpleSet* set_header, void* elem ) static void cvTsSimpleSetRemove( CvTsSimpleSet* set_header, int index ) { - assert( set_header->free_count < set_header->max_count && - 0 <= index && index < set_header->max_count ); - assert( set_header->array[index * set_header->elem_size] == 1 ); + CV_Assert( set_header->free_count < set_header->max_count && + 0 <= index && index < set_header->max_count ); + CV_Assert( set_header->array[index * set_header->elem_size] == 1 ); set_header->free_stack[set_header->free_count++] = index; set_header->array[index * set_header->elem_size] = 0; @@ -187,7 +191,7 @@ static CvTsSimpleGraph* cvTsCreateSimpleGraph( int max_vtx_count, int vtx_size, { CvTsSimpleGraph* graph; - assert( max_vtx_count > 1 && vtx_size >= 0 && edge_size >= 0 ); + CV_Assert( max_vtx_count > 1 && vtx_size >= 0 && edge_size >= 0 ); graph = (CvTsSimpleGraph*)cvAlloc( sizeof(*graph) + max_vtx_count * max_vtx_count * (edge_size + 1)); graph->vtx = cvTsCreateSimpleSet( max_vtx_count, vtx_size ); @@ -235,13 +239,13 @@ static void cvTsSimpleGraphAddEdge( CvTsSimpleGraph* graph, int idx1, int idx2, { int i, t, n = graph->oriented ? 1 : 2; - assert( cvTsSimpleSetFind( graph->vtx, idx1 ) && - cvTsSimpleSetFind( graph->vtx, idx2 )); + CV_Assert( cvTsSimpleSetFind( graph->vtx, idx1 ) && + cvTsSimpleSetFind( graph->vtx, idx2 )); for( i = 0; i < n; i++ ) { int ofs = (idx1*graph->vtx->max_count + idx2)*graph->edge_size; - assert( graph->matrix[ofs] == 0 ); + CV_Assert( graph->matrix[ofs] == 0 ); graph->matrix[ofs] = 1; if( graph->edge_size > 1 ) memcpy( graph->matrix + ofs + 1, edge, graph->edge_size - 1 ); @@ -255,13 +259,13 @@ static void cvTsSimpleGraphRemoveEdge( CvTsSimpleGraph* graph, int idx1, int id { int i, t, n = graph->oriented ? 1 : 2; - assert( cvTsSimpleSetFind( graph->vtx, idx1 ) && + CV_Assert( cvTsSimpleSetFind( graph->vtx, idx1 ) && cvTsSimpleSetFind( graph->vtx, idx2 )); for( i = 0; i < n; i++ ) { int ofs = (idx1*graph->vtx->max_count + idx2)*graph->edge_size; - assert( graph->matrix[ofs] == 1 ); + CV_Assert( graph->matrix[ofs] == 1 ); graph->matrix[ofs] = 0; CV_SWAP( idx1, idx2, t ); } @@ -291,7 +295,7 @@ static int cvTsSimpleGraphVertexDegree( CvTsSimpleGraph* graph, int index ) int i, count = 0; int edge_size = graph->edge_size; int max_vtx_count = graph->vtx->max_count; - assert( cvTsSimpleGraphFindVertex( graph, index ) != 0 ); + CV_Assert( cvTsSimpleGraphFindVertex( graph, index ) != 0 ); for( i = 0; i < max_vtx_count; i++ ) { @@ -301,7 +305,7 @@ static int cvTsSimpleGraphVertexDegree( CvTsSimpleGraph* graph, int index ) if( !graph->oriented ) { - assert( count % 2 == 0 ); + CV_Assert( count % 2 == 0 ); count /= 2; } return count; @@ -609,7 +613,7 @@ int Core_SeqBaseTest::test_get_seq_elem( int _struct_idx, int iters ) CvTsSimpleSeq* sseq = (CvTsSimpleSeq*)simple_struct[_struct_idx]; struct_idx = _struct_idx; - assert( seq->total == sseq->count ); + CV_Assert( seq->total == sseq->count ); if( sseq->count == 0 ) return 0; @@ -656,7 +660,7 @@ int Core_SeqBaseTest::test_get_seq_reading( int _struct_idx, int iters ) vector _elem(sseq->elem_size); schar* elem = &_elem[0]; - assert( total == sseq->count ); + CV_Assert( total == sseq->count ); this->struct_idx = _struct_idx; int pos = cvtest::randInt(rng) % 2; @@ -964,7 +968,7 @@ int Core_SeqBaseTest::test_seq_ops( int iters ) "The sequence doesn't become empty after clear" ); break; default: - assert(0); + CV_Assert(0); return -1; } @@ -1903,7 +1907,7 @@ int Core_GraphScanTest::create_random_graph( int _struct_idx ) for( i = 0; i < vtx_count; i++ ) cvGraphAddVtx( graph ); - assert( graph->active_count == vtx_count ); + CV_Assert( graph->active_count == vtx_count ); for( i = 0; i < edge_count; i++ ) { @@ -1914,7 +1918,7 @@ int Core_GraphScanTest::create_random_graph( int _struct_idx ) cvGraphAddEdge( graph, j, k ); } - assert( graph->active_count == vtx_count && graph->edges->active_count <= edge_count ); + CV_Assert( graph->active_count == vtx_count && graph->edges->active_count <= edge_count ); return 0; } diff --git a/modules/core/test/test_dxt.cpp b/modules/core/test/test_dxt.cpp index dbb378aa08..05d1f3062c 100644 --- a/modules/core/test/test_dxt.cpp +++ b/modules/core/test/test_dxt.cpp @@ -204,7 +204,7 @@ static void DCT_1D( const Mat& _src, Mat& _dst, int flags, const Mat& _wave=Mat( } } else - assert(0); + CV_Assert(0); } diff --git a/modules/core/test/test_mat.cpp b/modules/core/test/test_mat.cpp index 4f7cbb6725..041e7d979f 100644 --- a/modules/core/test/test_mat.cpp +++ b/modules/core/test/test_mat.cpp @@ -28,7 +28,7 @@ protected: template void testReduce( const Mat& src, Mat& sum, Mat& avg, Mat& max, Mat& min, int dim ) { - assert( src.channels() == 1 ); + CV_Assert( src.channels() == 1 ); if( dim == 0 ) // row { sum.create( 1, src.cols, CV_64FC1 ); @@ -138,7 +138,7 @@ int Core_ReduceTest::checkOp( const Mat& src, int dstType, int opType, const Mat eps = 0.6; } - assert( opRes.type() == CV_64FC1 ); + CV_Assert( opRes.type() == CV_64FC1 ); Mat _dst, dst, diff; cv::reduce( src, _dst, dim, opType, dstType ); _dst.convertTo( dst, CV_64FC1 ); @@ -192,7 +192,7 @@ int Core_ReduceTest::checkCase( int srcType, int dstType, int dim, Size sz ) else if( srcType == CV_64FC1 ) testReduce( src, sum, avg, max, min, dim ); else - assert( 0 ); + CV_Assert( 0 ); // 1. sum tempCode = checkOp( src, dstType, CV_REDUCE_SUM, sum, dim ); diff --git a/modules/core/test/test_math.cpp b/modules/core/test/test_math.cpp index 066475b19e..cbd0b7ceb3 100644 --- a/modules/core/test/test_math.cpp +++ b/modules/core/test/test_math.cpp @@ -1039,7 +1039,7 @@ static void cvTsPerspectiveTransform( const CvArr* _src, CvArr* _dst, const CvMa } else { - assert( mat_depth == CV_64F ); + CV_Assert( mat_depth == CV_64F ); for( i = 0; i < transmat->rows; i++ ) for( j = 0; j < cols; j++ ) mat[i*cols + j] = ((double*)(transmat->data.ptr + transmat->step*i))[j]; @@ -1065,7 +1065,7 @@ static void cvTsPerspectiveTransform( const CvArr* _src, CvArr* _dst, const CvMa buf[j] = ((double*)src)[j]; break; default: - assert(0); + CV_Assert(0); } switch( cn ) @@ -1095,7 +1095,7 @@ static void cvTsPerspectiveTransform( const CvArr* _src, CvArr* _dst, const CvMa } break; default: - assert(0); + CV_Assert(0); } switch( depth ) @@ -1109,7 +1109,7 @@ static void cvTsPerspectiveTransform( const CvArr* _src, CvArr* _dst, const CvMa ((double*)dst)[j] = buf[j]; break; default: - assert(0); + CV_Assert(0); } } } @@ -1458,8 +1458,8 @@ static double cvTsLU( CvMat* a, CvMat* b=NULL, CvMat* x=NULL, int* rank=0 ) double *a0 = a->data.db, *b0 = b ? b->data.db : 0; double *x0 = x ? x->data.db : 0; double t, det = 1.; - assert( CV_MAT_TYPE(a->type) == CV_64FC1 && - (!b || CV_ARE_TYPES_EQ(a,b)) && (!x || CV_ARE_TYPES_EQ(a,x))); + CV_Assert( CV_MAT_TYPE(a->type) == CV_64FC1 && + (!b || CV_ARE_TYPES_EQ(a,b)) && (!x || CV_ARE_TYPES_EQ(a,x))); for( i = 0; i < Nm; i++ ) { @@ -1514,7 +1514,7 @@ static double cvTsLU( CvMat* a, CvMat* b=NULL, CvMat* x=NULL, int* rank=0 ) if( x ) { - assert( b ); + CV_Assert( b ); for( i = N-1; i >= 0; i-- ) { diff --git a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp index dc1a7b89ef..a6f0b796bf 100644 --- a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp +++ b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp @@ -48,7 +48,6 @@ #include #include #include -#include #include "../include/common.hpp" #include "../include/ocl4dnn.hpp" #include "opencl_kernels_dnn.hpp" diff --git a/modules/features2d/test/test_descriptors_regression.cpp b/modules/features2d/test/test_descriptors_regression.cpp index f8760d6478..0de2b2bd55 100644 --- a/modules/features2d/test/test_descriptors_regression.cpp +++ b/modules/features2d/test/test_descriptors_regression.cpp @@ -159,7 +159,7 @@ protected: void emptyDataTest() { - assert( dextractor ); + CV_Assert( dextractor ); // One image. Mat image; @@ -205,7 +205,7 @@ protected: void regressionTest() { - assert( dextractor ); + CV_Assert( dextractor ); // Read the test image. string imgFilename = string(ts->get_data_path()) + FEATURES2D_DIR + "/" + IMAGE_FILENAME; diff --git a/modules/features2d/test/test_detectors_regression.cpp b/modules/features2d/test/test_detectors_regression.cpp index 5bdede7508..c082b43800 100644 --- a/modules/features2d/test/test_detectors_regression.cpp +++ b/modules/features2d/test/test_detectors_regression.cpp @@ -154,7 +154,7 @@ void CV_FeatureDetectorTest::compareKeypointSets( const vector& validK } } - assert( minDist >= 0 ); + CV_Assert( minDist >= 0 ); if( !isSimilarKeypoints( validKeypoints[v], calcKeypoints[nearestIdx] ) ) badPointCount++; } @@ -171,7 +171,7 @@ void CV_FeatureDetectorTest::compareKeypointSets( const vector& validK void CV_FeatureDetectorTest::regressionTest() { - assert( !fdetector.empty() ); + CV_Assert( !fdetector.empty() ); string imgFilename = string(ts->get_data_path()) + FEATURES2D_DIR + "/" + IMAGE_FILENAME; string resFilename = string(ts->get_data_path()) + DETECTOR_DIR + "/" + string(name) + ".xml.gz"; diff --git a/modules/features2d/test/test_nearestneighbors.cpp b/modules/features2d/test/test_nearestneighbors.cpp index f63ebb8d92..4f9d883866 100644 --- a/modules/features2d/test/test_nearestneighbors.cpp +++ b/modules/features2d/test/test_nearestneighbors.cpp @@ -312,7 +312,7 @@ void CV_FlannSavedIndexTest::createModel(const cv::Mat &data) case 1: createIndex( data, KDTreeIndexParams() ); break; //case 2: createIndex( data, CompositeIndexParams() ); break; // nothing to save for linear search //case 2: createIndex( data, AutotunedIndexParams() ); break; // possible linear index ! - default: assert(0); + default: CV_Assert(0); } string filename = tempfile(); index->save( filename ); diff --git a/modules/flann/include/opencv2/flann/nn_index.h b/modules/flann/include/opencv2/flann/nn_index.h index f6e17d19fc..23a1de7453 100644 --- a/modules/flann/include/opencv2/flann/nn_index.h +++ b/modules/flann/include/opencv2/flann/nn_index.h @@ -106,8 +106,8 @@ public: fprintf(stderr, "I can only search one feature at a time for range search\n"); return -1; } - assert(query.cols == veclen()); - assert(indices.cols == dists.cols); + CV_Assert(query.cols == veclen()); + CV_Assert(indices.cols == dists.cols); int n = 0; int* indices_ptr = NULL; diff --git a/modules/highgui/src/precomp.hpp b/modules/highgui/src/precomp.hpp index 12e823ba2c..6ce3a28aba 100644 --- a/modules/highgui/src/precomp.hpp +++ b/modules/highgui/src/precomp.hpp @@ -58,7 +58,6 @@ #include #include #include -#include #if defined _WIN32 || defined WINCE #include diff --git a/modules/highgui/src/window_carbon.cpp b/modules/highgui/src/window_carbon.cpp index 19bc9f1162..6fada90faa 100644 --- a/modules/highgui/src/window_carbon.cpp +++ b/modules/highgui/src/window_carbon.cpp @@ -119,7 +119,7 @@ static CvWindow* hg_windows = 0; if( !(exp) ) \ { \ printf("Assertion: %s %s: %d\n", #exp, __FILE__, __LINE__);\ - assert(exp); \ + CV_Assert(exp); \ } static int wasInitialized = 0; diff --git a/modules/highgui/src/window_gtk.cpp b/modules/highgui/src/window_gtk.cpp index 78e78e12a2..5ce88fb7d9 100644 --- a/modules/highgui/src/window_gtk.cpp +++ b/modules/highgui/src/window_gtk.cpp @@ -356,7 +356,7 @@ static void cvImageWidget_set_size(GtkWidget * widget, int max_width, int max_he } - assert( image_widget->scaled_image ); + CV_Assert( image_widget->scaled_image ); } static void diff --git a/modules/highgui/src/window_w32.cpp b/modules/highgui/src/window_w32.cpp index 50c3dc652f..13ad679b4b 100644 --- a/modules/highgui/src/window_w32.cpp +++ b/modules/highgui/src/window_w32.cpp @@ -60,7 +60,6 @@ using namespace cv; #include #include #include -#include #ifdef HAVE_OPENGL #include @@ -105,7 +104,7 @@ static const char* trackbar_text = static void FillBitmapInfo( BITMAPINFO* bmi, int width, int height, int bpp, int origin ) { - assert( bmi && width >= 0 && height >= 0 && (bpp == 8 || bpp == 24 || bpp == 32)); + CV_Assert( bmi && width >= 0 && height >= 0 && (bpp == 8 || bpp == 24 || bpp == 32)); BITMAPINFOHEADER* bmih = &(bmi->bmiHeader); @@ -1107,7 +1106,7 @@ static RECT icvCalcWindowRect( CvWindow* window ) { RECT crect = { 0 }, trect = { 0 }, rect = { 0 }; - assert(window); + CV_Assert(window); GetClientRect(window->frame, &crect); if (window->toolbar.toolbar) @@ -1157,7 +1156,7 @@ static bool icvGetBitmapData( CvWindow* window, SIZE* size, int* channels, void* static void icvUpdateWindowPos( CvWindow* window ) { RECT rect = { 0 }; - assert(window); + CV_Assert(window); if( (window->flags & CV_WINDOW_AUTOSIZE) && window->image ) { diff --git a/modules/highgui/src/window_winrt.cpp b/modules/highgui/src/window_winrt.cpp index 1572929b90..af771bd00b 100644 --- a/modules/highgui/src/window_winrt.cpp +++ b/modules/highgui/src/window_winrt.cpp @@ -30,7 +30,6 @@ #include #include #include -#include #include #include #include "window_winrt_bridge.hpp" diff --git a/modules/imgproc/src/approx.cpp b/modules/imgproc/src/approx.cpp index e581fb09e7..b7e51a4c62 100644 --- a/modules/imgproc/src/approx.cpp +++ b/modules/imgproc/src/approx.cpp @@ -129,7 +129,7 @@ CvSeq* icvApproximateChainTC89( CvChain* chain, int header_size, len = i; current = temp.next; - assert( current ); + CV_Assert( current ); /* Pass 1. Determines support region for all the remained points */ @@ -148,7 +148,7 @@ CvSeq* icvApproximateChainTC89( CvChain* chain, int header_size, int dx, dy; Cv32suf d; - assert( k <= len ); + CV_Assert( k <= len ); /* calc indices */ i1 = i - k; @@ -205,7 +205,7 @@ CvSeq* icvApproximateChainTC89( CvChain* chain, int header_size, ((double)dx2 * dx2 + (double)dy2 * dy2) )); sk.f = (float) (temp_num + 1.1); - assert( 0 <= sk.f && sk.f <= 2.2 ); + CV_Assert( 0 <= sk.f && sk.f <= 2.2 ); if( j < k && sk.i <= s ) break; @@ -258,7 +258,7 @@ CvSeq* icvApproximateChainTC89( CvChain* chain, int header_size, /* Pass 3. Removes non-dominant points with 1-length support region */ current = temp.next; - assert( current ); + CV_Assert( current ); prev_current = &temp; do @@ -293,7 +293,7 @@ CvSeq* icvApproximateChainTC89( CvChain* chain, int header_size, /* Pass 4. Cleans remained couples of points */ - assert( temp.next ); + CV_Assert( temp.next ); if( array[0].s != 0 && array[len - 1].s != 0 ) /* specific case */ { @@ -362,7 +362,7 @@ copy_vect: // gather points current = temp.next; - assert( current ); + CV_Assert( current ); do { @@ -439,7 +439,7 @@ cvApproxChains( CvSeq* src_seq, if( src_seq->v_next && len >= minimal_perimeter ) { - assert( prev_contour != 0 ); + CV_Assert( prev_contour != 0 ); parent = prev_contour; prev_contour = 0; src_seq = src_seq->v_next; @@ -590,7 +590,7 @@ approxPolyDP_( const Point_* src_contour, int count0, Point_* dst_contour, dx = end_pt.x - start_pt.x; dy = end_pt.y - start_pt.y; - assert( dx != 0 || dy != 0 ); + CV_Assert( dx != 0 || dy != 0 ); while( pos != slice.end ) { @@ -815,7 +815,7 @@ cvApproxPoly( const void* array, int header_size, CV_Error( CV_StsBadArg, "Invalid approximation method" ); } - assert( contour ); + CV_Assert( contour ); if( header_size >= (int)sizeof(CvContour)) cvBoundingRect( contour, 1 ); @@ -836,7 +836,7 @@ cvApproxPoly( const void* array, int header_size, if( src_seq->v_next ) { - assert( prev_contour != 0 ); + CV_Assert( prev_contour != 0 ); parent = prev_contour; prev_contour = 0; src_seq = src_seq->v_next; diff --git a/modules/imgproc/src/bilateral_filter.simd.hpp b/modules/imgproc/src/bilateral_filter.simd.hpp index 65abcd4e40..0d2c394368 100644 --- a/modules/imgproc/src/bilateral_filter.simd.hpp +++ b/modules/imgproc/src/bilateral_filter.simd.hpp @@ -205,7 +205,7 @@ public: } else { - assert( cn == 3 ); + CV_Assert( cn == 3 ); AutoBuffer buf(alignSize(size.width, CV_SIMD_WIDTH)*3 + size.width + CV_SIMD_WIDTH - 1); memset(buf.data(), 0, buf.size() * sizeof(float)); float *sum_b = alignPtr(buf.data(), CV_SIMD_WIDTH); diff --git a/modules/imgproc/src/contours.cpp b/modules/imgproc/src/contours.cpp index 241f1443f5..2afa96b293 100644 --- a/modules/imgproc/src/contours.cpp +++ b/modules/imgproc/src/contours.cpp @@ -97,7 +97,7 @@ cvReadChainPoint( CvChainPtReader * reader ) reader->ptr = ptr; reader->code = (schar)code; - assert( (code & ~7) == 0 ); + CV_Assert( (code & ~7) == 0 ); reader->pt.x = pt.x + icvCodeDeltas[code].x; reader->pt.y = pt.y + icvCodeDeltas[code].y; } @@ -1187,7 +1187,7 @@ cvFindNextContour( CvContourScanner scanner ) } /* hole flag of the parent must differ from the flag of the contour */ - assert( par_info->is_hole != is_hole ); + CV_Assert( par_info->is_hole != is_hole ); if( par_info->contour == 0 ) /* removed contour */ goto resume_scan; } diff --git a/modules/imgproc/src/convhull.cpp b/modules/imgproc/src/convhull.cpp index b964ca3f62..d6e02cc0fb 100644 --- a/modules/imgproc/src/convhull.cpp +++ b/modules/imgproc/src/convhull.cpp @@ -716,7 +716,7 @@ CV_IMPL CvSeq* cvConvexityDefects( const CvArr* array, dx0 = (double)hull_next->x - (double)hull_cur->x; dy0 = (double)hull_next->y - (double)hull_cur->y; - assert( dx0 != 0 || dy0 != 0 ); + CV_Assert( dx0 != 0 || dy0 != 0 ); scale = 1./std::sqrt(dx0*dx0 + dy0*dy0); defect.start = hull_cur; diff --git a/modules/imgproc/src/drawing.cpp b/modules/imgproc/src/drawing.cpp index aa889d1dc8..8bbe4fa0e3 100644 --- a/modules/imgproc/src/drawing.cpp +++ b/modules/imgproc/src/drawing.cpp @@ -138,7 +138,7 @@ bool clipLine( Size2l img_size, Point2l& pt1, Point2l& pt2 ) } } - assert( (c1 & c2) != 0 || (x1 | y1 | x2 | y2) >= 0 ); + CV_Assert( (c1 & c2) != 0 || (x1 | y1 | x2 | y2) >= 0 ); } return (c1 | c2) == 0; @@ -222,7 +222,7 @@ LineIterator::LineIterator(const Mat& img, Point pt1, Point pt2, if( connectivity == 8 ) { - assert( dx >= 0 && dy >= 0 ); + CV_Assert( dx >= 0 && dy >= 0 ); err = dx - (dy + dy); plusDelta = dx + dx; @@ -233,7 +233,7 @@ LineIterator::LineIterator(const Mat& img, Point pt1, Point pt2, } else /* connectivity == 4 */ { - assert( dx >= 0 && dy >= 0 ); + CV_Assert( dx >= 0 && dy >= 0 ); err = 0; plusDelta = (dx + dx) + (dy + dy); @@ -1102,7 +1102,7 @@ FillConvexPoly( Mat& img, const Point2l* v, int npts, const void* color, int lin p0.x <<= XY_SHIFT - shift; p0.y <<= XY_SHIFT - shift; - assert( 0 <= shift && shift <= XY_SHIFT ); + CV_Assert( 0 <= shift && shift <= XY_SHIFT ); xmin = xmax = v[0].x; ymin = ymax = v[0].y; @@ -1322,7 +1322,7 @@ FillEdgeCollection( Mat& img, std::vector& edges, const void* color ) for( i = 0; i < total; i++ ) { PolyEdge& e1 = edges[i]; - assert( e1.y0 < e1.y1 ); + CV_Assert( e1.y0 < e1.y1 ); // Determine x-coordinate of the end of the edge. // (This is not necessary x-coordinate of any vertex in the array.) int64 x1 = e1.x + (e1.y1 - e1.y0) * e1.dx; @@ -2596,7 +2596,7 @@ cvDrawContours( void* _img, CvSeq* contour, char code; CV_READ_SEQ_ELEM( code, reader ); - assert( (code & ~7) == 0 ); + CV_Assert( (code & ~7) == 0 ); if( code != prev_code ) { diff --git a/modules/imgproc/src/emd.cpp b/modules/imgproc/src/emd.cpp index 20ab6feafb..3e065b0404 100644 --- a/modules/imgproc/src/emd.cpp +++ b/modules/imgproc/src/emd.cpp @@ -336,7 +336,7 @@ static int icvInitEMD( const float* signature1, int size1, char *buffer, *buffer_end; memset( state, 0, sizeof( *state )); - assert( cost_step % sizeof(float) == 0 ); + CV_Assert( cost_step % sizeof(float) == 0 ); cost_step /= sizeof(float); /* calculate buffer size */ @@ -510,7 +510,7 @@ static int icvInitEMD( const float* signature1, int size1, } else { - assert( cost ); + CV_Assert( cost ); val = cost[cost_step*ci + cj]; } state->cost[i][j] = val; @@ -552,7 +552,7 @@ static int icvInitEMD( const float* signature1, int size1, buffer += dsize; } - assert( buffer <= buffer_end ); + CV_Assert( buffer <= buffer_end ); icvRussel( state ); diff --git a/modules/imgproc/src/hough.cpp b/modules/imgproc/src/hough.cpp index 9ed6cc5c57..5fa24f4789 100644 --- a/modules/imgproc/src/hough.cpp +++ b/modules/imgproc/src/hough.cpp @@ -340,8 +340,8 @@ HoughLinesSDiv( InputArray image, OutputArray lines, int type, rv = r0 * std::cos( phi ); i = (int)rv * tn; i += cvFloor( phi1 ); - assert( i >= 0 ); - assert( i < rn * tn ); + CV_Assert( i >= 0 ); + CV_Assert( i < rn * tn ); caccum[i] = (uchar) (caccum[i] + ((i ^ iprev) != 0)); iprev = i; if( cmax < caccum[i] ) @@ -405,8 +405,8 @@ HoughLinesSDiv( InputArray image, OutputArray lines, int type, i = CV_IMAX( i, -1 ); i = CV_IMIN( i, sfn ); mcaccum[i]++; - assert( i >= -1 ); - assert( i <= sfn ); + CV_Assert( i >= -1 ); + CV_Assert( i <= sfn ); } } diff --git a/modules/imgproc/src/median_blur.simd.hpp b/modules/imgproc/src/median_blur.simd.hpp index c3203f2a07..068b7d638f 100644 --- a/modules/imgproc/src/median_blur.simd.hpp +++ b/modules/imgproc/src/median_blur.simd.hpp @@ -463,7 +463,7 @@ medianBlur_8u_Om( const Mat& _src, Mat& _dst, int m ) } else { - assert( cn == 4 ); + CV_Assert( cn == 4 ); for( k = 0; k < m*4; k += 4 ) { UPDATE_ACC01( src_top[k], 0, -- ); diff --git a/modules/imgproc/src/moments.cpp b/modules/imgproc/src/moments.cpp index 9e7e6d2dfd..204c8654af 100644 --- a/modules/imgproc/src/moments.cpp +++ b/modules/imgproc/src/moments.cpp @@ -52,7 +52,7 @@ static void completeMomentState( Moments* moments ) double cx = 0, cy = 0; double mu20, mu11, mu02; double inv_m00 = 0.0; - assert( moments != 0 ); + CV_Assert( moments != 0 ); if( fabs(moments->m00) > DBL_EPSILON ) { diff --git a/modules/imgproc/src/precomp.hpp b/modules/imgproc/src/precomp.hpp index 400b7cc2db..b300192e9c 100644 --- a/modules/imgproc/src/precomp.hpp +++ b/modules/imgproc/src/precomp.hpp @@ -54,7 +54,6 @@ #include "hal_replacement.hpp" #include -#include #include #include #include diff --git a/modules/imgproc/src/resize.cpp b/modules/imgproc/src/resize.cpp index 4f82bddfa0..4f1a4576ce 100644 --- a/modules/imgproc/src/resize.cpp +++ b/modules/imgproc/src/resize.cpp @@ -3191,7 +3191,7 @@ static int computeResizeAreaTab( int ssize, int dsize, int cn, double scale, Dec if( sx1 - fsx1 > 1e-3 ) { - assert( k < ssize*2 ); + CV_Assert( k < ssize*2 ); tab[k].di = dx * cn; tab[k].si = (sx1 - 1) * cn; tab[k++].alpha = (float)((sx1 - fsx1) / cellWidth); @@ -3199,7 +3199,7 @@ static int computeResizeAreaTab( int ssize, int dsize, int cn, double scale, Dec for(int sx = sx1; sx < sx2; sx++ ) { - assert( k < ssize*2 ); + CV_Assert( k < ssize*2 ); tab[k].di = dx * cn; tab[k].si = sx * cn; tab[k++].alpha = float(1.0 / cellWidth); @@ -3207,7 +3207,7 @@ static int computeResizeAreaTab( int ssize, int dsize, int cn, double scale, Dec if( fsx2 - sx2 > 1e-3 ) { - assert( k < ssize*2 ); + CV_Assert( k < ssize*2 ); tab[k].di = dx * cn; tab[k].si = sx2 * cn; tab[k++].alpha = (float)(std::min(std::min(fsx2 - sx2, 1.), cellWidth) / cellWidth); @@ -3899,7 +3899,7 @@ void resize(int src_type, { if( k == 0 || ytab[k].di != ytab[k-1].di ) { - assert( ytab[k].di == dy ); + CV_Assert( ytab[k].di == dy ); tabofs[dy++] = k; } } diff --git a/modules/imgproc/src/samplers.cpp b/modules/imgproc/src/samplers.cpp index a0b2aba223..287e78c6df 100644 --- a/modules/imgproc/src/samplers.cpp +++ b/modules/imgproc/src/samplers.cpp @@ -74,7 +74,7 @@ adjustRect( const uchar* src, size_t src_step, int pix_size, src += rect.width*pix_size; rect.width = 0; } - assert( rect.width <= win_size.width ); + CV_Assert( rect.width <= win_size.width ); } if( ip.y >= 0 ) diff --git a/modules/imgproc/src/segmentation.cpp b/modules/imgproc/src/segmentation.cpp index c78931221f..79cb641e80 100644 --- a/modules/imgproc/src/segmentation.cpp +++ b/modules/imgproc/src/segmentation.cpp @@ -155,7 +155,7 @@ void cv::watershed( InputArray _src, InputOutputArray _markers ) dr = std::abs((ptr1)[2] - (ptr2)[2]);\ diff = ws_max(db,dg); \ diff = ws_max(diff,dr); \ - assert( 0 <= diff && diff <= 255 ); \ + CV_Assert( 0 <= diff && diff <= 255 ); \ } CV_Assert( src.type() == CV_8UC3 && dst.type() == CV_32SC1 ); @@ -215,7 +215,7 @@ void cv::watershed( InputArray _src, InputOutputArray _markers ) } // Add to according queue - assert( 0 <= idx && idx <= 255 ); + CV_Assert( 0 <= idx && idx <= 255 ); ws_push( idx, i*mstep + j, i*istep + j*3 ); m[0] = IN_QUEUE; } @@ -286,7 +286,7 @@ void cv::watershed( InputArray _src, InputOutputArray _markers ) } // Set label to current pixel in marker image - assert( lab != 0 ); + CV_Assert( lab != 0 ); m[0] = lab; if( lab == WSHED ) diff --git a/modules/imgproc/src/subdivision2d.cpp b/modules/imgproc/src/subdivision2d.cpp index c254c5f9c6..980a03b7db 100644 --- a/modules/imgproc/src/subdivision2d.cpp +++ b/modules/imgproc/src/subdivision2d.cpp @@ -436,7 +436,7 @@ int Subdiv2D::insert(Point2f pt) else CV_Error_(CV_StsError, ("Subdiv2D::locate returned invalid location = %d", location) ); - assert( curr_edge != 0 ); + CV_Assert( curr_edge != 0 ); validGeometry = false; curr_point = newPoint(pt, false); diff --git a/modules/imgproc/test/test_approxpoly.cpp b/modules/imgproc/test/test_approxpoly.cpp index 81a9772bb1..80ab408f0a 100644 --- a/modules/imgproc/test/test_approxpoly.cpp +++ b/modules/imgproc/test/test_approxpoly.cpp @@ -182,7 +182,7 @@ int CV_ApproxPolyTest::check_slice( CvPoint StartPt, CvPoint EndPt, //////////////////////////////// if( SrcReader == NULL ) { - assert( false ); + CV_Assert( false ); return 0; } @@ -237,7 +237,7 @@ int CV_ApproxPolyTest::check( CvSeq* SrcSeq, CvSeq* DstSeq, float Eps ) int Count; int i,j; - assert( SrcSeq && DstSeq ); + CV_Assert( SrcSeq && DstSeq ); ////////// init //////////////////// Count = SrcSeq->total; diff --git a/modules/imgproc/test/test_bilateral_filter.cpp b/modules/imgproc/test/test_bilateral_filter.cpp index f5e8caab80..badd879a8a 100644 --- a/modules/imgproc/test/test_bilateral_filter.cpp +++ b/modules/imgproc/test/test_bilateral_filter.cpp @@ -203,7 +203,7 @@ namespace opencv_test { namespace { } else { - assert( cn == 3 ); + CV_Assert( cn == 3 ); for( j = 0; j < size.width*3; j += 3 ) { float sum_b = 0, sum_g = 0, sum_r = 0, wsum = 0; diff --git a/modules/imgproc/test/test_canny.cpp b/modules/imgproc/test/test_canny.cpp index 9d9b7c397e..e040da61ad 100644 --- a/modules/imgproc/test/test_canny.cpp +++ b/modules/imgproc/test/test_canny.cpp @@ -249,7 +249,7 @@ test_Canny( const Mat& src, Mat& dst, } else { - assert( fabs(tg) > tan_3pi_8 ); + CV_Assert( fabs(tg) > tan_3pi_8 ); x1 = x2 = x; y1 = y + 1; y2 = y - 1; } diff --git a/modules/imgproc/test/test_color.cpp b/modules/imgproc/test/test_color.cpp index 204203d053..3cd57c9d38 100644 --- a/modules/imgproc/test/test_color.cpp +++ b/modules/imgproc/test/test_color.cpp @@ -217,7 +217,7 @@ void CV_ColorCvtBaseTest::convert_forward( const Mat& src, Mat& dst ) float* dst_buf = &_dst_buf[0]; int i, j; - assert( (cn == 3 || cn == 4) && (dst_cn == 3 || dst_cn == 1) ); + CV_Assert( (cn == 3 || cn == 4) && (dst_cn == 3 || dst_cn == 1) ); for( i = 0; i < src.rows; i++ ) { @@ -281,7 +281,7 @@ void CV_ColorCvtBaseTest::convert_forward( const Mat& src, Mat& dst ) } break; default: - assert(0); + CV_Assert(0); } } } @@ -312,7 +312,7 @@ void CV_ColorCvtBaseTest::convert_backward( const Mat& src, const Mat& dst, Mat& float* dst_buf = &_dst_buf[0]; int i, j; - assert( cn == 3 || cn == 4 ); + CV_Assert( cn == 3 || cn == 4 ); for( i = 0; i < src.rows; i++ ) { @@ -385,7 +385,7 @@ void CV_ColorCvtBaseTest::convert_backward( const Mat& src, const Mat& dst, Mat& } break; default: - assert(0); + CV_Assert(0); } } } @@ -1571,7 +1571,7 @@ void CV_ColorRGBTest::convert_forward( const Mat& src, Mat& dst ) } break; default: - assert(0); + CV_Assert(0); } } } @@ -1677,7 +1677,7 @@ void CV_ColorRGBTest::convert_backward( const Mat& /*src*/, const Mat& src, Mat& } break; default: - assert(0); + CV_Assert(0); } } } diff --git a/modules/imgproc/test/test_contours.cpp b/modules/imgproc/test/test_contours.cpp index c07d19098b..224a2e1f1a 100644 --- a/modules/imgproc/test/test_contours.cpp +++ b/modules/imgproc/test/test_contours.cpp @@ -208,7 +208,7 @@ cvTsMarkContours( IplImage* img, int val ) int i, j; int step = img->widthStep; - assert( img->depth == IPL_DEPTH_8U && img->nChannels == 1 && (val&1) != 0); + CV_Assert( img->depth == IPL_DEPTH_8U && img->nChannels == 1 && (val&1) != 0); for( i = 1; i < img->height - 1; i++ ) for( j = 1; j < img->width - 1; j++ ) diff --git a/modules/imgproc/test/test_convhull.cpp b/modules/imgproc/test/test_convhull.cpp index bc5c940827..aa553e5efe 100644 --- a/modules/imgproc/test/test_convhull.cpp +++ b/modules/imgproc/test/test_convhull.cpp @@ -301,7 +301,7 @@ void CV_BaseShapeDescrTest::generate_point_set( void* pointsSet ) else { CvMat* ptm = (CvMat*)pointsSet; - assert( CV_IS_MAT(ptm) && CV_IS_MAT_CONT(ptm->type) ); + CV_Assert( CV_IS_MAT(ptm) && CV_IS_MAT_CONT(ptm->type) ); total = ptm->rows + ptm->cols - 1; point_type = CV_MAT_TYPE(ptm->type); data = ptm->data.ptr; @@ -310,7 +310,7 @@ void CV_BaseShapeDescrTest::generate_point_set( void* pointsSet ) n = CV_MAT_CN(point_type); point_type = CV_MAT_DEPTH(point_type); - assert( (point_type == CV_32S || point_type == CV_32F) && n <= 4 ); + CV_Assert( (point_type == CV_32S || point_type == CV_32F) && n <= 4 ); for( i = 0; i < total; i++ ) { @@ -1335,7 +1335,7 @@ void CV_FitEllipseTest::generate_point_set( void* pointsSet ) else { CvMat* ptm = (CvMat*)pointsSet; - assert( CV_IS_MAT(ptm) && CV_IS_MAT_CONT(ptm->type) ); + CV_Assert( CV_IS_MAT(ptm) && CV_IS_MAT_CONT(ptm->type) ); total = ptm->rows + ptm->cols - 1; point_type = CV_MAT_TYPE(ptm->type); data = ptm->data.ptr; @@ -1621,7 +1621,7 @@ void CV_FitLineTest::generate_point_set( void* pointsSet ) else { CvMat* ptm = (CvMat*)pointsSet; - assert( CV_IS_MAT(ptm) && CV_IS_MAT_CONT(ptm->type) ); + CV_Assert( CV_IS_MAT(ptm) && CV_IS_MAT_CONT(ptm->type) ); total = ptm->rows + ptm->cols - 1; point_type = CV_MAT_DEPTH(CV_MAT_TYPE(ptm->type)); data = ptm->data.ptr; @@ -1788,13 +1788,13 @@ cvTsGenerateTousledBlob( CvPoint2D32f center, CvSize2D32f axes, else { CvMat* ptm = (CvMat*)points; - assert( CV_IS_MAT(ptm) && CV_IS_MAT_CONT(ptm->type) ); + CV_Assert( CV_IS_MAT(ptm) && CV_IS_MAT_CONT(ptm->type) ); total = ptm->rows + ptm->cols - 1; point_type = CV_MAT_TYPE(ptm->type); data = ptm->data.ptr; } - assert( point_type == CV_32SC2 || point_type == CV_32FC2 ); + CV_Assert( point_type == CV_32SC2 || point_type == CV_32FC2 ); for( i = 0; i < total; i++ ) { @@ -1874,8 +1874,8 @@ void CV_ContourMomentsTest::generate_point_set( void* pointsSet ) center.x = (float)(img_size.width*0.5 + (cvtest::randReal(rng)-0.5)*(img_size.width - max_sz*2)*0.8); center.y = (float)(img_size.height*0.5 + (cvtest::randReal(rng)-0.5)*(img_size.height - max_sz*2)*0.8); - assert( 0 < center.x - max_sz && center.x + max_sz < img_size.width && - 0 < center.y - max_sz && center.y + max_sz < img_size.height ); + CV_Assert( 0 < center.x - max_sz && center.x + max_sz < img_size.width && + 0 < center.y - max_sz && center.y + max_sz < img_size.height ); max_r_scale = cvtest::randReal(rng)*max_max_r_scale*0.01; angle = cvtest::randReal(rng)*360; diff --git a/modules/imgproc/test/test_distancetransform.cpp b/modules/imgproc/test/test_distancetransform.cpp index 652b5bfd24..32fc7aa2c3 100644 --- a/modules/imgproc/test/test_distancetransform.cpp +++ b/modules/imgproc/test/test_distancetransform.cpp @@ -161,7 +161,7 @@ cvTsDistTransform( const CvMat* _src, CvMat* _dst, int dist_type, float delta[16]; int tstep, count; - assert( mask_size == 3 || mask_size == 5 ); + CV_Assert( mask_size == 3 || mask_size == 5 ); if( dist_type == CV_DIST_USER ) memcpy( mask, _mask, sizeof(mask) ); diff --git a/modules/imgproc/test/test_filter.cpp b/modules/imgproc/test/test_filter.cpp index 11d87a0abe..3e28a2caad 100644 --- a/modules/imgproc/test/test_filter.cpp +++ b/modules/imgproc/test/test_filter.cpp @@ -992,8 +992,8 @@ static void test_medianFilter( const Mat& src, Mat& dst, int m ) median_pair *buf0 = &_buf0[0], *buf1 = &_buf1[0]; int step = (int)(src.step/src.elemSize()); - assert( src.rows == dst.rows + m - 1 && src.cols == dst.cols + m - 1 && - src.type() == dst.type() && src.type() == CV_8UC1 ); + CV_Assert( src.rows == dst.rows + m - 1 && src.cols == dst.cols + m - 1 && + src.type() == dst.type() && src.type() == CV_8UC1 ); for( i = 0; i < dst.rows; i++ ) { @@ -1050,7 +1050,7 @@ static void test_medianFilter( const Mat& src, Mat& dst, int m ) *buf1++ = buf0[k++]; else { - assert( col_buf[l] < INT_MAX ); + CV_Assert( col_buf[l] < INT_MAX ); *buf1++ = median_pair(ins_col,col_buf[l++]); } } @@ -1061,7 +1061,7 @@ static void test_medianFilter( const Mat& src, Mat& dst, int m ) if( del_col < 0 ) n += m; buf1 -= n; - assert( n == m2 ); + CV_Assert( n == m2 ); dst1[j] = (uchar)buf1[n/2].val; median_pair* tbuf; CV_SWAP( buf0, buf1, tbuf ); diff --git a/modules/imgproc/test/test_imgwarp.cpp b/modules/imgproc/test/test_imgwarp.cpp index 7d0360dfb1..1257a472b7 100644 --- a/modules/imgproc/test/test_imgwarp.cpp +++ b/modules/imgproc/test/test_imgwarp.cpp @@ -169,7 +169,7 @@ int CV_ImgWarpBaseTest::prepare_test_case( int test_case_idx ) } break; default: - assert(0); + CV_Assert(0); } /*switch( depth ) @@ -482,7 +482,7 @@ static void test_remap( const Mat& src, Mat& dst, const Mat& mapx, const Mat& ma } break; default: - assert(0); + CV_Assert(0); } } } diff --git a/modules/imgproc/test/test_imgwarp_strict.cpp b/modules/imgproc/test/test_imgwarp_strict.cpp index cadc303cb3..5ca3d09ec4 100644 --- a/modules/imgproc/test/test_imgwarp_strict.cpp +++ b/modules/imgproc/test/test_imgwarp_strict.cpp @@ -935,7 +935,7 @@ void CV_Remap_Test::remap_generic(const Mat& _src, Mat& _dst) else if (interpolation == INTER_LANCZOS4) ksize = 8; else if (interpolation != INTER_LINEAR) - assert(0); + CV_Assert(0); int ofs = (ksize / 2) - 1; CV_Assert(_src.depth() == CV_32F && _dst.type() == _src.type()); diff --git a/modules/imgproc/test/test_templmatch.cpp b/modules/imgproc/test/test_templmatch.cpp index 035be685a9..858d7a35fe 100644 --- a/modules/imgproc/test/test_templmatch.cpp +++ b/modules/imgproc/test/test_templmatch.cpp @@ -185,7 +185,7 @@ static void cvTsMatchTemplate( const CvMat* img, const CvMat* templ, CvMat* resu b_denom = 1.; } - assert( CV_TM_SQDIFF <= method && method <= CV_TM_CCOEFF_NORMED ); + CV_Assert( CV_TM_SQDIFF <= method && method <= CV_TM_CCOEFF_NORMED ); for( i = 0; i < result->rows; i++ ) { diff --git a/modules/imgproc/test/test_thresh.cpp b/modules/imgproc/test/test_thresh.cpp index a61095d5cc..a8e961675d 100644 --- a/modules/imgproc/test/test_thresh.cpp +++ b/modules/imgproc/test/test_thresh.cpp @@ -157,7 +157,7 @@ static void test_threshold( const Mat& _src, Mat& _dst, imaxval = cvRound(maxval); } - assert( depth == CV_8U || depth == CV_16S || depth == CV_16U || depth == CV_32F || depth == CV_64F ); + CV_Assert( depth == CV_8U || depth == CV_16S || depth == CV_16U || depth == CV_32F || depth == CV_64F ); switch( thresh_type ) { @@ -407,7 +407,7 @@ static void test_threshold( const Mat& _src, Mat& _dst, } break; default: - assert(0); + CV_Assert(0); } } diff --git a/modules/ml/src/precomp.hpp b/modules/ml/src/precomp.hpp index 6d50357b7a..328cc4732a 100644 --- a/modules/ml/src/precomp.hpp +++ b/modules/ml/src/precomp.hpp @@ -48,7 +48,6 @@ #include "opencv2/core/private.hpp" -#include #include #include #include diff --git a/modules/ml/src/tree.cpp b/modules/ml/src/tree.cpp index 5dae889013..b69ddaece2 100644 --- a/modules/ml/src/tree.cpp +++ b/modules/ml/src/tree.cpp @@ -869,7 +869,7 @@ DTreesImpl::WSplit DTreesImpl::findSplitCatClass( int vi, const vector& _si } else { - assert( m == 2 ); + CV_Assert( m == 2 ); dbl_ptr = (double**)(c_weights + _mi); for( j = 0; j < mi; j++ ) dbl_ptr[j] = cjk + j*2 + 1; diff --git a/modules/objdetect/src/cascadedetect.cpp b/modules/objdetect/src/cascadedetect.cpp index bd62cd21a1..c2d2221fda 100644 --- a/modules/objdetect/src/cascadedetect.cpp +++ b/modules/objdetect/src/cascadedetect.cpp @@ -960,10 +960,10 @@ int CascadeClassifierImpl::runAt( Ptr& evaluator, Point pt, in { CV_INSTRUMENT_REGION(); - assert( !oldCascade && - (data.featureType == FeatureEvaluator::HAAR || - data.featureType == FeatureEvaluator::LBP || - data.featureType == FeatureEvaluator::HOG) ); + CV_Assert( !oldCascade && + (data.featureType == FeatureEvaluator::HAAR || + data.featureType == FeatureEvaluator::LBP || + data.featureType == FeatureEvaluator::HOG) ); if( !evaluator->setWindow(pt, scaleIdx) ) return -1; diff --git a/modules/objdetect/src/detection_based_tracker.cpp b/modules/objdetect/src/detection_based_tracker.cpp index 5a6ccce328..14e0fe6617 100644 --- a/modules/objdetect/src/detection_based_tracker.cpp +++ b/modules/objdetect/src/detection_based_tracker.cpp @@ -42,7 +42,6 @@ //M*/ #include "precomp.hpp" -#include #ifdef CV_CXX11 #define USE_STD_THREADS diff --git a/modules/objdetect/src/haar.cpp b/modules/objdetect/src/haar.cpp index 9e20111f43..ee485f914f 100644 --- a/modules/objdetect/src/haar.cpp +++ b/modules/objdetect/src/haar.cpp @@ -291,7 +291,7 @@ icvCreateHidHaarClassifierCascade( CvHaarClassifierCascade* cascade ) } cascade->hid_cascade = out; - assert( (char*)haar_node_ptr - (char*)out <= datasize ); + CV_Assert( (char*)haar_node_ptr - (char*)out <= datasize ); return out; } @@ -622,7 +622,7 @@ cvRunHaarClassifierCascadeSum( const CvHaarClassifierCascade* _cascade, if( cascade->is_tree ) { CvHidHaarStageClassifier* ptr = cascade->stage_classifier; - assert( start_stage == 0 ); + CV_Assert( start_stage == 0 ); while( ptr ) { diff --git a/modules/objdetect/src/hog.cpp b/modules/objdetect/src/hog.cpp index 8a2107734a..1ff2191dd0 100644 --- a/modules/objdetect/src/hog.cpp +++ b/modules/objdetect/src/hog.cpp @@ -852,7 +852,7 @@ void HOGCache::init(const HOGDescriptor* _descriptor, data->gradWeight = weights(i,j); } - assert( count1 + count2 + count4 == rawBlockSize ); + CV_Assert( count1 + count2 + count4 == rawBlockSize ); // defragment pixData for( j = 0; j < count2; j++ ) pixData[j + count1] = pixData[j + rawBlockSize]; @@ -874,7 +874,7 @@ void HOGCache::init(const HOGDescriptor* _descriptor, const float* HOGCache::getBlock(Point pt, float* buf) { float* blockHist = buf; - assert(descriptor != 0); + CV_Assert(descriptor != 0); // Size blockSize = descriptor->blockSize; pt += imgoffset; diff --git a/modules/objdetect/test/test_cascadeandhog.cpp b/modules/objdetect/test/test_cascadeandhog.cpp index 83100dd796..e6bf9a23ca 100644 --- a/modules/objdetect/test/test_cascadeandhog.cpp +++ b/modules/objdetect/test/test_cascadeandhog.cpp @@ -191,7 +191,7 @@ void CV_DetectorTest::run( int ) // write detectors validationFS << DETECTORS << "{"; - assert( detectorNames.size() == detectorFilenames.size() ); + CV_Assert( detectorNames.size() == detectorFilenames.size() ); nit = detectorNames.begin(); for( int di = 0; nit != detectorNames.end(); ++nit, di++ ) { @@ -291,7 +291,7 @@ static bool isZero( uchar i ) {return i == 0;} int CV_DetectorTest::validate( int detectorIdx, vector >& objects ) { - assert( imageFilenames.size() == objects.size() ); + CV_Assert( imageFilenames.size() == objects.size() ); int imageIdx = 0; int totalNoPair = 0, totalValRectCount = 0; @@ -504,7 +504,7 @@ int CV_HOGDetectorTest::detectMultiScale( int di, const Mat& img, if( detectorFilenames[di].empty() ) hog.setSVMDetector(HOGDescriptor::getDefaultPeopleDetector()); else - assert(0); + CV_Assert(0); hog.detectMultiScale(img, objects); return cvtest::TS::OK; } @@ -787,7 +787,7 @@ void HOGCacheTester::init(const HOGDescriptorTester* _descriptor, data->gradWeight = weights(i,j); } - assert( count1 + count2 + count4 == rawBlockSize ); + CV_Assert( count1 + count2 + count4 == rawBlockSize ); // defragment pixData for( j = 0; j < count2; j++ ) pixData[j + count1] = pixData[j + rawBlockSize]; @@ -809,7 +809,7 @@ void HOGCacheTester::init(const HOGDescriptorTester* _descriptor, const float* HOGCacheTester::getBlock(Point pt, float* buf) { float* blockHist = buf; - assert(descriptor != 0); + CV_Assert(descriptor != 0); Size blockSize = descriptor->blockSize; pt += imgoffset; @@ -1285,7 +1285,7 @@ void HOGDescriptorTester::computeGradient(const Mat& img, Mat& grad, Mat& qangle hidx += _nbins; else if( hidx >= _nbins ) hidx -= _nbins; - assert( (unsigned)hidx < (unsigned)_nbins ); + CV_Assert( (unsigned)hidx < (unsigned)_nbins ); qanglePtr[x*2] = (uchar)hidx; hidx++; diff --git a/modules/ts/src/ocl_perf.cpp b/modules/ts/src/ocl_perf.cpp index 8dacf219f6..90c2ece66a 100644 --- a/modules/ts/src/ocl_perf.cpp +++ b/modules/ts/src/ocl_perf.cpp @@ -50,7 +50,7 @@ namespace perf { void checkDeviceMaxMemoryAllocSize(const Size& size, int type, int factor) { - assert(factor > 0); + CV_Assert(factor > 0); if (!cv::ocl::useOpenCL()) return; diff --git a/modules/ts/src/ts.cpp b/modules/ts/src/ts.cpp index dfd20fbc36..aed205fbf6 100644 --- a/modules/ts/src/ts.cpp +++ b/modules/ts/src/ts.cpp @@ -377,7 +377,7 @@ void BaseTest::run( int start_from ) void BaseTest::run_func(void) { - assert(0); + CV_Assert(0); } diff --git a/modules/ts/src/ts_arrtest.cpp b/modules/ts/src/ts_arrtest.cpp index 365cf1550e..1c74fb4a33 100644 --- a/modules/ts/src/ts_arrtest.cpp +++ b/modules/ts/src/ts_arrtest.cpp @@ -268,14 +268,14 @@ void ArrayTest::fill_array( int /*test_case_idx*/, int i, int j, Mat& arr ) double ArrayTest::get_success_error_level( int /*test_case_idx*/, int i, int j ) { int elem_depth = CV_MAT_DEPTH(cvGetElemType(test_array[i][j])); - assert( i == OUTPUT || i == INPUT_OUTPUT ); + CV_Assert( i == OUTPUT || i == INPUT_OUTPUT ); return elem_depth < CV_32F ? 0 : elem_depth == CV_32F ? FLT_EPSILON*100: DBL_EPSILON*5000; } void ArrayTest::prepare_to_validation( int /*test_case_idx*/ ) { - assert(0); + CV_Assert(0); } @@ -293,7 +293,7 @@ int ArrayTest::validate_test_results( int test_case_idx ) int i1 = i == 0 ? REF_OUTPUT : REF_INPUT_OUTPUT; size_t sizei = test_array[i0].size(); - assert( sizei == test_array[i1].size() ); + CV_Assert( sizei == test_array[i1].size() ); for( j = 0; j < sizei; j++ ) { double err_level; diff --git a/modules/ts/src/ts_func.cpp b/modules/ts/src/ts_func.cpp index 6f1c389307..4b7dcbd234 100644 --- a/modules/ts/src/ts_func.cpp +++ b/modules/ts/src/ts_func.cpp @@ -2094,7 +2094,7 @@ int cmpEps( const Mat& arr, const Mat& refarr, double* _realmaxdiff, } break; default: - assert(0); + CV_Assert(0); return CMP_EPS_BIG_DIFF; } if(_realmaxdiff) @@ -2705,7 +2705,7 @@ static void calcSobelKernel1D( int order, int _aperture_size, int size, vector= cv::getTickFrequency() * 10) { std::cout << '.' << std::endl; @@ -1638,7 +1638,7 @@ performance_metrics& TestBase::calcMetrics() } else { - assert(false); + CV_Assert(false); } int offset = static_cast(start - times.begin()); @@ -1714,7 +1714,7 @@ void TestBase::validateMetrics() } else { - assert(false); + CV_Assert(false); } } diff --git a/modules/videoio/src/cap_cmu.cpp b/modules/videoio/src/cap_cmu.cpp index a9a499b4cd..72712cec0a 100644 --- a/modules/videoio/src/cap_cmu.cpp +++ b/modules/videoio/src/cap_cmu.cpp @@ -212,7 +212,7 @@ int CvCaptureCAM_CMU::getDepth() // TODO if( format==7 ) { - assert(0); + CV_Assert(0); return 1; } // irrelvant to depth @@ -233,7 +233,7 @@ int CvCaptureCAM_CMU::getNChannels() int mode = cmucam->GetVideoMode(); if( format==7 ){ - assert(0); + CV_Assert(0); return 1; } diff --git a/modules/videoio/src/cap_dc1394.cpp b/modules/videoio/src/cap_dc1394.cpp index 368bf3aa38..7f6c694ce4 100644 --- a/modules/videoio/src/cap_dc1394.cpp +++ b/modules/videoio/src/cap_dc1394.cpp @@ -272,7 +272,7 @@ static CvCaptureCAM_DC1394 * icvCaptureFromCAM_DC1394 (int index) format_idx = preferred_modes[i] - FORMAT_MIN; continue; } - assert(format_idx != -1); + CV_Assert(format_idx != -1); if ( ! icvFormatSupportedCAM_DC1394(pcap->format, formats) ) continue; if ( icvModeSupportedCAM_DC1394(pcap->format, preferred_modes[i], modes[format_idx]) ){ diff --git a/modules/videoio/src/cap_dc1394_v2.cpp b/modules/videoio/src/cap_dc1394_v2.cpp index 938b31f4a9..3e771a0476 100644 --- a/modules/videoio/src/cap_dc1394_v2.cpp +++ b/modules/videoio/src/cap_dc1394_v2.cpp @@ -64,7 +64,6 @@ static uint32_t getControlRegister(dc1394camera_t *camera, uint64_t offset) uint32_t value = 0; dc1394error_t err = dc1394_get_control_register(camera, offset, &value); - assert(err == DC1394_SUCCESS); return err == DC1394_SUCCESS ? value : 0xffffffff; } diff --git a/modules/videoio/src/cap_ffmpeg_impl.hpp b/modules/videoio/src/cap_ffmpeg_impl.hpp index 6877a963ef..d5be759f43 100644 --- a/modules/videoio/src/cap_ffmpeg_impl.hpp +++ b/modules/videoio/src/cap_ffmpeg_impl.hpp @@ -44,7 +44,6 @@ #if !(defined(_WIN32) || defined(WINCE)) # include #endif -#include #include #include @@ -405,7 +404,7 @@ static inline int _opencv_ffmpeg_interrupt_callback(void *ptr) { AVInterruptCallbackMetadata* metadata = (AVInterruptCallbackMetadata*)ptr; - assert(metadata); + CV_Assert(metadata); if (metadata->timeout_after_ms == 0) { @@ -2113,7 +2112,7 @@ bool CvVideoWriter_FFMPEG::writeFrame( const unsigned char* data, int step, int } } else { - assert(false); + CV_Assert(false); } if( (width & -2) != frame_width || (height & -2) != frame_height || !data ) @@ -2165,7 +2164,7 @@ bool CvVideoWriter_FFMPEG::writeFrame( const unsigned char* data, int step, int } if ( c->pix_fmt != input_pix_fmt ) { - assert( input_picture ); + CV_Assert( input_picture ); // let input_picture point to the raw data buffer of 'image' _opencv_ffmpeg_av_image_fill_arrays(input_picture, (uint8_t *) data, (AVPixelFormat)input_pix_fmt, width, height); @@ -2441,7 +2440,7 @@ bool CvVideoWriter_FFMPEG::open( const char * filename, int fourcc, #else oc = av_alloc_format_context(); #endif - assert (oc); + CV_Assert (oc); /* set file name */ oc->oformat = fmt; diff --git a/modules/videoio/src/cap_giganetix.cpp b/modules/videoio/src/cap_giganetix.cpp index 98fba269b0..aa86c1cb3a 100644 --- a/modules/videoio/src/cap_giganetix.cpp +++ b/modules/videoio/src/cap_giganetix.cpp @@ -441,7 +441,7 @@ CvCaptureCAM_Giganetix::grabImage () { gige::IImageInfo imageInfo; m_device->GetImageInfo (&imageInfo); - assert(imageInfo.IsValid()); + CV_Assert(imageInfo.IsValid()); if (m_device->GetPendingImagesCount() == 1) { diff --git a/modules/videoio/src/cap_libv4l.cpp b/modules/videoio/src/cap_libv4l.cpp index ba2b7985a3..6fd0f757a8 100644 --- a/modules/videoio/src/cap_libv4l.cpp +++ b/modules/videoio/src/cap_libv4l.cpp @@ -249,7 +249,6 @@ make & enjoy! #include #include #include /* for videodev2.h */ -#include #include #include @@ -1125,7 +1124,7 @@ static int read_frame_v4l2(CvCaptureCAM_V4L* capture) { } } - assert(buf.index < capture->req.count); + CV_Assert(buf.index < capture->req.count); #ifdef USE_TEMP_BUFFER memcpy(capture->buffers[MAX_V4L_BUFFERS].start, diff --git a/modules/videoio/src/cap_qt.cpp b/modules/videoio/src/cap_qt.cpp index 8acc5d5ab6..e2cd145d4d 100644 --- a/modules/videoio/src/cap_qt.cpp +++ b/modules/videoio/src/cap_qt.cpp @@ -55,7 +55,6 @@ // standard includes #include -#include // Mac OS includes #include @@ -608,14 +607,14 @@ static CvCapture_QT_Cam * icvCaptureFromCam_QT (const int index) /// capture properties currently unimplemented for QuickTime camera interface static double icvGetProperty_QT_Cam (CvCapture_QT_Cam * capture, int property_id) { - assert (0); + CV_Assert (0); return 0; } /// capture properties currently unimplemented for QuickTime camera interface static int icvSetProperty_QT_Cam (CvCapture_QT_Cam * capture, int property_id, double value) { - assert (0); + CV_Assert (0); return 0; } diff --git a/modules/videoio/src/cap_v4l.cpp b/modules/videoio/src/cap_v4l.cpp index 2740c62e29..a4694b4b48 100644 --- a/modules/videoio/src/cap_v4l.cpp +++ b/modules/videoio/src/cap_v4l.cpp @@ -220,7 +220,6 @@ make & enjoy! #include #include -#include #include #include #include @@ -929,8 +928,8 @@ bool CvCaptureCAM_V4L::read_frame_v4l2() return false; } - assert(buf.index < req.count); - assert(buffers[buf.index].length == buf.length); + CV_Assert(buf.index < req.count); + CV_Assert(buffers[buf.index].length == buf.length); //We shouldn't use this buffer in the queue while not retrieve frame from it. buffers[buf.index].buffer = buf; diff --git a/modules/videoio/src/precomp.hpp b/modules/videoio/src/precomp.hpp index 800d471362..77fb29e849 100644 --- a/modules/videoio/src/precomp.hpp +++ b/modules/videoio/src/precomp.hpp @@ -68,7 +68,6 @@ #include #include #include -#include // FIXIT remove this #if defined _WIN32 || defined WINCE #if !defined _WIN32_WINNT From 58dc39793095c82676d1a1ff0c6c1e6fb1146646 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 27 Nov 2021 02:51:57 +0000 Subject: [PATCH 103/226] dnn(test): add two_inputs test with FP32/U8 data types - remove similar test from IE scope under HAVE_INF_ENGINE --- modules/dnn/test/test_layers.cpp | 51 --------------------------- modules/dnn/test/test_misc.cpp | 60 ++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 51 deletions(-) diff --git a/modules/dnn/test/test_layers.cpp b/modules/dnn/test/test_layers.cpp index 836b0aab9a..e61fd733f9 100644 --- a/modules/dnn/test/test_layers.cpp +++ b/modules/dnn/test/test_layers.cpp @@ -1380,57 +1380,6 @@ INSTANTIATE_TEST_CASE_P(/*nothing*/, Test_DLDT_two_inputs_3dim, Combine( testing::ValuesIn(list_sizes) )); -typedef testing::TestWithParam > > Test_DLDT_two_inputs; -TEST_P(Test_DLDT_two_inputs, as_backend) -{ - static const float kScale = 0.5f; - static const float kScaleInv = 1.0f / kScale; - - Backend backendId = get<0>(get<2>(GetParam())); - Target targetId = get<1>(get<2>(GetParam())); - - Net net; - LayerParams lp; - lp.type = "Eltwise"; - lp.name = "testLayer"; - lp.set("operation", "sum"); - int eltwiseId = net.addLayerToPrev(lp.name, lp.type, lp); // connect to a first input - net.connect(0, 1, eltwiseId, 1); // connect to a second input - - int inpSize[] = {1, 2, 3, 4}; - Mat firstInp(4, &inpSize[0], get<0>(GetParam())); - Mat secondInp(4, &inpSize[0], get<1>(GetParam())); - randu(firstInp, 0, 255); - randu(secondInp, 0, 255); - - net.setInputsNames({"data", "second_input"}); - net.setInput(firstInp, "data", kScale); - net.setInput(secondInp, "second_input", kScaleInv); - net.setPreferableBackend(backendId); - net.setPreferableTarget(targetId); - Mat out = net.forward(); - - Mat ref; - addWeighted(firstInp, kScale, secondInp, kScaleInv, 0, ref, CV_32F); - // Output values are in range [0, 637.5]. - double l1 = (targetId == DNN_TARGET_OPENCL_FP16 || targetId == DNN_TARGET_MYRIAD) ? 0.06 : 1e-6; - double lInf = (targetId == DNN_TARGET_OPENCL_FP16 || targetId == DNN_TARGET_MYRIAD) ? 0.3 : 1e-5; - normAssert(out, ref, "", l1, lInf); - if (cvtest::debugLevel > 0 || HasFailure()) - { - std::cout << "input1 scale=" << kScale << " input2 scale=" << kScaleInv << std::endl; - std::cout << "input1: " << firstInp.size << " " << firstInp.reshape(1, 1) << std::endl; - std::cout << "input2: " << secondInp.size << " " << secondInp.reshape(1, 1) << std::endl; - std::cout << "ref: " << ref.reshape(1, 1) << std::endl; - std::cout << "out: " << out.reshape(1, 1) << std::endl; - } -} - -INSTANTIATE_TEST_CASE_P(/*nothing*/, Test_DLDT_two_inputs, Combine( - Values(CV_8U, CV_32F), Values(CV_8U, CV_32F), - dnnBackendsAndTargets() -)); - class UnsupportedLayer : public Layer { public: diff --git a/modules/dnn/test/test_misc.cpp b/modules/dnn/test/test_misc.cpp index 11e0f0ec2d..9971450478 100644 --- a/modules/dnn/test/test_misc.cpp +++ b/modules/dnn/test/test_misc.cpp @@ -828,4 +828,64 @@ INSTANTIATE_TEST_CASE_P(/**/, Test_Model_Optimizer, #endif // HAVE_INF_ENGINE +typedef testing::TestWithParam > > Test_two_inputs; +TEST_P(Test_two_inputs, basic) +{ + static const float kScale = 0.5f; + static const float kScaleInv = 1.0f / kScale; + + Backend backendId = get<0>(get<2>(GetParam())); + Target targetId = get<1>(get<2>(GetParam())); + + Net net; + LayerParams lp; + lp.type = "Eltwise"; + lp.name = "testLayer"; + lp.set("operation", "sum"); + int eltwiseId = net.addLayerToPrev(lp.name, lp.type, lp); // connect to a first input + net.connect(0, 1, eltwiseId, 1); // connect to a second input + + int inpSize[] = {1, 2, 3, 4}; + Mat firstInp(4, &inpSize[0], get<0>(GetParam())); + Mat secondInp(4, &inpSize[0], get<1>(GetParam())); + randu(firstInp, 0, 100); + randu(secondInp, 0, 100); + +#ifndef CV_CXX11 + std::vector input_names; + input_names.push_back("data"); + input_names.push_back("second_input"); + net.setInputsNames(input_names); +#else + net.setInputsNames({"data", "second_input"}); +#endif + net.setInput(firstInp, "data", kScale); + net.setInput(secondInp, "second_input", kScaleInv); + net.setPreferableBackend(backendId); + net.setPreferableTarget(targetId); + Mat out = net.forward(); + + Mat ref; + addWeighted(firstInp, kScale, secondInp, kScaleInv, 0, ref, CV_32F); + + double l1 = (targetId == DNN_TARGET_OPENCL_FP16 || targetId == DNN_TARGET_MYRIAD) ? 0.06 : 1e-6; + double lInf = (targetId == DNN_TARGET_OPENCL_FP16 || targetId == DNN_TARGET_MYRIAD) ? 0.3 : 1e-5; + normAssert(out, ref, "", l1, lInf); + + if (cvtest::debugLevel > 0 || HasFailure()) + { + std::cout << "input1 scale=" << kScale << " input2 scale=" << kScaleInv << std::endl; + std::cout << "input1: " << firstInp.size << " " << firstInp.reshape(1, 1) << std::endl; + std::cout << "input2: " << secondInp.size << " " << secondInp.reshape(1, 1) << std::endl; + std::cout << "ref: " << ref.reshape(1, 1) << std::endl; + std::cout << "out: " << out.reshape(1, 1) << std::endl; + } +} + +INSTANTIATE_TEST_CASE_P(/*nothing*/, Test_two_inputs, Combine( + Values(CV_32F, CV_8U), + Values(CV_32F, CV_8U), + dnnBackendsAndTargets() +)); + }} // namespace From 58b06222ffa637b8278c04f70ffd60ae762a7bb2 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sun, 28 Nov 2021 04:29:54 +0000 Subject: [PATCH 104/226] dnn(DataLayer): fix CPU/OpenCL code paths for FP16 handling --- modules/dnn/src/dnn.cpp | 115 ++++++++++++++++++++++------------------ 1 file changed, 62 insertions(+), 53 deletions(-) diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index 8182394387..b35dda9ddf 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -597,29 +597,26 @@ struct DataLayer : public Layer CV_TRACE_FUNCTION(); CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + // FIXIT: add wrapper without exception suppression CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget), forward_ocl(inputs_arr, outputs_arr, internals_arr)) - if (outputs_arr.depth() == CV_16S) - { - forward_fallback(inputs_arr, outputs_arr, internals_arr); - return; - } + bool isFP16 = outputs_arr.depth() == CV_16S; std::vector outputs, internals; outputs_arr.getMatVector(outputs); internals_arr.getMatVector(internals); - // Supported modes: - // | Input type | Output type | - // | fp32 | fp32 | - // | uint8 | fp32 | for (int i = 0; i < inputsData.size(); ++i) { double scale = scaleFactors[i]; Scalar& mean = means[i]; + CV_Assert(mean == Scalar() || inputsData[i].size[1] <= 4); - CV_CheckTypeEQ(outputs[i].type(), CV_32FC1, ""); + if (isFP16) + CV_CheckTypeEQ(outputs[i].type(), CV_16SC1, ""); + else + CV_CheckTypeEQ(outputs[i].type(), CV_32FC1, ""); bool singleMean = true; for (int j = 1; j < std::min(4, inputsData[i].size[1]) && singleMean; ++j) @@ -629,34 +626,49 @@ struct DataLayer : public Layer if (singleMean) { - inputsData[i].convertTo(outputs[i], CV_32F, scale, -mean[0] * scale); + if (isFP16) + { + Mat input_f32; + inputsData[i].convertTo(input_f32, CV_32F, scale, -mean[0] * scale); + convertFp16(input_f32, outputs[i]); + } + else + { + inputsData[i].convertTo(outputs[i], CV_32F, scale, -mean[0] * scale); + } } else { for (int n = 0; n < inputsData[i].size[0]; ++n) + { for (int c = 0; c < inputsData[i].size[1]; ++c) { Mat inp = getPlane(inputsData[i], n, c); Mat out = getPlane(outputs[i], n, c); - inp.convertTo(out, CV_32F, scale, -mean[c] * scale); + if (isFP16) + { + Mat input_f32; + inp.convertTo(input_f32, CV_32F, scale, -mean[c] * scale); + convertFp16(input_f32, out); + } + else + { + inp.convertTo(out, CV_32F, scale, -mean[c] * scale); + } } + } } } } #ifdef HAVE_OPENCL - std::vector tmp_expressions; bool forward_ocl(InputArrayOfArrays, OutputArrayOfArrays outputs_, OutputArrayOfArrays internals_) { - // Supported modes: - // | Input type | Output type | - // | fp32 | fp32 | - // | fp32 | fp16 | - // | uint8 | fp32 | + bool isFP16 = outputs_.depth() == CV_16S; + std::vector outputs; outputs_.getUMatVector(outputs); - tmp_expressions.clear(); for (int i = 0; i < inputsData.size(); ++i) { Mat inputData = inputsData[i]; @@ -664,58 +676,55 @@ struct DataLayer : public Layer double scale = scaleFactors[i]; Scalar& mean = means[i]; - CV_Assert(mean == Scalar() || inputsData[i].size[1] <= 4); + CV_Assert(mean == Scalar() || inputData.size[1] <= 4); + if (isFP16) + CV_CheckTypeEQ(outputs[i].type(), CV_16SC1, ""); + else + CV_CheckTypeEQ(outputs[i].type(), CV_32FC1, ""); + bool singleMean = true; - for (int j = 1; j < std::min(4, inputsData[i].size[1]) && singleMean; ++j) + for (int j = 1; j < std::min(4, inputData.size[1]) && singleMean; ++j) { singleMean = mean[j] == mean[j - 1]; } - if (outputs_.depth() == CV_16S) + if (singleMean) { - if (singleMean) + if (isFP16) { - tmp_expressions.push_back(Mat(scale * (inputsData[i] - mean[0]))); - convertFp16(tmp_expressions.back(), outputs[i]); + UMat input_i; + inputData.convertTo(input_i, CV_32F, scale, -mean[0] * scale); + convertFp16(input_i, outputs[i]); } else { - for (int n = 0; n < inputsData[i].size[0]; ++n) - for (int c = 0; c < inputsData[i].size[1]; ++c) - { - Mat inp = getPlane(inputsData[i], n, c); - - std::vector plane(4, Range::all()); - plane[0] = Range(n, n + 1); - plane[1] = Range(c, c + 1); - UMat out = outputs[i](plane).reshape(1, inp.dims, inp.size); - - tmp_expressions.push_back(scale * (inp - mean[c])); - convertFp16(tmp_expressions.back(), out); - } + inputData.convertTo(outputs[i], CV_32F, scale, -mean[0] * scale); } } else { - CV_Assert(outputs_.depth() == CV_32F); - if (singleMean) + for (int n = 0; n < inputData.size[0]; ++n) { - inputsData[i].convertTo(outputs[i], CV_32F, scale, -mean[0] * scale); - } - else - { - for (int n = 0; n < inputsData[i].size[0]; ++n) - for (int c = 0; c < inputsData[i].size[1]; ++c) + for (int c = 0; c < inputData.size[1]; ++c) + { + Mat inp = getPlane(inputData, n, c); + + std::vector plane(4, Range::all()); + plane[0] = Range(n, n + 1); + plane[1] = Range(c, c + 1); + UMat out = outputs[i](plane).reshape(1, inp.dims, inp.size); + + if (isFP16) + { + UMat input_i; + inp.convertTo(input_i, CV_32F, scale, -mean[c] * scale); + convertFp16(input_i, out); + } + else { - Mat inp = getPlane(inputsData[i], n, c); - - std::vector plane(4, Range::all()); - plane[0] = Range(n, n + 1); - plane[1] = Range(c, c + 1); - UMat out = outputs[i](plane).reshape(1, inp.dims, inp.size); - inp.convertTo(out, CV_32F, scale, -mean[c] * scale); } + } } } } From b594ed99b834e38a25cb8175d839810a30a83d83 Mon Sep 17 00:00:00 2001 From: Supernovae <51359628+shubham-shahh@users.noreply.github.com> Date: Sun, 28 Nov 2021 18:24:29 +0530 Subject: [PATCH 105/226] Merge pull request #20933 from shubham-shahh:master Improved overall readability of the code * grid_nms.cu: minor fix-ups * Update grid_stride_range.hpp * Update tf_importer.cpp --- modules/dnn/src/cuda/grid_nms.cu | 10 +++++----- modules/dnn/src/cuda/grid_stride_range.hpp | 2 +- modules/dnn/src/tensorflow/tf_importer.cpp | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/dnn/src/cuda/grid_nms.cu b/modules/dnn/src/cuda/grid_nms.cu index 0aeb34add3..51b95d3b80 100644 --- a/modules/dnn/src/cuda/grid_nms.cu +++ b/modules/dnn/src/cuda/grid_nms.cu @@ -68,7 +68,7 @@ namespace raw { * to compute IOU(GROUP_B, GROUP_A). We still have to compute IOU(GROUP_A, GROUP_A) though since * each group has many boxes and we need IOUs amongst boxes within a group. * - * We arbitarily choose a scheme to exit : exit if group_i is greater than group_j. This way we only + * We arbitrarily choose a scheme to exit : exit if group_i is greater than group_j. This way we only * compute IOUs between groups once. While nearly half the blocks are wasted, it's ok since they exit * early on and the working blocks are compute heavy. */ @@ -92,7 +92,7 @@ namespace raw { */ /* The `j` box is fixed for each thread. All `i` boxes will be required for every thread. - * We store the `i` boxes in shared memory to allow global memory coalesing. + * We store the `i` boxes in shared memory to allow global memory coalescing. */ using vector_type = get_vector_type_t; __shared__ vector_type group_i_boxes[BLOCK_SIZE]; @@ -162,7 +162,7 @@ namespace raw { * this loop has been highly tuned. Please profile and verify carefully before making changes. */ /* UNROLL_SIZE is the number of boxes that must be processed per iteration. We manually unroll - * the loop since the compiler cannot effectively unroll on its own preassumably due to presence + * the loop since the compiler cannot effectively unroll on its own presumably due to presence * of instructions forcing warp synchronization. */ constexpr int UNROLL_SIZE = 4; @@ -290,7 +290,7 @@ namespace raw { if (boxes == 0) return; - /* We have a fixed number of threads and an arbitary number of boxes. We use an array of + /* We have a fixed number of threads and an arbitrary number of boxes. We use an array of * bits to store which boxes haven't been eliminated and which are still active. We organize * the array of bits into a matrix of bits of the shape (num_rows, BLOCK_SIZE, 32) which * is equivalent to (num_rows, BLOCK_SIZE) where the type is a 32-bit unsigned integer. @@ -464,4 +464,4 @@ std::size_t getGridNMSWorkspaceSizePerBatchItem(std::size_t num_classes, std::si template void grid_nms(const Stream& stream, Span workspace, TensorSpan indices, TensorSpan count, TensorView<__half> bboxes, int, bool normalized_bbox, float nms_threshold); template void grid_nms(const Stream& stream, Span workspace, TensorSpan indices, TensorSpan count, TensorView bboxes, int, bool normalized_bbox, float nms_threshold); -}}}} /* namespace cv::dnn::cuda4dnn::kernels */ \ No newline at end of file +}}}} /* namespace cv::dnn::cuda4dnn::kernels */ diff --git a/modules/dnn/src/cuda/grid_stride_range.hpp b/modules/dnn/src/cuda/grid_stride_range.hpp index cf263c54e2..36d14a9368 100644 --- a/modules/dnn/src/cuda/grid_stride_range.hpp +++ b/modules/dnn/src/cuda/grid_stride_range.hpp @@ -36,7 +36,7 @@ public: __device__ bool operator!=(const iterator& other) const { /* NOTE HACK * 'pos' can move in large steps (see operator++) - * expansion of range for loop uses != as the loop conditioion + * expansion of range for loop uses != as the loop condition * => operator!= must return false if 'pos' crosses the end */ return pos < other.pos; diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index a6f9c07980..9fb8f60b41 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -1154,7 +1154,7 @@ void TFImporter::parseExpandDims(tensorflow::GraphDef& net, const tensorflow::No // Convert OpenCV's NHC to NCH first. if(outShapeSize == 3) { - // If axis equal to outShapeSize, that mean we expand in Channel dimmension, and do not add permuteLayer. + // If axis equal to outShapeSize, that mean we expand in Channel dimension, and do not add permuteLayer. if(axis != outShapeSize) { int order[] = {0, 2, 1}; // From OpenCV's NHC to NCH. From a97f21ba4e044501d9bbb636fc6b33dcdddd9e24 Mon Sep 17 00:00:00 2001 From: Suleyman TURKMEN Date: Sun, 28 Nov 2021 15:56:28 +0300 Subject: [PATCH 106/226] Merge pull request #20957 from sturkmen72:update-documentation Update documentation * Update DNN-based Face Detection And Recognition tutorial * samples(dnn/face): update face_detect.cpp * final changes Co-authored-by: Alexander Alekhin --- doc/tutorials/dnn/dnn_face/dnn_face.markdown | 67 +++-- samples/dnn/face_detect.cpp | 255 +++++++++++++++---- samples/dnn/face_detect.py | 126 ++++++--- samples/dnn/face_match.cpp | 103 -------- samples/dnn/face_match.py | 57 ----- 5 files changed, 333 insertions(+), 275 deletions(-) delete mode 100644 samples/dnn/face_match.cpp delete mode 100644 samples/dnn/face_match.py diff --git a/doc/tutorials/dnn/dnn_face/dnn_face.markdown b/doc/tutorials/dnn/dnn_face/dnn_face.markdown index 202be3e0e3..f55cdb79e8 100644 --- a/doc/tutorials/dnn/dnn_face/dnn_face.markdown +++ b/doc/tutorials/dnn/dnn_face/dnn_face.markdown @@ -36,14 +36,34 @@ There are two models (ONNX format) pre-trained and required for this module: ### DNNFaceDetector -```cpp -// Initialize FaceDetectorYN -Ptr faceDetector = FaceDetectorYN::create(onnx_path, "", image.size(), score_thresh, nms_thresh, top_k); +@add_toggle_cpp +- **Downloadable code**: Click + [here](https://github.com/opencv/opencv/tree/master/samples/dnn/face_detect.cpp) -// Forward -Mat faces; -faceDetector->detect(image, faces); -``` +- **Code at glance:** + @include samples/dnn/face_detect.cpp +@end_toggle + +@add_toggle_python +- **Downloadable code**: Click + [here](https://github.com/opencv/opencv/tree/master/samples/dnn/face_detect.py) + +- **Code at glance:** + @include samples/dnn/face_detect.py +@end_toggle + +Explanation +----------- + +@add_toggle_cpp +@snippet dnn/face_detect.cpp initialize_FaceDetectorYN +@snippet dnn/face_detect.cpp inference +@end_toggle + +@add_toggle_python +@snippet dnn/face_detect.py initialize_FaceDetectorYN +@snippet dnn/face_detect.py inference +@end_toggle The detection output `faces` is a two-dimension array of type CV_32F, whose rows are the detected face instances, columns are the location of a face and 5 facial landmarks. The format of each row is as follows: @@ -57,28 +77,25 @@ x1, y1, w, h, x_re, y_re, x_le, y_le, x_nt, y_nt, x_rcm, y_rcm, x_lcm, y_lcm Following Face Detection, run codes below to extract face feature from facial image. -```cpp -// Initialize FaceRecognizerSF with model path (cv::String) -Ptr faceRecognizer = FaceRecognizerSF::create(model_path, ""); +@add_toggle_cpp +@snippet dnn/face_detect.cpp initialize_FaceRecognizerSF +@snippet dnn/face_detect.cpp facerecognizer +@end_toggle -// Aligning and cropping facial image through the first face of faces detected by dnn_face::DNNFaceDetector -Mat aligned_face; -faceRecognizer->alignCrop(image, faces.row(0), aligned_face); - -// Run feature extraction with given aligned_face (cv::Mat) -Mat feature; -faceRecognizer->feature(aligned_face, feature); -feature = feature.clone(); -``` +@add_toggle_python +@snippet dnn/face_detect.py initialize_FaceRecognizerSF +@snippet dnn/face_detect.py facerecognizer +@end_toggle After obtaining face features *feature1* and *feature2* of two facial images, run codes below to calculate the identity discrepancy between the two faces. -```cpp -// Calculating the discrepancy between two face features by using cosine distance. -double cos_score = faceRecognizer->match(feature1, feature2, FaceRecognizer::DisType::COSINE); -// Calculating the discrepancy between two face features by using normL2 distance. -double L2_score = faceRecognizer->match(feature1, feature2, FaceRecognizer::DisType::NORM_L2); -``` +@add_toggle_cpp +@snippet dnn/face_detect.cpp match +@end_toggle + +@add_toggle_python +@snippet dnn/face_detect.py match +@end_toggle For example, two faces have same identity if the cosine distance is greater than or equal to 0.363, or the normL2 distance is less than or equal to 1.128. diff --git a/samples/dnn/face_detect.cpp b/samples/dnn/face_detect.cpp index 8d91a10968..161940cb4a 100644 --- a/samples/dnn/face_detect.cpp +++ b/samples/dnn/face_detect.cpp @@ -8,125 +8,272 @@ using namespace cv; using namespace std; -static Mat visualize(Mat input, Mat faces, int thickness=2) +static +void visualize(Mat& input, int frame, Mat& faces, double fps, int thickness = 2) { - Mat output = input.clone(); + std::string fpsString = cv::format("FPS : %.2f", (float)fps); + if (frame >= 0) + cout << "Frame " << frame << ", "; + cout << "FPS: " << fpsString << endl; for (int i = 0; i < faces.rows; i++) { // Print results cout << "Face " << i << ", top-left coordinates: (" << faces.at(i, 0) << ", " << faces.at(i, 1) << "), " << "box width: " << faces.at(i, 2) << ", box height: " << faces.at(i, 3) << ", " - << "score: " << faces.at(i, 14) << "\n"; + << "score: " << cv::format("%.2f", faces.at(i, 14)) + << endl; // Draw bounding box - rectangle(output, Rect2i(int(faces.at(i, 0)), int(faces.at(i, 1)), int(faces.at(i, 2)), int(faces.at(i, 3))), Scalar(0, 255, 0), thickness); + rectangle(input, Rect2i(int(faces.at(i, 0)), int(faces.at(i, 1)), int(faces.at(i, 2)), int(faces.at(i, 3))), Scalar(0, 255, 0), thickness); // Draw landmarks - circle(output, Point2i(int(faces.at(i, 4)), int(faces.at(i, 5))), 2, Scalar(255, 0, 0), thickness); - circle(output, Point2i(int(faces.at(i, 6)), int(faces.at(i, 7))), 2, Scalar( 0, 0, 255), thickness); - circle(output, Point2i(int(faces.at(i, 8)), int(faces.at(i, 9))), 2, Scalar( 0, 255, 0), thickness); - circle(output, Point2i(int(faces.at(i, 10)), int(faces.at(i, 11))), 2, Scalar(255, 0, 255), thickness); - circle(output, Point2i(int(faces.at(i, 12)), int(faces.at(i, 13))), 2, Scalar( 0, 255, 255), thickness); + circle(input, Point2i(int(faces.at(i, 4)), int(faces.at(i, 5))), 2, Scalar(255, 0, 0), thickness); + circle(input, Point2i(int(faces.at(i, 6)), int(faces.at(i, 7))), 2, Scalar(0, 0, 255), thickness); + circle(input, Point2i(int(faces.at(i, 8)), int(faces.at(i, 9))), 2, Scalar(0, 255, 0), thickness); + circle(input, Point2i(int(faces.at(i, 10)), int(faces.at(i, 11))), 2, Scalar(255, 0, 255), thickness); + circle(input, Point2i(int(faces.at(i, 12)), int(faces.at(i, 13))), 2, Scalar(0, 255, 255), thickness); } - return output; + putText(input, fpsString, Point(0, 15), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0), 2); } -int main(int argc, char ** argv) +int main(int argc, char** argv) { CommandLineParser parser(argc, argv, - "{help h | | Print this message.}" - "{input i | | Path to the input image. Omit for detecting on default camera.}" - "{model m | yunet.onnx | Path to the model. Download yunet.onnx in https://github.com/ShiqiYu/libfacedetection.train/tree/master/tasks/task1/onnx.}" - "{score_threshold | 0.9 | Filter out faces of score < score_threshold.}" - "{nms_threshold | 0.3 | Suppress bounding boxes of iou >= nms_threshold.}" - "{top_k | 5000 | Keep top_k bounding boxes before NMS.}" - "{save s | false | Set true to save results. This flag is invalid when using camera.}" - "{vis v | true | Set true to open a window for result visualization. This flag is invalid when using camera.}" + "{help h | | Print this message}" + "{image1 i1 | | Path to the input image1. Omit for detecting through VideoCapture}" + "{image2 i2 | | Path to the input image2. When image1 and image2 parameters given then the program try to find a face on both images and runs face recognition algorithm}" + "{video v | 0 | Path to the input video}" + "{scale sc | 1.0 | Scale factor used to resize input video frames}" + "{fd_model fd | yunet.onnx | Path to the model. Download yunet.onnx in https://github.com/ShiqiYu/libfacedetection.train/tree/master/tasks/task1/onnx }" + "{fr_model fr | face_recognizer_fast.onnx | Path to the face recognition model. Download the model at https://drive.google.com/file/d/1ClK9WiB492c5OZFKveF3XiHCejoOxINW/view}" + "{score_threshold | 0.9 | Filter out faces of score < score_threshold}" + "{nms_threshold | 0.3 | Suppress bounding boxes of iou >= nms_threshold}" + "{top_k | 5000 | Keep top_k bounding boxes before NMS}" + "{save s | false | Set true to save results. This flag is invalid when using camera}" ); - if (argc == 1 || parser.has("help")) + if (parser.has("help")) { parser.printMessage(); - return -1; + return 0; } - String modelPath = parser.get("model"); + String fd_modelPath = parser.get("fd_model"); + String fr_modelPath = parser.get("fr_model"); float scoreThreshold = parser.get("score_threshold"); float nmsThreshold = parser.get("nms_threshold"); int topK = parser.get("top_k"); bool save = parser.get("save"); - bool vis = parser.get("vis"); + double cosine_similar_thresh = 0.363; + double l2norm_similar_thresh = 1.128; + + //! [initialize_FaceDetectorYN] // Initialize FaceDetectorYN - Ptr detector = FaceDetectorYN::create(modelPath, "", Size(320, 320), scoreThreshold, nmsThreshold, topK); + Ptr detector = FaceDetectorYN::create(fd_modelPath, "", Size(320, 320), scoreThreshold, nmsThreshold, topK); + //! [initialize_FaceDetectorYN] + + TickMeter tm; // If input is an image - if (parser.has("input")) + if (parser.has("image1")) { - String input = parser.get("input"); - Mat image = imread(input); + String input1 = parser.get("image1"); + Mat image1 = imread(samples::findFile(input1)); + if (image1.empty()) + { + std::cerr << "Cannot read image: " << input1 << std::endl; + return 2; + } + tm.start(); + + //! [inference] // Set input size before inference - detector->setInputSize(image.size()); + detector->setInputSize(image1.size()); - // Inference - Mat faces; - detector->detect(image, faces); + Mat faces1; + detector->detect(image1, faces1); + if (faces1.rows < 1) + { + std::cerr << "Cannot find a face in " << input1 << std::endl; + return 1; + } + //! [inference] + tm.stop(); // Draw results on the input image - Mat result = visualize(image, faces); + visualize(image1, -1, faces1, tm.getFPS()); // Save results if save is true - if(save) + if (save) { - cout << "Results saved to result.jpg\n"; - imwrite("result.jpg", result); + cout << "Saving result.jpg...\n"; + imwrite("result.jpg", image1); } // Visualize results - if (vis) + imshow("image1", image1); + pollKey(); // handle UI events to show content + + if (parser.has("image2")) { - namedWindow(input, WINDOW_AUTOSIZE); - imshow(input, result); - waitKey(0); + String input2 = parser.get("image2"); + Mat image2 = imread(samples::findFile(input2)); + if (image2.empty()) + { + std::cerr << "Cannot read image2: " << input2 << std::endl; + return 2; + } + + tm.reset(); + tm.start(); + detector->setInputSize(image2.size()); + + Mat faces2; + detector->detect(image2, faces2); + if (faces2.rows < 1) + { + std::cerr << "Cannot find a face in " << input2 << std::endl; + return 1; + } + tm.stop(); + visualize(image2, -1, faces2, tm.getFPS()); + if (save) + { + cout << "Saving result2.jpg...\n"; + imwrite("result2.jpg", image2); + } + imshow("image2", image2); + pollKey(); + + //! [initialize_FaceRecognizerSF] + // Initialize FaceRecognizerSF + Ptr faceRecognizer = FaceRecognizerSF::create(fr_modelPath, ""); + //! [initialize_FaceRecognizerSF] + + + //! [facerecognizer] + // Aligning and cropping facial image through the first face of faces detected. + Mat aligned_face1, aligned_face2; + faceRecognizer->alignCrop(image1, faces1.row(0), aligned_face1); + faceRecognizer->alignCrop(image2, faces2.row(0), aligned_face2); + + // Run feature extraction with given aligned_face + Mat feature1, feature2; + faceRecognizer->feature(aligned_face1, feature1); + feature1 = feature1.clone(); + faceRecognizer->feature(aligned_face2, feature2); + feature2 = feature2.clone(); + //! [facerecognizer] + + //! [match] + double cos_score = faceRecognizer->match(feature1, feature2, FaceRecognizerSF::DisType::FR_COSINE); + double L2_score = faceRecognizer->match(feature1, feature2, FaceRecognizerSF::DisType::FR_NORM_L2); + //! [match] + + if (cos_score >= cosine_similar_thresh) + { + std::cout << "They have the same identity;"; + } + else + { + std::cout << "They have different identities;"; + } + std::cout << " Cosine Similarity: " << cos_score << ", threshold: " << cosine_similar_thresh << ". (higher value means higher similarity, max 1.0)\n"; + + if (L2_score <= l2norm_similar_thresh) + { + std::cout << "They have the same identity;"; + } + else + { + std::cout << "They have different identities."; + } + std::cout << " NormL2 Distance: " << L2_score << ", threshold: " << l2norm_similar_thresh << ". (lower value means higher similarity, min 0.0)\n"; } + cout << "Press any key to exit..." << endl; + waitKey(0); } else { - int deviceId = 0; - VideoCapture cap; - cap.open(deviceId, CAP_ANY); - int frameWidth = int(cap.get(CAP_PROP_FRAME_WIDTH)); - int frameHeight = int(cap.get(CAP_PROP_FRAME_HEIGHT)); + int frameWidth, frameHeight; + float scale = parser.get("scale"); + VideoCapture capture; + std::string video = parser.get("video"); + if (video.size() == 1 && isdigit(video[0])) + capture.open(parser.get("video")); + else + capture.open(samples::findFileOrKeep(video)); // keep GStreamer pipelines + if (capture.isOpened()) + { + frameWidth = int(capture.get(CAP_PROP_FRAME_WIDTH) * scale); + frameHeight = int(capture.get(CAP_PROP_FRAME_HEIGHT) * scale); + cout << "Video " << video + << ": width=" << frameWidth + << ", height=" << frameHeight + << endl; + } + else + { + cout << "Could not initialize video capturing: " << video << "\n"; + return 1; + } + detector->setInputSize(Size(frameWidth, frameHeight)); - Mat frame; - TickMeter tm; - String msg = "FPS: "; - while(waitKey(1) < 0) // Press any key to exit + cout << "Press 'SPACE' to save frame, any other key to exit..." << endl; + int nFrame = 0; + for (;;) { // Get frame - if (!cap.read(frame)) + Mat frame; + if (!capture.read(frame)) { - cerr << "No frames grabbed!\n"; + cerr << "Can't grab frame! Stop\n"; break; } + resize(frame, frame, Size(frameWidth, frameHeight)); + // Inference Mat faces; tm.start(); detector->detect(frame, faces); tm.stop(); + Mat result = frame.clone(); // Draw results on the input image - Mat result = visualize(frame, faces); - putText(result, msg + to_string(tm.getFPS()), Point(0, 15), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); + visualize(result, nFrame, faces, tm.getFPS()); // Visualize results imshow("Live", result); - tm.reset(); + int key = waitKey(1); + bool saveFrame = save; + if (key == ' ') + { + saveFrame = true; + key = 0; // handled + } + + if (saveFrame) + { + std::string frame_name = cv::format("frame_%05d.png", nFrame); + std::string result_name = cv::format("result_%05d.jpg", nFrame); + cout << "Saving '" << frame_name << "' and '" << result_name << "' ...\n"; + imwrite(frame_name, frame); + imwrite(result_name, result); + } + + ++nFrame; + + if (key > 0) + break; } + cout << "Processed " << nFrame << " frames" << endl; } -} \ No newline at end of file + cout << "Done." << endl; + return 0; +} diff --git a/samples/dnn/face_detect.py b/samples/dnn/face_detect.py index 65069d6590..8900a7f7ad 100644 --- a/samples/dnn/face_detect.py +++ b/samples/dnn/face_detect.py @@ -12,90 +12,144 @@ def str2bool(v): raise NotImplementedError parser = argparse.ArgumentParser() -parser.add_argument('--input', '-i', type=str, help='Path to the input image.') -parser.add_argument('--model', '-m', type=str, default='yunet.onnx', help='Path to the model. Download the model at https://github.com/ShiqiYu/libfacedetection.train/tree/master/tasks/task1/onnx.') +parser.add_argument('--image1', '-i1', type=str, help='Path to the input image1. Omit for detecting on default camera.') +parser.add_argument('--image2', '-i2', type=str, help='Path to the input image2. When image1 and image2 parameters given then the program try to find a face on both images and runs face recognition algorithm.') +parser.add_argument('--video', '-v', type=str, help='Path to the input video.') +parser.add_argument('--scale', '-sc', type=float, default=1.0, help='Scale factor used to resize input video frames.') +parser.add_argument('--face_detection_model', '-fd', type=str, default='yunet.onnx', help='Path to the face detection model. Download the model at https://github.com/ShiqiYu/libfacedetection.train/tree/master/tasks/task1/onnx.') +parser.add_argument('--face_recognition_model', '-fr', type=str, default='face_recognizer_fast.onnx', help='Path to the face recognition model. Download the model at https://drive.google.com/file/d/1ClK9WiB492c5OZFKveF3XiHCejoOxINW/view.') parser.add_argument('--score_threshold', type=float, default=0.9, help='Filtering out faces of score < score_threshold.') parser.add_argument('--nms_threshold', type=float, default=0.3, help='Suppress bounding boxes of iou >= nms_threshold.') parser.add_argument('--top_k', type=int, default=5000, help='Keep top_k bounding boxes before NMS.') parser.add_argument('--save', '-s', type=str2bool, default=False, help='Set true to save results. This flag is invalid when using camera.') -parser.add_argument('--vis', '-v', type=str2bool, default=True, help='Set true to open a window for result visualization. This flag is invalid when using camera.') args = parser.parse_args() -def visualize(input, faces, thickness=2): - output = input.copy() +def visualize(input, faces, fps, thickness=2): if faces[1] is not None: for idx, face in enumerate(faces[1]): print('Face {}, top-left coordinates: ({:.0f}, {:.0f}), box width: {:.0f}, box height {:.0f}, score: {:.2f}'.format(idx, face[0], face[1], face[2], face[3], face[-1])) coords = face[:-1].astype(np.int32) - cv.rectangle(output, (coords[0], coords[1]), (coords[0]+coords[2], coords[1]+coords[3]), (0, 255, 0), 2) - cv.circle(output, (coords[4], coords[5]), 2, (255, 0, 0), 2) - cv.circle(output, (coords[6], coords[7]), 2, (0, 0, 255), 2) - cv.circle(output, (coords[8], coords[9]), 2, (0, 255, 0), 2) - cv.circle(output, (coords[10], coords[11]), 2, (255, 0, 255), 2) - cv.circle(output, (coords[12], coords[13]), 2, (0, 255, 255), 2) - return output + cv.rectangle(input, (coords[0], coords[1]), (coords[0]+coords[2], coords[1]+coords[3]), (0, 255, 0), thickness) + cv.circle(input, (coords[4], coords[5]), 2, (255, 0, 0), thickness) + cv.circle(input, (coords[6], coords[7]), 2, (0, 0, 255), thickness) + cv.circle(input, (coords[8], coords[9]), 2, (0, 255, 0), thickness) + cv.circle(input, (coords[10], coords[11]), 2, (255, 0, 255), thickness) + cv.circle(input, (coords[12], coords[13]), 2, (0, 255, 255), thickness) + cv.putText(input, 'FPS: {:.2f}'.format(fps), (1, 16), cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2) if __name__ == '__main__': - # Instantiate FaceDetectorYN + ## [initialize_FaceDetectorYN] detector = cv.FaceDetectorYN.create( - args.model, + args.face_detection_model, "", (320, 320), args.score_threshold, args.nms_threshold, args.top_k ) + ## [initialize_FaceDetectorYN] + + tm = cv.TickMeter() # If input is an image - if args.input is not None: - image = cv.imread(args.input) + if args.image1 is not None: + img1 = cv.imread(cv.samples.findFile(args.image1)) + tm.start() + ## [inference] # Set input size before inference - detector.setInputSize((image.shape[1], image.shape[0])) + detector.setInputSize((img1.shape[1], img1.shape[0])) - # Inference - faces = detector.detect(image) + faces1 = detector.detect(img1) + ## [inference] + + tm.stop() + assert faces1[1] is not None, 'Cannot find a face in {}'.format(args.image1) # Draw results on the input image - result = visualize(image, faces) + visualize(img1, faces1, tm.getFPS()) # Save results if save is true if args.save: - print('Resutls saved to result.jpg\n') - cv.imwrite('result.jpg', result) + print('Results saved to result.jpg\n') + cv.imwrite('result.jpg', img1) # Visualize results in a new window - if args.vis: - cv.namedWindow(args.input, cv.WINDOW_AUTOSIZE) - cv.imshow(args.input, result) - cv.waitKey(0) + cv.imshow("image1", img1) + + if args.image2 is not None: + img2 = cv.imread(cv.samples.findFile(args.image2)) + + tm.reset() + tm.start() + detector.setInputSize((img2.shape[1], img2.shape[0])) + faces2 = detector.detect(img2) + tm.stop() + assert faces2[1] is not None, 'Cannot find a face in {}'.format(args.image2) + visualize(img2, faces2, tm.getFPS()) + cv.imshow("image2", img2) + + ## [initialize_FaceRecognizerSF] + recognizer = cv.FaceRecognizerSF.create( + args.face_recognition_model,"") + ## [initialize_FaceRecognizerSF] + + ## [facerecognizer] + # Align faces + face1_align = recognizer.alignCrop(img1, faces1[1][0]) + face2_align = recognizer.alignCrop(img2, faces2[1][0]) + + # Extract features + face1_feature = recognizer.feature(face1_align) + face2_feature = recognizer.feature(face2_align) + ## [facerecognizer] + + cosine_similarity_threshold = 0.363 + l2_similarity_threshold = 1.128 + + ## [match] + cosine_score = recognizer.match(face1_feature, face2_feature, cv.FaceRecognizerSF_FR_COSINE) + l2_score = recognizer.match(face1_feature, face2_feature, cv.FaceRecognizerSF_FR_NORM_L2) + ## [match] + + msg = 'different identities' + if cosine_score >= cosine_similarity_threshold: + msg = 'the same identity' + print('They have {}. Cosine Similarity: {}, threshold: {} (higher value means higher similarity, max 1.0).'.format(msg, cosine_score, cosine_similarity_threshold)) + + msg = 'different identities' + if l2_score <= l2_similarity_threshold: + msg = 'the same identity' + print('They have {}. NormL2 Distance: {}, threshold: {} (lower value means higher similarity, min 0.0).'.format(msg, l2_score, l2_similarity_threshold)) + cv.waitKey(0) else: # Omit input to call default camera - deviceId = 0 + if args.video is not None: + deviceId = args.video + else: + deviceId = 0 cap = cv.VideoCapture(deviceId) - frameWidth = int(cap.get(cv.CAP_PROP_FRAME_WIDTH)) - frameHeight = int(cap.get(cv.CAP_PROP_FRAME_HEIGHT)) + frameWidth = int(cap.get(cv.CAP_PROP_FRAME_WIDTH)*args.scale) + frameHeight = int(cap.get(cv.CAP_PROP_FRAME_HEIGHT)*args.scale) detector.setInputSize([frameWidth, frameHeight]) - tm = cv.TickMeter() while cv.waitKey(1) < 0: hasFrame, frame = cap.read() if not hasFrame: print('No frames grabbed!') break + frame = cv.resize(frame, (frameWidth, frameHeight)) + # Inference tm.start() faces = detector.detect(frame) # faces is a tuple tm.stop() # Draw results on the input image - frame = visualize(frame, faces) + visualize(frame, faces, tm.getFPS()) - cv.putText(frame, 'FPS: {}'.format(tm.getFPS()), (0, 15), cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0)) - - # Visualize results in a new Window + # Visualize results cv.imshow('Live', frame) - - tm.reset() \ No newline at end of file + cv.destroyAllWindows() diff --git a/samples/dnn/face_match.cpp b/samples/dnn/face_match.cpp deleted file mode 100644 index f24134b890..0000000000 --- a/samples/dnn/face_match.cpp +++ /dev/null @@ -1,103 +0,0 @@ -// This file is part of OpenCV project. -// It is subject to the license terms in the LICENSE file found in the top-level directory -// of this distribution and at http://opencv.org/license.html. - -#include "opencv2/dnn.hpp" -#include "opencv2/imgproc.hpp" -#include "opencv2/highgui.hpp" - -#include - -#include "opencv2/objdetect.hpp" - - -using namespace cv; -using namespace std; - - -int main(int argc, char ** argv) -{ - if (argc != 5) - { - std::cerr << "Usage " << argv[0] << ": " - << " " - << " " - << "" - << "\n"; - return -1; - } - - String det_onnx_path = argv[1]; - String reg_onnx_path = argv[2]; - String image1_path = argv[3]; - String image2_path = argv[4]; - std::cout< faceDetector; - - faceDetector = FaceDetectorYN::create(det_onnx_path, "", image1.size(), score_thresh, nms_thresh, top_k); - Mat faces_1; - faceDetector->detect(image1, faces_1); - if (faces_1.rows < 1) - { - std::cerr << "Cannot find a face in " << image1_path << "\n"; - return -1; - } - - faceDetector = FaceDetectorYN::create(det_onnx_path, "", image2.size(), score_thresh, nms_thresh, top_k); - Mat faces_2; - faceDetector->detect(image2, faces_2); - if (faces_2.rows < 1) - { - std::cerr << "Cannot find a face in " << image2_path << "\n"; - return -1; - } - - // Initialize FaceRecognizerSF - Ptr faceRecognizer = FaceRecognizerSF::create(reg_onnx_path, ""); - - - Mat aligned_face1, aligned_face2; - faceRecognizer->alignCrop(image1, faces_1.row(0), aligned_face1); - faceRecognizer->alignCrop(image2, faces_2.row(0), aligned_face2); - - Mat feature1, feature2; - faceRecognizer->feature(aligned_face1, feature1); - feature1 = feature1.clone(); - faceRecognizer->feature(aligned_face2, feature2); - feature2 = feature2.clone(); - - double cos_score = faceRecognizer->match(feature1, feature2, FaceRecognizerSF::DisType::FR_COSINE); - double L2_score = faceRecognizer->match(feature1, feature2, FaceRecognizerSF::DisType::FR_NORM_L2); - - if(cos_score >= cosine_similar_thresh) - { - std::cout << "They have the same identity;"; - } - else - { - std::cout << "They have different identities;"; - } - std::cout << " Cosine Similarity: " << cos_score << ", threshold: " << cosine_similar_thresh << ". (higher value means higher similarity, max 1.0)\n"; - - if(L2_score <= l2norm_similar_thresh) - { - std::cout << "They have the same identity;"; - } - else - { - std::cout << "They have different identities."; - } - std::cout << " NormL2 Distance: " << L2_score << ", threshold: " << l2norm_similar_thresh << ". (lower value means higher similarity, min 0.0)\n"; - - return 0; -} diff --git a/samples/dnn/face_match.py b/samples/dnn/face_match.py deleted file mode 100644 index 916c76abf1..0000000000 --- a/samples/dnn/face_match.py +++ /dev/null @@ -1,57 +0,0 @@ -import argparse - -import numpy as np -import cv2 as cv - -parser = argparse.ArgumentParser() -parser.add_argument('--input1', '-i1', type=str, help='Path to the input image1.') -parser.add_argument('--input2', '-i2', type=str, help='Path to the input image2.') -parser.add_argument('--face_detection_model', '-fd', type=str, help='Path to the face detection model. Download the model at https://github.com/ShiqiYu/libfacedetection.train/tree/master/tasks/task1/onnx.') -parser.add_argument('--face_recognition_model', '-fr', type=str, help='Path to the face recognition model. Download the model at https://drive.google.com/file/d/1ClK9WiB492c5OZFKveF3XiHCejoOxINW/view.') -args = parser.parse_args() - -# Read the input image -img1 = cv.imread(args.input1) -img2 = cv.imread(args.input2) - -# Instantiate face detector and recognizer -detector = cv.FaceDetectorYN.create( - args.face_detection_model, - "", - (img1.shape[1], img1.shape[0]) -) -recognizer = cv.FaceRecognizerSF.create( - args.face_recognition_model, - "" -) - -# Detect face -detector.setInputSize((img1.shape[1], img1.shape[0])) -face1 = detector.detect(img1) -detector.setInputSize((img2.shape[1], img2.shape[0])) -face2 = detector.detect(img2) -assert face1[1].shape[0] > 0, 'Cannot find a face in {}'.format(args.input1) -assert face2[1].shape[0] > 0, 'Cannot find a face in {}'.format(args.input2) - -# Align faces -face1_align = recognizer.alignCrop(img1, face1[1][0]) -face2_align = recognizer.alignCrop(img2, face2[1][0]) - -# Extract features -face1_feature = recognizer.feature(face1_align) -face2_feature = recognizer.feature(face2_align) - -# Calculate distance (0: cosine, 1: L2) -cosine_similarity_threshold = 0.363 -cosine_score = recognizer.match(face1_feature, face2_feature, 0) -msg = 'different identities' -if cosine_score >= cosine_similarity_threshold: - msg = 'the same identity' -print('They have {}. Cosine Similarity: {}, threshold: {} (higher value means higher similarity, max 1.0).'.format(msg, cosine_score, cosine_similarity_threshold)) - -l2_similarity_threshold = 1.128 -l2_score = recognizer.match(face1_feature, face2_feature, 1) -msg = 'different identities' -if l2_score <= l2_similarity_threshold: - msg = 'the same identity' -print('They have {}. NormL2 Distance: {}, threshold: {} (lower value means higher similarity, min 0.0).'.format(msg, l2_score, l2_similarity_threshold)) From f044037ec5d1f4f33d5ad2ab94194721f885396c Mon Sep 17 00:00:00 2001 From: rogday Date: Sun, 28 Nov 2021 19:17:46 +0300 Subject: [PATCH 107/226] Merge pull request #20733 from rogday:argmaxnd Implement ArgMax and ArgMin * add reduceArgMax and reduceArgMin * fix review comments * address review concerns --- modules/core/include/opencv2/core.hpp | 37 +++++- .../core/detail/dispatch_helper.impl.hpp | 49 ++++++++ modules/core/perf/perf_reduce.cpp | 29 +++++ modules/core/src/minmax.cpp | 118 ++++++++++++++++++ modules/core/test/ref_reduce_arg.impl.hpp | 78 ++++++++++++ modules/core/test/test_arithm.cpp | 69 ++++++++++ 6 files changed, 378 insertions(+), 2 deletions(-) create mode 100644 modules/core/include/opencv2/core/detail/dispatch_helper.impl.hpp create mode 100644 modules/core/test/ref_reduce_arg.impl.hpp diff --git a/modules/core/include/opencv2/core.hpp b/modules/core/include/opencv2/core.hpp index 48023844a9..70ea4f8c1f 100644 --- a/modules/core/include/opencv2/core.hpp +++ b/modules/core/include/opencv2/core.hpp @@ -819,12 +819,45 @@ mixChannels , or split . @param minLoc pointer to the returned minimum location (in 2D case); NULL is used if not required. @param maxLoc pointer to the returned maximum location (in 2D case); NULL is used if not required. @param mask optional mask used to select a sub-array. -@sa max, min, compare, inRange, extractImageCOI, mixChannels, split, Mat::reshape +@sa max, min, reduceArgMin, reduceArgMax, compare, inRange, extractImageCOI, mixChannels, split, Mat::reshape */ CV_EXPORTS_W void minMaxLoc(InputArray src, CV_OUT double* minVal, CV_OUT double* maxVal = 0, CV_OUT Point* minLoc = 0, CV_OUT Point* maxLoc = 0, InputArray mask = noArray()); +/** + * @brief Finds indices of min elements along provided axis + * + * @note + * - If input or output array is not continuous, this function will create an internal copy. + * - NaN handling is left unspecified, see patchNaNs(). + * - The returned index is always in bounds of input matrix. + * + * @param src input single-channel array. + * @param dst output array of type CV_32SC1 with the same dimensionality as src, + * except for axis being reduced - it should be set to 1. + * @param lastIndex whether to get the index of first or last occurrence of min. + * @param axis axis to reduce along. + * @sa reduceArgMax, minMaxLoc, min, max, compare, reduce + */ +CV_EXPORTS_W void reduceArgMin(InputArray src, OutputArray dst, int axis, bool lastIndex = false); + +/** + * @brief Finds indices of max elements along provided axis + * + * @note + * - If input or output array is not continuous, this function will create an internal copy. + * - NaN handling is left unspecified, see patchNaNs(). + * - The returned index is always in bounds of input matrix. + * + * @param src input single-channel array. + * @param dst output array of type CV_32SC1 with the same dimensionality as src, + * except for axis being reduced - it should be set to 1. + * @param lastIndex whether to get the index of first or last occurrence of max. + * @param axis axis to reduce along. + * @sa reduceArgMin, minMaxLoc, min, max, compare, reduce + */ +CV_EXPORTS_W void reduceArgMax(InputArray src, OutputArray dst, int axis, bool lastIndex = false); /** @brief Finds the global minimum and maximum in an array @@ -886,7 +919,7 @@ a single row. 1 means that the matrix is reduced to a single column. @param rtype reduction operation that could be one of #ReduceTypes @param dtype when negative, the output vector will have the same type as the input matrix, otherwise, its type will be CV_MAKE_TYPE(CV_MAT_DEPTH(dtype), src.channels()). -@sa repeat +@sa repeat, reduceArgMin, reduceArgMax */ CV_EXPORTS_W void reduce(InputArray src, OutputArray dst, int dim, int rtype, int dtype = -1); diff --git a/modules/core/include/opencv2/core/detail/dispatch_helper.impl.hpp b/modules/core/include/opencv2/core/detail/dispatch_helper.impl.hpp new file mode 100644 index 0000000000..d6ec676922 --- /dev/null +++ b/modules/core/include/opencv2/core/detail/dispatch_helper.impl.hpp @@ -0,0 +1,49 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef OPENCV_CORE_DETAIL_DISPATCH_HELPER_IMPL_HPP +#define OPENCV_CORE_DETAIL_DISPATCH_HELPER_IMPL_HPP + +//! @cond IGNORED + +namespace cv { +namespace detail { + +template class Functor, typename... Args> +static inline void depthDispatch(const int depth, Args&&... args) +{ + switch (depth) + { + case CV_8U: + Functor{}(std::forward(args)...); + break; + case CV_8S: + Functor{}(std::forward(args)...); + break; + case CV_16U: + Functor{}(std::forward(args)...); + break; + case CV_16S: + Functor{}(std::forward(args)...); + break; + case CV_32S: + Functor{}(std::forward(args)...); + break; + case CV_32F: + Functor{}(std::forward(args)...); + break; + case CV_64F: + Functor{}(std::forward(args)...); + break; + case CV_16F: + default: + CV_Error(cv::Error::BadDepth, "Unsupported matrix type."); + }; +} + +}} + +//! @endcond + +#endif //OPENCV_CORE_DETAIL_DISPATCH_HELPER_IMPL_HPP diff --git a/modules/core/perf/perf_reduce.cpp b/modules/core/perf/perf_reduce.cpp index 4fc5aa55f9..8f9c2e8349 100644 --- a/modules/core/perf/perf_reduce.cpp +++ b/modules/core/perf/perf_reduce.cpp @@ -65,4 +65,33 @@ PERF_TEST_P(Size_MatType_ROp, reduceC, SANITY_CHECK(vec, 1); } +typedef tuple Size_MatType_RMode_t; +typedef perf::TestBaseWithParam Size_MatType_RMode; + +PERF_TEST_P(Size_MatType_RMode, DISABLED_reduceArgMinMax, testing::Combine( + testing::Values(TYPICAL_MAT_SIZES), + testing::Values(CV_8U, CV_32F), + testing::Values(0, 1) +) +) +{ + Size srcSize = get<0>(GetParam()); + int matType = get<1>(GetParam()); + int axis = get<2>(GetParam()); + + Mat src(srcSize, matType); + + std::vector dstSize(src.dims); + std::copy(src.size.p, src.size.p + src.dims, dstSize.begin()); + dstSize[axis] = 1; + + Mat dst(dstSize, CV_32S, 0.); + + declare.in(src, WARMUP_RNG).out(dst); + + TEST_CYCLE() cv::reduceArgMin(src, dst, axis, true); + + SANITY_CHECK_NOTHING(); +} + } // namespace diff --git a/modules/core/src/minmax.cpp b/modules/core/src/minmax.cpp index 61bddc3d35..96a7d027ad 100644 --- a/modules/core/src/minmax.cpp +++ b/modules/core/src/minmax.cpp @@ -7,6 +7,9 @@ #include "opencl_kernels_core.hpp" #include "opencv2/core/openvx/ovx_defs.hpp" #include "stat.hpp" +#include "opencv2/core/detail/dispatch_helper.impl.hpp" + +#include #undef HAVE_IPP #undef CV_IPP_RUN_FAST @@ -1570,3 +1573,118 @@ void cv::minMaxLoc( InputArray _img, double* minVal, double* maxVal, if( maxLoc ) std::swap(maxLoc->x, maxLoc->y); } + +enum class ReduceMode +{ + FIRST_MIN = 0, //!< get index of first min occurrence + LAST_MIN = 1, //!< get index of last min occurrence + FIRST_MAX = 2, //!< get index of first max occurrence + LAST_MAX = 3, //!< get index of last max occurrence +}; + +template +struct reduceMinMaxImpl +{ + void operator()(const cv::Mat& src, cv::Mat& dst, ReduceMode mode, const int axis) const + { + switch(mode) + { + case ReduceMode::FIRST_MIN: + reduceMinMaxApply(src, dst, axis); + break; + case ReduceMode::LAST_MIN: + reduceMinMaxApply(src, dst, axis); + break; + case ReduceMode::FIRST_MAX: + reduceMinMaxApply(src, dst, axis); + break; + case ReduceMode::LAST_MAX: + reduceMinMaxApply(src, dst, axis); + break; + } + } + + template class Cmp> + static void reduceMinMaxApply(const cv::Mat& src, cv::Mat& dst, const int axis) + { + Cmp cmp; + + const auto *src_ptr = src.ptr(); + auto *dst_ptr = dst.ptr(); + + const size_t outer_size = src.total(0, axis); + const auto mid_size = static_cast(src.size[axis]); + + const size_t outer_step = src.total(axis); + const size_t dst_step = dst.total(axis); + + const size_t mid_step = src.total(axis + 1); + + for (size_t outer = 0; outer < outer_size; ++outer) + { + const size_t outer_offset = outer * outer_step; + const size_t dst_offset = outer * dst_step; + for (size_t mid = 0; mid != mid_size; ++mid) + { + const size_t src_offset = outer_offset + mid * mid_step; + for (size_t inner = 0; inner < mid_step; inner++) + { + int32_t& index = dst_ptr[dst_offset + inner]; + + const size_t prev = outer_offset + index * mid_step + inner; + const size_t curr = src_offset + inner; + + if (cmp(src_ptr[curr], src_ptr[prev])) + { + index = static_cast(mid); + } + } + } + } + } +}; + +static void reduceMinMax(cv::InputArray src, cv::OutputArray dst, ReduceMode mode, int axis) +{ + CV_INSTRUMENT_REGION(); + + cv::Mat srcMat = src.getMat(); + axis = (axis + srcMat.dims) % srcMat.dims; + CV_Assert(srcMat.channels() == 1 && axis >= 0 && axis < srcMat.dims); + + std::vector sizes(srcMat.dims); + std::copy(srcMat.size.p, srcMat.size.p + srcMat.dims, sizes.begin()); + sizes[axis] = 1; + + dst.create(srcMat.dims, sizes.data(), CV_32SC1); // indices + cv::Mat dstMat = dst.getMat(); + dstMat.setTo(cv::Scalar::all(0)); + + if (!srcMat.isContinuous()) + { + srcMat = srcMat.clone(); + } + + bool needs_copy = !dstMat.isContinuous(); + if (needs_copy) + { + dstMat = dstMat.clone(); + } + + cv::detail::depthDispatch(srcMat.depth(), srcMat, dstMat, mode, axis); + + if (needs_copy) + { + dstMat.copyTo(dst); + } +} + +void cv::reduceArgMin(InputArray src, OutputArray dst, int axis, bool lastIndex) +{ + reduceMinMax(src, dst, lastIndex ? ReduceMode::LAST_MIN : ReduceMode::FIRST_MIN, axis); +} + +void cv::reduceArgMax(InputArray src, OutputArray dst, int axis, bool lastIndex) +{ + reduceMinMax(src, dst, lastIndex ? ReduceMode::LAST_MAX : ReduceMode::FIRST_MAX, axis); +} diff --git a/modules/core/test/ref_reduce_arg.impl.hpp b/modules/core/test/ref_reduce_arg.impl.hpp new file mode 100644 index 0000000000..22b7c2b7a6 --- /dev/null +++ b/modules/core/test/ref_reduce_arg.impl.hpp @@ -0,0 +1,78 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef OPENCV_TEST_REF_REDUCE_ARG_HPP +#define OPENCV_TEST_REF_REDUCE_ARG_HPP + +#include "opencv2/core/detail/dispatch_helper.impl.hpp" + +#include +#include + +namespace cvtest { + +template +struct reduceMinMaxImpl +{ + void operator()(const cv::Mat& src, cv::Mat& dst, const int axis) const + { + Cmp cmp; + std::vector sizes(src.dims); + std::copy(src.size.p, src.size.p + src.dims, sizes.begin()); + + std::vector idx(sizes.size(), cv::Range(0, 1)); + idx[axis] = cv::Range::all(); + const int n = std::accumulate(begin(sizes), end(sizes), 1, std::multiplies()); + const std::vector newShape{1, src.size[axis]}; + for (int i = 0; i < n ; ++i) + { + cv::Mat sub = src(idx); + + auto begin = sub.begin(); + auto it = std::min_element(begin, sub.end(), cmp); + *dst(idx).ptr() = static_cast(std::distance(begin, it)); + + for (int j = static_cast(idx.size()) - 1; j >= 0; --j) + { + if (j == axis) + { + continue; + } + const int old_s = idx[j].start; + const int new_s = (old_s + 1) % sizes[j]; + if (new_s > old_s) + { + idx[j] = cv::Range(new_s, new_s + 1); + break; + } + idx[j] = cv::Range(0, 1); + } + } + } +}; + +template class Cmp> +struct MinMaxReducer{ + template + using Impl = reduceMinMaxImpl, T>; + + static void reduce(const Mat& src, Mat& dst, int axis) + { + axis = (axis + src.dims) % src.dims; + CV_Assert(src.channels() == 1 && axis >= 0 && axis < src.dims); + + std::vector sizes(src.dims); + std::copy(src.size.p, src.size.p + src.dims, sizes.begin()); + sizes[axis] = 1; + + dst.create(sizes, CV_32SC1); // indices + dst.setTo(cv::Scalar::all(0)); + + cv::detail::depthDispatch(src.depth(), src, dst, axis); + } +}; + +} + +#endif //OPENCV_TEST_REF_REDUCE_ARG_HPP diff --git a/modules/core/test/test_arithm.cpp b/modules/core/test/test_arithm.cpp index 9e8e242d60..4a9522d3d1 100644 --- a/modules/core/test/test_arithm.cpp +++ b/modules/core/test/test_arithm.cpp @@ -2,6 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. #include "test_precomp.hpp" +#include "ref_reduce_arg.impl.hpp" namespace opencv_test { namespace { @@ -1387,6 +1388,73 @@ struct MinMaxLocOp : public BaseElemWiseOp } }; +struct reduceArgMinMaxOp : public BaseElemWiseOp +{ + reduceArgMinMaxOp() : BaseElemWiseOp(1, FIX_ALPHA+FIX_BETA+FIX_GAMMA, 1, 1, Scalar::all(0)) + { + context = ARITHM_MAX_NDIMS*2 + 2; + }; + int getRandomType(RNG& rng) override + { + return cvtest::randomType(rng, _OutputArray::DEPTH_MASK_ALL_BUT_8S, 1, 1); + } + void getRandomSize(RNG& rng, vector& size) override + { + cvtest::randomSize(rng, 2, ARITHM_MAX_NDIMS, 6, size); + } + void generateScalars(int depth, RNG& rng) override + { + BaseElemWiseOp::generateScalars(depth, rng); + isLast = (randInt(rng) % 2 == 0); + isMax = (randInt(rng) % 2 == 0); + axis = randInt(rng); + } + int getAxis(const Mat& src) const + { + int dims = src.dims; + return static_cast(axis % (2 * dims)) - dims; // [-dims; dims - 1] + } + void op(const vector& src, Mat& dst, const Mat&) override + { + const Mat& inp = src[0]; + const int axis_ = getAxis(inp); + if (isMax) + { + cv::reduceArgMax(inp, dst, axis_, isLast); + } + else + { + cv::reduceArgMin(inp, dst, axis_, isLast); + } + } + void refop(const vector& src, Mat& dst, const Mat&) override + { + const Mat& inp = src[0]; + const int axis_ = getAxis(inp); + + if (!isLast && !isMax) + { + cvtest::MinMaxReducer::reduce(inp, dst, axis_); + } + else if (!isLast && isMax) + { + cvtest::MinMaxReducer::reduce(inp, dst, axis_); + } + else if (isLast && !isMax) + { + cvtest::MinMaxReducer::reduce(inp, dst, axis_); + } + else + { + cvtest::MinMaxReducer::reduce(inp, dst, axis_); + } + } + + bool isLast; + bool isMax; + uint32_t axis; +}; + typedef Ptr ElemWiseOpPtr; class ElemWiseTest : public ::testing::TestWithParam {}; @@ -1492,6 +1560,7 @@ INSTANTIATE_TEST_CASE_P(Core_MeanStdDev, ElemWiseTest, ::testing::Values(ElemWis INSTANTIATE_TEST_CASE_P(Core_Sum, ElemWiseTest, ::testing::Values(ElemWiseOpPtr(new SumOp))); INSTANTIATE_TEST_CASE_P(Core_Norm, ElemWiseTest, ::testing::Values(ElemWiseOpPtr(new NormOp))); INSTANTIATE_TEST_CASE_P(Core_MinMaxLoc, ElemWiseTest, ::testing::Values(ElemWiseOpPtr(new MinMaxLocOp))); +INSTANTIATE_TEST_CASE_P(Core_reduceArgMinMax, ElemWiseTest, ::testing::Values(ElemWiseOpPtr(new reduceArgMinMaxOp))); INSTANTIATE_TEST_CASE_P(Core_CartToPolarToCart, ElemWiseTest, ::testing::Values(ElemWiseOpPtr(new CartToPolarToCartOp))); From d58b5ef74b91ba68ef89f9ce8be8725d0dfcaa0d Mon Sep 17 00:00:00 2001 From: Anna Khakimova Date: Mon, 29 Nov 2021 14:20:53 +0300 Subject: [PATCH 108/226] Merge pull request #21119 from anna-khakimova:ak/simd_addc * GAPI Fluid: SIMD for AddC kernel * Final version * Applied comments. --- modules/gapi/include/opencv2/gapi/core.hpp | 1 + .../gapi/perf/common/gapi_core_perf_tests.hpp | 2 +- .../perf/common/gapi_core_perf_tests_inl.hpp | 16 +- .../perf/cpu/gapi_core_perf_tests_cpu.cpp | 3 +- .../perf/cpu/gapi_core_perf_tests_fluid.cpp | 11 +- .../perf/gpu/gapi_core_perf_tests_gpu.cpp | 3 +- .../gapi/src/backends/fluid/gfluidcore.cpp | 150 ++++++---- .../fluid/gfluidcore_func.dispatch.cpp | 27 ++ .../src/backends/fluid/gfluidcore_func.hpp | 23 ++ .../backends/fluid/gfluidcore_func.simd.hpp | 265 +++++++++++++++++- 10 files changed, 424 insertions(+), 77 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/core.hpp b/modules/gapi/include/opencv2/gapi/core.hpp index 35f875a809..052c6a944c 100644 --- a/modules/gapi/include/opencv2/gapi/core.hpp +++ b/modules/gapi/include/opencv2/gapi/core.hpp @@ -57,6 +57,7 @@ namespace core { G_TYPED_KERNEL(GAddC, , "org.opencv.core.math.addC") { static GMatDesc outMeta(GMatDesc a, GScalarDesc, int ddepth) { + GAPI_Assert(a.chan <= 4); return a.withDepth(ddepth); } }; diff --git a/modules/gapi/perf/common/gapi_core_perf_tests.hpp b/modules/gapi/perf/common/gapi_core_perf_tests.hpp index f5916a6aaf..39557f1acb 100644 --- a/modules/gapi/perf/common/gapi_core_perf_tests.hpp +++ b/modules/gapi/perf/common/gapi_core_perf_tests.hpp @@ -28,7 +28,7 @@ namespace opencv_test //------------------------------------------------------------------------------ class AddPerfTest : public TestPerfParams> {}; - class AddCPerfTest : public TestPerfParams> {}; + class AddCPerfTest : public TestPerfParams> {}; class SubPerfTest : public TestPerfParams> {}; class SubCPerfTest : public TestPerfParams> {}; class SubRCPerfTest : public TestPerfParams> {}; diff --git a/modules/gapi/perf/common/gapi_core_perf_tests_inl.hpp b/modules/gapi/perf/common/gapi_core_perf_tests_inl.hpp index fbbda1a31d..b0568f9bae 100644 --- a/modules/gapi/perf/common/gapi_core_perf_tests_inl.hpp +++ b/modules/gapi/perf/common/gapi_core_perf_tests_inl.hpp @@ -61,10 +61,13 @@ PERF_TEST_P_(AddPerfTest, TestPerformance) PERF_TEST_P_(AddCPerfTest, TestPerformance) { - Size sz = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - int dtype = get<2>(GetParam()); - cv::GCompileArgs compile_args = get<3>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + int dtype = -1; + cv::GCompileArgs compile_args; + + std::tie(cmpF, sz, type, dtype, compile_args) = GetParam(); initMatsRandU(type, sz, dtype, false); @@ -88,8 +91,9 @@ PERF_TEST_P_(AddCPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - // FIXIT unrealiable check: EXPECT_EQ(0, cv::countNonZero(out_mat_gapi != out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } diff --git a/modules/gapi/perf/cpu/gapi_core_perf_tests_cpu.cpp b/modules/gapi/perf/cpu/gapi_core_perf_tests_cpu.cpp index 09196fd24f..d7ead88327 100644 --- a/modules/gapi/perf/cpu/gapi_core_perf_tests_cpu.cpp +++ b/modules/gapi/perf/cpu/gapi_core_perf_tests_cpu.cpp @@ -22,7 +22,8 @@ INSTANTIATE_TEST_CASE_P(AddPerfTestCPU, AddPerfTest, Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(AddCPerfTestCPU, AddCPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), Values(-1, CV_8U, CV_16U, CV_32F), Values(cv::compile_args(CORE_CPU)))); diff --git a/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp b/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp index 6c80231f32..a367937520 100644 --- a/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp +++ b/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp @@ -18,11 +18,12 @@ INSTANTIATE_TEST_CASE_P(AddPerfTestFluid, AddPerfTest, Values(-1, CV_8U, CV_32F), Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(AddCPerfTestFluid, AddCPerfTest, -// Combine(Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), -// Values(-1, CV_8U, CV_16U, CV_32F), -// Values(cv::compile_args(CORE_FLUID)))); + INSTANTIATE_TEST_CASE_P(AddCPerfTestFluid, AddCPerfTest, + Combine(Values(Tolerance_FloatRel_IntAbs(1e-6, 1).to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(-1, CV_8U, CV_16U, CV_16S, CV_32F), + Values(cv::compile_args(CORE_FLUID)))); INSTANTIATE_TEST_CASE_P(SubPerfTestFluid, SubPerfTest, Combine(Values(szSmall128, szVGA, sz720p, sz1080p), diff --git a/modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp b/modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp index 0b260bf553..d1d6deff2d 100644 --- a/modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp +++ b/modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp @@ -20,7 +20,8 @@ INSTANTIATE_TEST_CASE_P(AddPerfTestGPU, AddPerfTest, Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(AddCPerfTestGPU, AddCPerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), Values( -1, CV_8U, CV_16U, CV_32F ), Values(cv::compile_args(CORE_GPU)))); diff --git a/modules/gapi/src/backends/fluid/gfluidcore.cpp b/modules/gapi/src/backends/fluid/gfluidcore.cpp index 7a3d90acc7..16a87ea314 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore.cpp +++ b/modules/gapi/src/backends/fluid/gfluidcore.cpp @@ -645,8 +645,8 @@ CV_ALWAYS_INLINE int sub_simd(const SRC in1[], const SRC in2[], DST out[], int l #endif // CV_SIMD template -static void run_arithm(Buffer &dst, const View &src1, const View &src2, Arithm arithm, - double scale=1) +static CV_ALWAYS_INLINE void run_arithm(Buffer &dst, const View &src1, const View &src2, + Arithm arithm, double scale=1) { static_assert(std::is_same::value, "wrong types"); @@ -844,19 +844,15 @@ GAPI_FLUID_KERNEL(GFluidAbsDiff, cv::gapi::core::GAbsDiff, false) // //-------------------------------------- -static inline v_uint16x8 v_add_16u(const v_uint16x8 &x, const v_uint16x8 &y) { return x + y; } static inline v_uint16x8 v_sub_16u(const v_uint16x8 &x, const v_uint16x8 &y) { return x - y; } static inline v_uint16x8 v_subr_16u(const v_uint16x8 &x, const v_uint16x8 &y) { return y - x; } -static inline v_float32x4 v_add_32f(const v_float32x4 &x, const v_float32x4 &y) { return x + y; } static inline v_float32x4 v_sub_32f(const v_float32x4 &x, const v_float32x4 &y) { return x - y; } static inline v_float32x4 v_subr_32f(const v_float32x4 &x, const v_float32x4 &y) { return y - x; } -static inline int s_add_8u(uchar x, uchar y) { return x + y; } static inline int s_sub_8u(uchar x, uchar y) { return x - y; } static inline int s_subr_8u(uchar x, uchar y) { return y - x; } -static inline float s_add_32f(float x, float y) { return x + y; } static inline float s_sub_32f(float x, float y) { return x - y; } static inline float s_subr_32f(float x, float y) { return y - x; } @@ -946,11 +942,6 @@ static void run_arithm_s1(uchar out[], const float in[], int width, const float } } -static void run_arithm_s_add3(uchar out[], const uchar in[], int width, const uchar scalar[]) -{ - run_arithm_s3(out, in, width, scalar, v_add_16u, s_add_8u); -} - static void run_arithm_s_sub3(uchar out[], const uchar in[], int width, const uchar scalar[]) { run_arithm_s3(out, in, width, scalar, v_sub_16u, s_sub_8u); @@ -961,11 +952,6 @@ static void run_arithm_s_subr3(uchar out[], const uchar in[], int width, const u run_arithm_s3(out, in, width, scalar, v_subr_16u, s_subr_8u); // reverse: subr } -static void run_arithm_s_add1(uchar out[], const float in[], int width, const float scalar[]) -{ - run_arithm_s1(out, in, width, scalar, v_add_32f, s_add_32f); -} - static void run_arithm_s_sub1(uchar out[], const float in[], int width, const float scalar[]) { run_arithm_s1(out, in, width, scalar, v_sub_32f, s_sub_32f); @@ -1279,8 +1265,8 @@ static void run_absdiffc(Buffer &dst, const View &src, const float scalar[]) } template -static void run_arithm_s(Buffer &dst, const View &src, const float scalar[4], Arithm arithm, - float scale=1) +CV_ALWAYS_INLINE void run_arithm_s(Buffer &dst, const View &src, const float scalar[], + Arithm arithm, float scale=1) { const auto *in = src.InLine(0); auto *out = dst.OutLine(); @@ -1288,48 +1274,45 @@ static void run_arithm_s(Buffer &dst, const View &src, const float scalar[4], Ar int width = dst.length(); int chan = dst.meta().chan; - // What if we cast the scalar into the SRC type? - const SRC myscal[4] = { static_cast(scalar[0]), static_cast(scalar[1]), - static_cast(scalar[2]), static_cast(scalar[3]) }; - bool usemyscal = (myscal[0] == scalar[0]) && (myscal[1] == scalar[1]) && - (myscal[2] == scalar[2]) && (myscal[3] == scalar[3]); - switch (arithm) { case ARITHM_ADD: - if (usemyscal) - { - if (std::is_same::value && - std::is_same::value && - chan == 3) - run_arithm_s_add3((uchar*)out, (const uchar*)in, width, (const uchar*)myscal); - else if (std::is_same::value && - std::is_same::value && - chan == 1) - run_arithm_s_add1((uchar*)out, (const float*)in, width, (const float*)myscal); - else - run_arithm_s(out, in, width, chan, myscal, add); - } - else - run_arithm_s(out, in, width, chan, scalar, add); + { + int w = 0; +#if CV_SIMD + w = addc_simd(in, scalar, out, width, chan); +#endif + + for (; w < width * chan; ++w) + out[w] = add(in[w], scalar[w % chan]); + break; + } case ARITHM_SUBTRACT: + { + // What if we cast the scalar into the SRC type? + const SRC myscal[4] = { static_cast(scalar[0]), static_cast(scalar[1]), + static_cast(scalar[2]), static_cast(scalar[3]) }; + bool usemyscal = (myscal[0] == scalar[0]) && (myscal[1] == scalar[1]) && + (myscal[2] == scalar[2]) && (myscal[3] == scalar[3]); + if (usemyscal) { - if (std::is_same::value && - std::is_same::value && + if (std::is_same::value && + std::is_same::value && chan == 3) run_arithm_s_sub3((uchar*)out, (const uchar*)in, width, (const uchar*)myscal); - else if (std::is_same::value && - std::is_same::value && - chan == 1) + else if (std::is_same::value && + std::is_same::value && + chan == 1) run_arithm_s_sub1((uchar*)out, (const float*)in, width, (const float*)myscal); else - run_arithm_s(out, in, width, chan, myscal, sub); + run_arithm_s(out, in, width, chan, myscal, sub); } else - run_arithm_s(out, in, width, chan, scalar, sub); + run_arithm_s(out, in, width, chan, scalar, sub); break; + } // TODO: optimize miltiplication and division case ARITHM_MULTIPLY: for (int w=0; w < width; w++) @@ -1433,30 +1416,75 @@ GAPI_FLUID_KERNEL(GFluidAbsDiffC, cv::gapi::core::GAbsDiffC, true) } }; -GAPI_FLUID_KERNEL(GFluidAddC, cv::gapi::core::GAddC, false) +GAPI_FLUID_KERNEL(GFluidAddC, cv::gapi::core::GAddC, true) { static const int Window = 1; - static void run(const View &src, const cv::Scalar &_scalar, int /*dtype*/, Buffer &dst) + static void run(const View &src, const cv::Scalar &_scalar, int /*dtype*/, Buffer &dst, Buffer &scratch) { - const float scalar[4] = { - static_cast(_scalar[0]), - static_cast(_scalar[1]), - static_cast(_scalar[2]), - static_cast(_scalar[3]) - }; + GAPI_Assert(src.meta().chan <= 4); + + if (dst.y() == 0) + { + const int chan = src.meta().chan; + float* sc = scratch.OutLine(); + + for (int i = 0; i < scratch.length(); ++i) + sc[i] = static_cast(_scalar[i % chan]); + } + + const float* scalar = scratch.OutLine(); // DST SRC OP __VA_ARGS__ - UNARY_(uchar , uchar , run_arithm_s, dst, src, scalar, ARITHM_ADD); - UNARY_(uchar , short, run_arithm_s, dst, src, scalar, ARITHM_ADD); - UNARY_(uchar , float, run_arithm_s, dst, src, scalar, ARITHM_ADD); - UNARY_( short, short, run_arithm_s, dst, src, scalar, ARITHM_ADD); - UNARY_( float, uchar , run_arithm_s, dst, src, scalar, ARITHM_ADD); - UNARY_( float, short, run_arithm_s, dst, src, scalar, ARITHM_ADD); - UNARY_( float, float, run_arithm_s, dst, src, scalar, ARITHM_ADD); + UNARY_(uchar, uchar, run_arithm_s, dst, src, scalar, ARITHM_ADD); + UNARY_(uchar, ushort, run_arithm_s, dst, src, scalar, ARITHM_ADD); + UNARY_(uchar, short, run_arithm_s, dst, src, scalar, ARITHM_ADD); + UNARY_(uchar, float, run_arithm_s, dst, src, scalar, ARITHM_ADD); + UNARY_(ushort, ushort, run_arithm_s, dst, src, scalar, ARITHM_ADD); + UNARY_(ushort, short, run_arithm_s, dst, src, scalar, ARITHM_ADD); + UNARY_(ushort, uchar, run_arithm_s, dst, src, scalar, ARITHM_ADD); + UNARY_(ushort, float, run_arithm_s, dst, src, scalar, ARITHM_ADD); + UNARY_(short, short, run_arithm_s, dst, src, scalar, ARITHM_ADD); + UNARY_(short, ushort, run_arithm_s, dst, src, scalar, ARITHM_ADD); + UNARY_(short, uchar, run_arithm_s, dst, src, scalar, ARITHM_ADD); + UNARY_(short, float, run_arithm_s, dst, src, scalar, ARITHM_ADD); + UNARY_(float, uchar, run_arithm_s, dst, src, scalar, ARITHM_ADD); + UNARY_(float, ushort, run_arithm_s, dst, src, scalar, ARITHM_ADD); + UNARY_(float, short, run_arithm_s, dst, src, scalar, ARITHM_ADD); + UNARY_(float, float, run_arithm_s, dst, src, scalar, ARITHM_ADD); CV_Error(cv::Error::StsBadArg, "unsupported combination of types"); } + + static void initScratch(const GMatDesc&, const GScalarDesc&, int, Buffer& scratch) + { +#if CV_SIMD + // 512 bits / 32 bits = 16 elements of float32 can contain a AVX 512 SIMD vector. + constexpr int maxNlanes = 16; + + // +2 is offset for 3-channel case. + // Offset is need to right load coefficients from scalar array to SIMD vectors for 3-channel case. + // Scalar array looks like: scalar[] = {C1, C2, C3, C1, C2, C3, ...} + // The first scalar SIMD vector should looks like: + // C1 C2 C3 C1 + // The second: + // C2 C3 C1 C2 + // The third: + // C3 C1 C2 C3 + constexpr int offset = 2; + constexpr int buflen = maxNlanes + offset; +#else + constexpr int buflen = 4; +#endif + cv::Size bufsize(buflen, 1); + GMatDesc bufdesc = { CV_32F, 1, bufsize }; + Buffer buffer(bufdesc); + scratch = std::move(buffer); + } + + static void resetScratch(Buffer& /* scratch */) + { + } }; GAPI_FLUID_KERNEL(GFluidSubC, cv::gapi::core::GSubC, false) diff --git a/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp b/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp index 297c065427..b6842e2390 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp +++ b/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp @@ -85,6 +85,33 @@ MUL_SIMD(float, float) #undef MUL_SIMD +#define ADDC_SIMD(SRC, DST) \ +int addc_simd(const SRC in[], const float scalar[], DST out[], \ + const int width, const int chan) \ +{ \ + CV_CPU_DISPATCH(addc_simd, (in, scalar, out, width, chan), \ + CV_CPU_DISPATCH_MODES_ALL); \ +} + +ADDC_SIMD(uchar, uchar) +ADDC_SIMD(ushort, uchar) +ADDC_SIMD(short, uchar) +ADDC_SIMD(float, uchar) +ADDC_SIMD(short, short) +ADDC_SIMD(ushort, short) +ADDC_SIMD(uchar, short) +ADDC_SIMD(float, short) +ADDC_SIMD(ushort, ushort) +ADDC_SIMD(uchar, ushort) +ADDC_SIMD(short, ushort) +ADDC_SIMD(float, ushort) +ADDC_SIMD(uchar, float) +ADDC_SIMD(ushort, float) +ADDC_SIMD(short, float) +ADDC_SIMD(float, float) + +#undef ADDC_SIMD + } // namespace fluid } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/backends/fluid/gfluidcore_func.hpp b/modules/gapi/src/backends/fluid/gfluidcore_func.hpp index 3ae41c6aef..ba48f7a621 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore_func.hpp +++ b/modules/gapi/src/backends/fluid/gfluidcore_func.hpp @@ -60,6 +60,29 @@ MUL_SIMD(float, float) #undef MUL_SIMD +#define ADDC_SIMD(SRC, DST) \ +int addc_simd(const SRC in[], const float scalar[], DST out[], \ + const int width, const int chan); + +ADDC_SIMD(uchar, uchar) +ADDC_SIMD(ushort, uchar) +ADDC_SIMD(short, uchar) +ADDC_SIMD(float, uchar) +ADDC_SIMD(short, short) +ADDC_SIMD(ushort, short) +ADDC_SIMD(uchar, short) +ADDC_SIMD(float, short) +ADDC_SIMD(ushort, ushort) +ADDC_SIMD(uchar, ushort) +ADDC_SIMD(short, ushort) +ADDC_SIMD(float, ushort) +ADDC_SIMD(uchar, float) +ADDC_SIMD(ushort, float) +ADDC_SIMD(short, float) +ADDC_SIMD(float, float) + +#undef ADDC_SIMD + } // namespace fluid } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp b/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp index 5139d54745..071c05633a 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp +++ b/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp @@ -81,6 +81,29 @@ MUL_SIMD(float, float) #undef MUL_SIMD +#define ADDC_SIMD(SRC, DST) \ +int addc_simd(const SRC in[], const float scalar[], DST out[], \ + const int width, const int chan); + +ADDC_SIMD(uchar, uchar) +ADDC_SIMD(ushort, uchar) +ADDC_SIMD(short, uchar) +ADDC_SIMD(float, uchar) +ADDC_SIMD(short, short) +ADDC_SIMD(ushort, short) +ADDC_SIMD(uchar, short) +ADDC_SIMD(float, short) +ADDC_SIMD(ushort, ushort) +ADDC_SIMD(uchar, ushort) +ADDC_SIMD(short, ushort) +ADDC_SIMD(float, ushort) +ADDC_SIMD(uchar, float) +ADDC_SIMD(ushort, float) +ADDC_SIMD(short, float) +ADDC_SIMD(float, float) + +#undef ADDC_SIMD + #ifndef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY struct scale_tag {}; @@ -95,6 +118,7 @@ using vector_type_of_t = typename vector_type_of::type; template<> struct vector_type_of { using type = v_uint8; }; template<> struct vector_type_of { using type = v_uint16; }; template<> struct vector_type_of { using type = v_int16; }; +template<> struct vector_type_of { using type = v_float32; }; CV_ALWAYS_INLINE v_float32 vg_load_f32(const float* in) { @@ -136,12 +160,12 @@ CV_ALWAYS_INLINE v_float32 div_op(not_scale_tag, const v_float32& a, const v_flo return a / div; } -CV_ALWAYS_INLINE void v_store_i16(short* dst, v_int32& res1, v_int32& res2) +CV_ALWAYS_INLINE void v_store_i16(short* dst, const v_int32& res1, const v_int32& res2) { vx_store(dst, v_pack(res1, res2)); } -CV_ALWAYS_INLINE void v_store_i16(ushort* dst, v_int32& res1, v_int32& res2) +CV_ALWAYS_INLINE void v_store_i16(ushort* dst, const v_int32& res1, const v_int32& res2) { vx_store(dst, v_pack_u(res1, res2)); } @@ -821,6 +845,243 @@ MUL_SIMD(float, float) #undef MUL_SIMD +//------------------------- +// +// Fluid kernels: AddC +// +//------------------------- + +CV_ALWAYS_INLINE void addc_pack_store_c3(short* outx, const v_int32& c1, + const v_int32& c2, const v_int32& c3, + const v_int32& c4, const v_int32& c5, + const v_int32& c6) +{ + constexpr int nlanes = v_int16::nlanes; + vx_store(outx, v_pack(c1, c2)); + vx_store(&outx[nlanes], v_pack(c3, c4)); + vx_store(&outx[2*nlanes], v_pack(c5, c6)); +} + +CV_ALWAYS_INLINE void addc_pack_store_c3(ushort* outx, const v_int32& c1, + const v_int32& c2, const v_int32& c3, + const v_int32& c4, const v_int32& c5, + const v_int32& c6) +{ + constexpr int nlanes = v_uint16::nlanes; + vx_store(outx, v_pack_u(c1, c2)); + vx_store(&outx[nlanes], v_pack_u(c3, c4)); + vx_store(&outx[2*nlanes], v_pack_u(c5, c6)); +} + +template +CV_ALWAYS_INLINE +typename std::enable_if<(std::is_same::value || + std::is_same::value), void>::type +addc_simd_common_impl(const SRC* inx, DST* outx, const v_float32& sc, const int nlanes) +{ + v_float32 a1 = vg_load_f32(inx); + v_float32 a2 = vg_load_f32(&inx[nlanes/2]); + + v_store_i16(outx, v_round(a1 + sc), v_round(a2 + sc)); +} + +//------------------------------------------------------------------------------------------------- + +template +CV_ALWAYS_INLINE void addc_simd_common_impl(const SRC* inx, uchar* outx, const v_float32& sc, const int nlanes) +{ + v_float32 a1 = vg_load_f32(inx); + v_float32 a2 = vg_load_f32(&inx[nlanes/4]); + v_float32 a3 = vg_load_f32(&inx[nlanes/2]); + v_float32 a4 = vg_load_f32(&inx[3 * nlanes/4]); + + vx_store(outx, v_pack_u(v_pack(v_round(a1 + sc), + v_round(a2 + sc)), + v_pack(v_round(a3 + sc), + v_round(a4 + sc)))); +} + +//------------------------------------------------------------------------------------------------- + +template +CV_ALWAYS_INLINE void addc_simd_common_impl(const SRC* inx, float* outx, const v_float32& sc, const int) +{ + v_float32 a1 = vg_load_f32(inx); + vx_store(outx, a1 + sc); +} + +//------------------------------------------------------------------------------------------------- + +template +CV_ALWAYS_INLINE +typename std::enable_if::value || + std::is_same::value, void>::type +addc_simd_c3_impl(const SRC* inx, DST* outx, const v_float32& s1, const v_float32& s2, + const v_float32& s3, const int nlanes) +{ + v_float32 a1 = vg_load_f32(inx); + v_float32 a2 = vg_load_f32(&inx[nlanes / 2]); + v_float32 a3 = vg_load_f32(&inx[nlanes]); + v_float32 a4 = vg_load_f32(&inx[3 * nlanes / 2]); + v_float32 a5 = vg_load_f32(&inx[2 * nlanes]); + v_float32 a6 = vg_load_f32(&inx[5 * nlanes / 2]); + + addc_pack_store_c3(outx, v_round(a1 + s1), + v_round(a2 + s2), + v_round(a3 + s3), + v_round(a4 + s1), + v_round(a5 + s2), + v_round(a6 + s3)); +} + +//------------------------------------------------------------------------------------------------- + +template +CV_ALWAYS_INLINE void addc_simd_c3_impl(const SRC* inx, uchar* outx, + const v_float32& s1, const v_float32& s2, + const v_float32& s3, const int nlanes) +{ + vx_store(outx, + v_pack_u(v_pack(v_round(vg_load_f32(inx) + s1), + v_round(vg_load_f32(&inx[nlanes/4]) + s2)), + v_pack(v_round(vg_load_f32(&inx[nlanes/2]) + s3), + v_round(vg_load_f32(&inx[3*nlanes/4]) + s1)))); + + vx_store(&outx[nlanes], + v_pack_u(v_pack(v_round(vg_load_f32(&inx[nlanes]) + s2), + v_round(vg_load_f32(&inx[5*nlanes/4]) + s3)), + v_pack(v_round(vg_load_f32(&inx[3*nlanes/2]) + s1), + v_round(vg_load_f32(&inx[7*nlanes/4]) + s2)))); + + vx_store(&outx[2 * nlanes], + v_pack_u(v_pack(v_round(vg_load_f32(&inx[2*nlanes]) + s3), + v_round(vg_load_f32(&inx[9*nlanes/4]) + s1)), + v_pack(v_round(vg_load_f32(&inx[5*nlanes/2]) + s2), + v_round(vg_load_f32(&inx[11*nlanes/4]) + s3)))); +} + +//------------------------------------------------------------------------------------------------- + +template +CV_ALWAYS_INLINE void addc_simd_c3_impl(const SRC* in, float* out, + const v_float32& s1, const v_float32& s2, + const v_float32& s3, const int nlanes) +{ + v_float32 a1 = vg_load_f32(in); + v_float32 a2 = vg_load_f32(&in[nlanes]); + v_float32 a3 = vg_load_f32(&in[2*nlanes]); + + vx_store(out, a1 + s1); + vx_store(&out[nlanes], a2 + s2); + vx_store(&out[2*nlanes], a3 + s3); +} + +//------------------------------------------------------------------------------------------------- + +template +CV_ALWAYS_INLINE int addc_simd_c3(const SRC in[], const float scalar[], DST out[], const int length) +{ + constexpr int chan = 3; + constexpr int nlanes = vector_type_of_t::nlanes; + constexpr int lanes = chan * nlanes; + + if (length < lanes) + return 0; + + v_float32 s1 = vx_load(scalar); +#if CV_SIMD_WIDTH == 32 + v_float32 s2 = vx_load(&scalar[2]); + v_float32 s3 = vx_load(&scalar[1]); +#else + v_float32 s2 = vx_load(&scalar[1]); + v_float32 s3 = vx_load(&scalar[2]); +#endif + + int x = 0; + for (;;) + { + for (; x <= length - lanes; x += lanes) + { + addc_simd_c3_impl(&in[x], &out[x], s1, s2, s3, nlanes); + } + + if (x < length) + { + x = length - lanes; + continue; // process unaligned tail + } + break; + } + return x; +} + +template +CV_ALWAYS_INLINE int addc_simd_common(const SRC in[], const float scalar[], DST out[], const int length) +{ + constexpr int nlanes = vector_type_of_t::nlanes; + + if (length < nlanes) + return 0; + + v_float32 sc = vx_load(scalar); + + int x = 0; + for (;;) + { + for (; x <= length - nlanes; x += nlanes) + { + addc_simd_common_impl(&in[x], &out[x], sc, nlanes); + } + + if (x < length) + { + x = length - nlanes; + continue; // process unaligned tail + } + break; + } + return x; +} + +#define ADDC_SIMD(SRC, DST) \ +int addc_simd(const SRC in[], const float scalar[], DST out[], \ + const int width, const int chan) \ +{ \ + const int length = width * chan; \ + switch (chan) \ + { \ + case 1: \ + case 2: \ + case 4: \ + return addc_simd_common(in, scalar, out, length); \ + case 3: \ + return addc_simd_c3(in, scalar, out, length); \ + default: \ + GAPI_Assert(chan <= 4); \ + break; \ + } \ + return 0; \ +} + +ADDC_SIMD(uchar, uchar) +ADDC_SIMD(ushort, uchar) +ADDC_SIMD(short, uchar) +ADDC_SIMD(float, uchar) +ADDC_SIMD(short, short) +ADDC_SIMD(ushort, short) +ADDC_SIMD(uchar, short) +ADDC_SIMD(float, short) +ADDC_SIMD(ushort, ushort) +ADDC_SIMD(uchar, ushort) +ADDC_SIMD(short, ushort) +ADDC_SIMD(float, ushort) +ADDC_SIMD(uchar, float) +ADDC_SIMD(ushort, float) +ADDC_SIMD(short, float) +ADDC_SIMD(float, float) + +#undef ADDC_SIMD + #endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY CV_CPU_OPTIMIZATION_NAMESPACE_END From 68667d6057f6cd3fed80306c8845e03771a3f70a Mon Sep 17 00:00:00 2001 From: utibenkei Date: Mon, 29 Nov 2021 22:43:29 +0900 Subject: [PATCH 109/226] fix_android_ndk_camera_order_of_u_and_v --- modules/videoio/src/cap_android_camera.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/videoio/src/cap_android_camera.cpp b/modules/videoio/src/cap_android_camera.cpp index f51bfe949d..18fc604367 100644 --- a/modules/videoio/src/cap_android_camera.cpp +++ b/modules/videoio/src/cap_android_camera.cpp @@ -313,7 +313,7 @@ public: if (fourCC == FOURCC_UNKNOWN) { fourCC = FOURCC_NV21; } - } else if ( (uvPixelStride == 1) && (vPixel == uPixel + uLen) && (yLen == frameWidth * frameHeight) && (uLen == yLen / 4) && (vLen == uLen) ) { + } else if ( (uvPixelStride == 1) && (uPixel == vPixel + vLen) && (yLen == frameWidth * frameHeight) && (uLen == yLen / 4) && (vLen == uLen) ) { colorFormat = COLOR_FormatYUV420Planar; if (fourCC == FOURCC_UNKNOWN) { fourCC = FOURCC_YV12; @@ -327,7 +327,7 @@ public: buffer.clear(); buffer.insert(buffer.end(), yPixel, yPixel + yLen); - buffer.insert(buffer.end(), uPixel, uPixel + yLen / 2); + buffer.insert(buffer.end(), vPixel, vPixel + yLen / 2); return true; } From 05db8784ae43557a7958a752738d989372ece211 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Mon, 29 Nov 2021 19:56:23 +0300 Subject: [PATCH 110/226] fix Clip, LeakyReLU, LRN, Split defaults --- modules/dnn/src/layers/lrn_layer.cpp | 2 +- modules/dnn/src/onnx/onnx_importer.cpp | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/dnn/src/layers/lrn_layer.cpp b/modules/dnn/src/layers/lrn_layer.cpp index 434ba5ccbe..65cd160dfe 100644 --- a/modules/dnn/src/layers/lrn_layer.cpp +++ b/modules/dnn/src/layers/lrn_layer.cpp @@ -80,7 +80,7 @@ public: if (size % 2 != 1 || size <= 0) CV_Error(Error::StsBadArg, "LRN layer supports only positive odd values for local_size"); - alpha = params.get("alpha", 1); + alpha = params.get("alpha", 0.0001); beta = params.get("beta", 0.75); bias = params.get("bias", 1); normBySize = params.get("norm_by_size", true); diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index d70a81b9ad..f382d4637e 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -920,6 +920,7 @@ void ONNXImporter::parseSplit(LayerParams& layerParams, const opencv_onnx::NodeP layerParams.set("num_split", node_proto.output_size()); } layerParams.type = "Slice"; + layerParams.set("axis", layerParams.get("axis", 0)); addLayer(layerParams, node_proto); } @@ -1184,15 +1185,15 @@ void ONNXImporter::parseImageScaler(LayerParams& layerParams, const opencv_onnx: void ONNXImporter::parseClip(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { layerParams.type = "ReLU6"; - replaceLayerParam(layerParams, "min", "min_value"); - replaceLayerParam(layerParams, "max", "max_value"); + layerParams.set("min_value", layerParams.get("min", -FLT_MAX)); + layerParams.set("max_value", layerParams.get("max", FLT_MAX)); addLayer(layerParams, node_proto); } void ONNXImporter::parseLeakyRelu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { layerParams.type = "ReLU"; - replaceLayerParam(layerParams, "alpha", "negative_slope"); + layerParams.set("negative_slope", layerParams.get("alpha", 0.01)); addLayer(layerParams, node_proto); } From ea7d4be3f81c6b7a2523e04f4c1f124b97497740 Mon Sep 17 00:00:00 2001 From: Andrew Ryrie Date: Mon, 29 Nov 2021 21:43:00 +0000 Subject: [PATCH 111/226] Merge pull request #20658 from smbz:lstm_optimisation * dnn: LSTM optimisation This uses the AVX-optimised fastGEMM1T for matrix multiplications where available, instead of the standard cv::gemm. fastGEMM1T is already used by the fully-connected layer. This commit involves two minor modifications: - Use unaligned access. I don't believe this involves any performance hit in on modern CPUs (Nehalem and Bulldozer onwards) in the case where the address is actually aligned. - Allow for weight matrices where the number of columns is not a multiple of 8. I have not enabled AVX-512 as I don't have an AVX-512 CPU to test on. * Fix warning about initialisation order * Remove C++11 syntax * Fix build when AVX(2) is not available In this case the CV_TRY_X macros are defined to 0, rather than being undefined. * Minor changes as requested: - Don't check hardware support for AVX(2) when dispatch is disabled for these - Add braces * Fix out-of-bounds access in fully connected layer The old tail handling in fastGEMM1T implicitly rounded vecsize up to the next multiple of 8, and the fully connected layer implements padding up to the next multiple of 8 to cope with this. The new tail handling does not round the vecsize upwards like this but it does require that the vecsize is at least 8. To adapt to the new tail handling, the fully connected layer now rounds vecsize itself at the same time as adding the padding(which makes more sense anyway). This also means that the fully connected layer always passes a vecsize of at least 8 to fastGEMM1T, which fixes the out-of-bounds access problems. * Improve tail mask handling - Use static array for generating tail masks (as requested) - Apply tail mask to the weights as well as the input vectors to prevent spurious propagation of NaNs/Infs * Revert whitespace change * Improve readability of conditions for using AVX * dnn(lstm): minor coding style changes, replaced left aligned load --- modules/dnn/perf/perf_recurrent.cpp | 90 +++++++++++++++ .../dnn/src/layers/fully_connected_layer.cpp | 6 +- modules/dnn/src/layers/layers_common.simd.hpp | 65 ++++++++--- modules/dnn/src/layers/recurrent_layers.cpp | 103 +++++++++++++++++- 4 files changed, 245 insertions(+), 19 deletions(-) create mode 100644 modules/dnn/perf/perf_recurrent.cpp diff --git a/modules/dnn/perf/perf_recurrent.cpp b/modules/dnn/perf/perf_recurrent.cpp new file mode 100644 index 0000000000..fe2a51886c --- /dev/null +++ b/modules/dnn/perf/perf_recurrent.cpp @@ -0,0 +1,90 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "perf_precomp.hpp" + +namespace opencv_test { + +struct LstmParams { + // Batch size + int nrSamples; + + // Size of the input vector + int inputSize; + + // Size of the internal state vector + int hiddenSize; + + // Number of timesteps for the LSTM + int nrSteps; +}; + +static inline void PrintTo(const LstmParams& params, ::std::ostream* os) { + (*os) << "BATCH=" << params.nrSamples + << ", IN=" << params.inputSize + << ", HIDDEN=" << params.hiddenSize + << ", TS=" << params.nrSteps; +} + +static const LstmParams testLstmConfigs[] = { + {1, 192, 192, 100}, + {1, 1024, 192, 100}, + {1, 64, 192, 100}, + {1, 192, 512, 100}, + {64, 192, 192, 2}, + {64, 1024, 192, 2}, + {64, 64, 192, 2}, + {64, 192, 512, 2}, + {128, 192, 192, 2}, + {128, 1024, 192, 2}, + {128, 64, 192, 2}, + {128, 192, 512, 2} +}; + +class Layer_LSTM : public TestBaseWithParam {}; + +PERF_TEST_P_(Layer_LSTM, lstm) { + const LstmParams& params = GetParam(); + LayerParams lp; + lp.type = "LSTM"; + lp.name = "testLstm"; + lp.set("produce_cell_output", false); + lp.set("use_timestamp_dim", true); + + Mat weightH(params.hiddenSize * 4, params.hiddenSize, CV_32FC1, cv::Scalar(0)); + Mat weightX(params.hiddenSize * 4, params.inputSize, CV_32FC1, cv::Scalar(0)); + Mat bias(params.hiddenSize * 4, 1, CV_32FC1, cv::Scalar(0)); + Mat hInternal(params.nrSteps, params.hiddenSize, CV_32FC1, cv::Scalar(0)); + Mat cInternal(params.nrSteps, params.hiddenSize, CV_32FC1, cv::Scalar(0)); + lp.blobs.push_back(weightH); + lp.blobs.push_back(weightX); + lp.blobs.push_back(bias); + lp.blobs.push_back(hInternal); + lp.blobs.push_back(cInternal); + + std::vector inputDims; + inputDims.push_back(params.nrSamples); + inputDims.push_back(params.nrSteps); + inputDims.push_back(params.inputSize); + Mat input(inputDims.size(), inputDims.data(), CV_32FC1); + input = cv::Scalar(0); + + Net net; + net.addLayerToPrev(lp.name, lp.type, lp); + net.setInput(input); + + // Warm up + std::vector outputs(2); + net.forward(outputs, "testLstm"); + + TEST_CYCLE() + { + net.forward(outputs, "testLstm"); + } + SANITY_CHECK_NOTHING(); +} + +INSTANTIATE_TEST_CASE_P(/**/, Layer_LSTM, testing::ValuesIn(testLstmConfigs)); + +} // namespace diff --git a/modules/dnn/src/layers/fully_connected_layer.cpp b/modules/dnn/src/layers/fully_connected_layer.cpp index e25ca5a68f..5acce939f1 100644 --- a/modules/dnn/src/layers/fully_connected_layer.cpp +++ b/modules/dnn/src/layers/fully_connected_layer.cpp @@ -222,17 +222,17 @@ public: #if CV_TRY_AVX512_SKX if( useAVX512 ) - opt_AVX512_SKX::fastGEMM1T( sptr, wptr, wstep, biasptr, dptr, nw, vecsize); + opt_AVX512_SKX::fastGEMM1T( sptr, wptr, wstep, biasptr, dptr, nw, vecsize_aligned); else #endif #if CV_TRY_AVX2 if( useAVX2 ) - opt_AVX2::fastGEMM1T( sptr, wptr, wstep, biasptr, dptr, nw, vecsize); + opt_AVX2::fastGEMM1T( sptr, wptr, wstep, biasptr, dptr, nw, vecsize_aligned); else #endif #if CV_TRY_AVX if( useAVX ) - opt_AVX::fastGEMM1T( sptr, wptr, wstep, biasptr, dptr, nw, vecsize); + opt_AVX::fastGEMM1T( sptr, wptr, wstep, biasptr, dptr, nw, vecsize_aligned); else #endif { diff --git a/modules/dnn/src/layers/layers_common.simd.hpp b/modules/dnn/src/layers/layers_common.simd.hpp index 706695a7b2..accc644676 100644 --- a/modules/dnn/src/layers/layers_common.simd.hpp +++ b/modules/dnn/src/layers/layers_common.simd.hpp @@ -550,13 +550,24 @@ void fastDepthwiseConv( const float* wptr, _mm256_zeroupper(); } +// Used to generate the mask used when calculating tails +static const uint32_t tailMaskArray[15] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0xffffffffUL, 0xffffffffUL, 0xffffffffUL, 0xffffffffUL, 0xffffffffUL, 0xffffffffUL, 0xffffffffUL +}; + // dst = vec * weights^t + bias +// Requires that vecsize is at least 8 or equal to 0 to avoid memory access problems. Does not require alignment. void fastGEMM1T( const float* vec, const float* weights, size_t wstep, const float* bias, float* dst, int nvecs, int vecsize ) { int i = 0; + CV_Assert(vecsize >= 8 || vecsize == 0); + + __m256 tailMask = _mm256_loadu_ps(reinterpret_cast(tailMaskArray) + (vecsize % 8)); + for( ; i <= nvecs - 8; i += 8 ) { const float* wptr = weights + i*wstep; @@ -565,18 +576,36 @@ void fastGEMM1T( const float* vec, const float* weights, vs4 = _mm256_setzero_ps(), vs5 = _mm256_setzero_ps(), vs6 = _mm256_setzero_ps(), vs7 = _mm256_setzero_ps(); - for( int k = 0; k < vecsize; k += 8, wptr += 8 ) + int k = 0; + for( ; k <= vecsize-8; k += 8, wptr += 8 ) { - __m256 v = _mm256_load_ps(vec + k); + __m256 v = _mm256_loadu_ps(vec + k); - vs0 = _mm256_fmadd_ps(_mm256_load_ps(wptr), v, vs0); - vs1 = _mm256_fmadd_ps(_mm256_load_ps(wptr + wstep), v, vs1); - vs2 = _mm256_fmadd_ps(_mm256_load_ps(wptr + wstep*2), v, vs2); - vs3 = _mm256_fmadd_ps(_mm256_load_ps(wptr + wstep*3), v, vs3); - vs4 = _mm256_fmadd_ps(_mm256_load_ps(wptr + wstep*4), v, vs4); - vs5 = _mm256_fmadd_ps(_mm256_load_ps(wptr + wstep*5), v, vs5); - vs6 = _mm256_fmadd_ps(_mm256_load_ps(wptr + wstep*6), v, vs6); - vs7 = _mm256_fmadd_ps(_mm256_load_ps(wptr + wstep*7), v, vs7); + vs0 = _mm256_fmadd_ps(_mm256_loadu_ps(wptr), v, vs0); + vs1 = _mm256_fmadd_ps(_mm256_loadu_ps(wptr + wstep), v, vs1); + vs2 = _mm256_fmadd_ps(_mm256_loadu_ps(wptr + wstep*2), v, vs2); + vs3 = _mm256_fmadd_ps(_mm256_loadu_ps(wptr + wstep*3), v, vs3); + vs4 = _mm256_fmadd_ps(_mm256_loadu_ps(wptr + wstep*4), v, vs4); + vs5 = _mm256_fmadd_ps(_mm256_loadu_ps(wptr + wstep*5), v, vs5); + vs6 = _mm256_fmadd_ps(_mm256_loadu_ps(wptr + wstep*6), v, vs6); + vs7 = _mm256_fmadd_ps(_mm256_loadu_ps(wptr + wstep*7), v, vs7); + } + + if (k != vecsize) { + // Tail + k = vecsize - 8; + wptr = weights + i * wstep + k; + __m256 v = _mm256_loadu_ps(vec + k); + v = _mm256_and_ps(v, tailMask); + + vs0 = _mm256_fmadd_ps(_mm256_and_ps(_mm256_loadu_ps(wptr), tailMask), v, vs0); + vs1 = _mm256_fmadd_ps(_mm256_and_ps(_mm256_loadu_ps(wptr + wstep), tailMask), v, vs1); + vs2 = _mm256_fmadd_ps(_mm256_and_ps(_mm256_loadu_ps(wptr + wstep * 2), tailMask), v, vs2); + vs3 = _mm256_fmadd_ps(_mm256_and_ps(_mm256_loadu_ps(wptr + wstep * 3), tailMask), v, vs3); + vs4 = _mm256_fmadd_ps(_mm256_and_ps(_mm256_loadu_ps(wptr + wstep * 4), tailMask), v, vs4); + vs5 = _mm256_fmadd_ps(_mm256_and_ps(_mm256_loadu_ps(wptr + wstep * 5), tailMask), v, vs5); + vs6 = _mm256_fmadd_ps(_mm256_and_ps(_mm256_loadu_ps(wptr + wstep * 6), tailMask), v, vs6); + vs7 = _mm256_fmadd_ps(_mm256_and_ps(_mm256_loadu_ps(wptr + wstep * 7), tailMask), v, vs7); } __m256 s0 = _mm256_hadd_ps(_mm256_hadd_ps(vs0, vs1), _mm256_hadd_ps(vs2, vs3)); @@ -598,10 +627,20 @@ void fastGEMM1T( const float* vec, const float* weights, const float* wptr = weights + i*wstep; __m256 vs0 = _mm256_setzero_ps(); - for( int k = 0; k < vecsize; k += 8, wptr += 8 ) + int k = 0; + for( ; k <= vecsize-8; k += 8, wptr += 8 ) { - __m256 v = _mm256_load_ps(vec + k); - vs0 = _mm256_fmadd_ps(_mm256_load_ps(wptr), v, vs0); + __m256 v = _mm256_loadu_ps(vec + k); + vs0 = _mm256_fmadd_ps(_mm256_loadu_ps(wptr), v, vs0); + } + + if (k != vecsize) { + // Tail + k = vecsize - 8; + wptr = weights + i * wstep + k; + __m256 v = _mm256_loadu_ps(vec + k); + v = _mm256_and_ps(v, tailMask); + vs0 = _mm256_fmadd_ps(_mm256_and_ps(_mm256_loadu_ps(wptr), tailMask), v, vs0); } __m256 s0 = _mm256_hadd_ps(_mm256_hadd_ps(vs0, vs0), vs0); diff --git a/modules/dnn/src/layers/recurrent_layers.cpp b/modules/dnn/src/layers/recurrent_layers.cpp index 9088c13390..21dafa142d 100644 --- a/modules/dnn/src/layers/recurrent_layers.cpp +++ b/modules/dnn/src/layers/recurrent_layers.cpp @@ -46,6 +46,8 @@ #include #include +#include "layers_common.hpp" + namespace cv { namespace dnn @@ -118,10 +120,23 @@ class LSTMLayerImpl CV_FINAL : public LSTMLayer ActivationFunction g_activation; ActivationFunction h_activation; +#if CV_TRY_AVX + bool useAVX; +#endif +#if CV_TRY_AVX2 + bool useAVX2; +#endif + public: LSTMLayerImpl(const LayerParams& params) : numTimeStamps(0), numSamples(0) +#if CV_TRY_AVX + , useAVX(checkHardwareSupport(CPU_AVX)) +#endif +#if CV_TRY_AVX2 + , useAVX2(checkHardwareSupport(CPU_AVX2)) +#endif { setParamsFrom(params); @@ -343,6 +358,15 @@ public: hOutTs = hOutTs.colRange(i * hOutTs.cols / numDirs, (i + 1) * hOutTs.cols / numDirs); Mat cOutTs = produceCellOutput ? output[1].reshape(1, numSamplesTotal) : Mat(); +#if CV_TRY_AVX2 || CV_TRY_AVX + bool canUseAvx = gates.isContinuous() && bias.isContinuous() + && Wx.depth() == CV_32F && gates.depth() == CV_32F + && bias.depth() == CV_32F && Wx.cols >= 8; + bool canUseAvx_hInternal = hInternal.isContinuous() && gates.isContinuous() && bias.isContinuous() + && Wh.depth() == CV_32F && hInternal.depth() == CV_32F && gates.depth() == CV_32F + && Wh.cols >= 8; +#endif + int tsStart, tsEnd, tsInc; if (reverse || i == 1) { tsStart = numTimeStamps - 1; @@ -359,9 +383,82 @@ public: Range curRowRange(ts*numSamples, (ts + 1)*numSamples); Mat xCurr = xTs.rowRange(curRowRange); - gemm(xCurr, Wx, 1, gates, 0, gates, GEMM_2_T); // Wx * x_t - gemm(hInternal, Wh, 1, gates, 1, gates, GEMM_2_T); //+Wh * h_{t-1} - gemm(dummyOnes, bias, 1, gates, 1, gates); //+b +#if CV_TRY_AVX2 + if (useAVX2 && canUseAvx && xCurr.isContinuous()) + { + for (int n = 0; n < xCurr.rows; n++) { + opt_AVX2::fastGEMM1T( + xCurr.ptr(n), + Wx.ptr(), + Wx.step1(), + bias.ptr(), + gates.ptr(n), + Wx.rows, + Wx.cols + ); + } + } + else +#endif +#if CV_TRY_AVX + if (useAVX && canUseAvx && xCurr.isContinuous()) + { + for (int n = 0; n < xCurr.rows; n++) { + opt_AVX::fastGEMM1T( + xCurr.ptr(n), + Wx.ptr(), + Wx.step1(), + bias.ptr(), + gates.ptr(n), + Wx.rows, + Wx.cols + ); + } + } + else +#endif + { + gemm(xCurr, Wx, 1, gates, 0, gates, GEMM_2_T); // Wx * x_t + gemm(dummyOnes, bias, 1, gates, 1, gates); //+b + } + +#if CV_TRY_AVX2 + if (useAVX2 && canUseAvx_hInternal) + { + for (int n = 0; n < hInternal.rows; n++) { + opt_AVX2::fastGEMM1T( + hInternal.ptr(n), + Wh.ptr(), + Wh.step1(), + gates.ptr(n), + gates.ptr(n), + Wh.rows, + Wh.cols + ); + } + } + else +#endif +#if CV_TRY_AVX + if (useAVX && canUseAvx_hInternal) + { + for (int n = 0; n < hInternal.rows; n++) { + opt_AVX::fastGEMM1T( + hInternal.ptr(n), + Wh.ptr(), + Wh.step1(), + gates.ptr(n), + gates.ptr(n), + Wh.rows, + Wh.cols + ); + } + } + else +#endif + { + gemm(hInternal, Wh, 1, gates, 1, gates, GEMM_2_T); //+Wh * h_{t-1} + } Mat gateI = gates.colRange(0*numOut, 1*numOut); Mat gateF = gates.colRange(1*numOut, 2*numOut); From 66b2140892cbce5a2c45f1e7a4548929ca8e6530 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 30 Nov 2021 04:27:39 +0000 Subject: [PATCH 112/226] build: eliminate C4309 warning from protobuf files with MSVS2017 --- 3rdparty/protobuf/CMakeLists.txt | 2 +- modules/dnn/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/3rdparty/protobuf/CMakeLists.txt b/3rdparty/protobuf/CMakeLists.txt index 6de8148e58..e39de9823a 100644 --- a/3rdparty/protobuf/CMakeLists.txt +++ b/3rdparty/protobuf/CMakeLists.txt @@ -13,7 +13,7 @@ if(MSVC) /wd4701 /wd4703 # potentially uninitialized local/pointer variable 'value' used /wd4505 # unreferenced local function has been removed ) - if(MSVC_VERSION LESS 1910) # MSVS 2015 + if(MSVC_VERSION LESS 1920) # MSVS 2015/2017 ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4309) # 'static_cast': truncation of constant value endif() else() diff --git a/modules/dnn/CMakeLists.txt b/modules/dnn/CMakeLists.txt index 398ed5ba48..ba7fe5a923 100644 --- a/modules/dnn/CMakeLists.txt +++ b/modules/dnn/CMakeLists.txt @@ -51,7 +51,7 @@ if(MSVC) /wd4305 /wd4127 /wd4100 /wd4512 /wd4125 /wd4389 /wd4510 /wd4610 /wd4702 /wd4456 /wd4457 /wd4065 /wd4310 /wd4661 /wd4506 ) - if(MSVC_VERSION LESS 1910) # MSVS 2015, .pb.cc generated files + if(MSVC_VERSION LESS 1920) # MSVS 2015/2017, .pb.cc generated files ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4309) # 'static_cast': truncation of constant value endif() if(MSVC_VERSION LESS 1920) # Date: Tue, 30 Nov 2021 12:20:35 +0300 Subject: [PATCH 113/226] add alpha parameter to ELU layer --- modules/dnn/include/opencv2/dnn/all_layers.hpp | 2 ++ modules/dnn/src/layers/elementwise_layers.cpp | 18 ++++++++++++++---- modules/dnn/src/opencl/activations.cl | 5 +++-- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index e92ce2f565..991abba03a 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -453,6 +453,8 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN class CV_EXPORTS ELULayer : public ActivationLayer { public: + float alpha; + static Ptr create(const LayerParams ¶ms); }; diff --git a/modules/dnn/src/layers/elementwise_layers.cpp b/modules/dnn/src/layers/elementwise_layers.cpp index d8f0b654d3..21838fbe49 100644 --- a/modules/dnn/src/layers/elementwise_layers.cpp +++ b/modules/dnn/src/layers/elementwise_layers.cpp @@ -740,6 +740,9 @@ const char* const SigmoidFunctor::BaseDefaultFunctor::ocl_kernel struct ELUFunctor : public BaseDefaultFunctor { typedef ELULayer Layer; + float alpha; + + explicit ELUFunctor(float alpha_ = 1.f) : alpha(alpha_) {} bool supportBackend(int backendId, int) { @@ -749,14 +752,19 @@ struct ELUFunctor : public BaseDefaultFunctor inline float calculate(float x) const { - return x >= 0.f ? x : exp(x) - 1.f; + return x >= 0.f ? x : alpha * (exp(x) - 1.f); + } + + inline void setKernelParams(ocl::Kernel& kernel) const + { + kernel.set(3, alpha); } #ifdef HAVE_HALIDE void attachHalide(const Halide::Expr& input, Halide::Func& top) { Halide::Var x("x"), y("y"), c("c"), n("n"); - top(x, y, c, n) = select(input >= 0.0f, input, exp(input) - 1); + top(x, y, c, n) = select(input >= 0.0f, input, alpha * (exp(input) - 1)); } #endif // HAVE_HALIDE @@ -770,7 +778,7 @@ struct ELUFunctor : public BaseDefaultFunctor #ifdef HAVE_DNN_NGRAPH std::shared_ptr initNgraphAPI(const std::shared_ptr& node) { - return std::make_shared(node, 1.0); + return std::make_shared(node, alpha); } #endif // HAVE_DNN_NGRAPH @@ -1263,8 +1271,10 @@ Ptr SigmoidLayer::create(const LayerParams& params) Ptr ELULayer::create(const LayerParams& params) { - Ptr l(new ElementWiseLayer(ELUFunctor())); + float alpha = params.get("alpha", 1.0f); + Ptr l(new ElementWiseLayer(ELUFunctor(alpha))); l->setParamsFrom(params); + l->alpha = alpha; return l; } diff --git a/modules/dnn/src/opencl/activations.cl b/modules/dnn/src/opencl/activations.cl index 68f0dd7268..a75370d1bd 100644 --- a/modules/dnn/src/opencl/activations.cl +++ b/modules/dnn/src/opencl/activations.cl @@ -131,13 +131,14 @@ __kernel void PowForward(const int n, __global const T* in, __global T* out, out[index] = pow(shift + scale * in[index], power); } -__kernel void ELUForward(const int n, __global const T* in, __global T* out) +__kernel void ELUForward(const int n, __global const T* in, __global T* out, + const KERNEL_ARG_DTYPE alpha) { int index = get_global_id(0); if (index < n) { T src = in[index]; - out[index] = (src >= 0.f) ? src : exp(src) - 1; + out[index] = (src >= 0.f) ? src : alpha * (exp(src) - 1); } } From 4995aecd62e0a65cfb9a0f75ccbe2e16d43f6c5f Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Tue, 30 Nov 2021 14:43:18 +0300 Subject: [PATCH 114/226] add alpha parameter to ELU --- .../dnn/include/opencv2/dnn/all_layers.hpp | 2 ++ modules/dnn/src/cuda/activations.cu | 8 ++++---- modules/dnn/src/cuda/functors.hpp | 12 +++++++---- .../dnn/src/cuda4dnn/kernels/activations.hpp | 2 +- .../src/cuda4dnn/primitives/activation.hpp | 5 +++-- modules/dnn/src/layers/elementwise_layers.cpp | 20 ++++++++++++++----- modules/dnn/src/opencl/activations.cl | 5 +++-- 7 files changed, 36 insertions(+), 18 deletions(-) diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index c70ce2b50a..0bec9e35e2 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -545,6 +545,8 @@ CV__DNN_INLINE_NS_BEGIN class CV_EXPORTS ELULayer : public ActivationLayer { public: + float alpha; + static Ptr create(const LayerParams ¶ms); }; diff --git a/modules/dnn/src/cuda/activations.cu b/modules/dnn/src/cuda/activations.cu index c38fa0346f..0980b5dd46 100644 --- a/modules/dnn/src/cuda/activations.cu +++ b/modules/dnn/src/cuda/activations.cu @@ -119,8 +119,8 @@ void sigmoid(const Stream& stream, Span output, View input) { } template -void elu(const Stream& stream, Span output, View input) { - generic_op>(stream, output, input); +void elu(const Stream& stream, Span output, View input, T alpha) { + generic_op>(stream, output, input, {alpha}); } template @@ -187,7 +187,7 @@ template void tanh<__half>(const Stream&, Span<__half>, View<__half>); template void swish<__half>(const Stream&, Span<__half>, View<__half>); template void mish<__half>(const Stream&, Span<__half>, View<__half>); template void sigmoid<__half>(const Stream&, Span<__half>, View<__half>); -template void elu<__half>(const Stream&, Span<__half>, View<__half>); +template void elu<__half>(const Stream&, Span<__half>, View<__half>, __half); template void abs<__half>(const Stream& stream, Span<__half> output, View<__half> input); template void bnll<__half>(const Stream&, Span<__half>, View<__half>); template void ceil<__half>(const Stream&, Span<__half>, View<__half>); @@ -207,7 +207,7 @@ template void tanh(const Stream&, Span, View); template void swish(const Stream&, Span, View); template void mish(const Stream&, Span, View); template void sigmoid(const Stream&, Span, View); -template void elu(const Stream&, Span, View); +template void elu(const Stream&, Span, View, float); template void abs(const Stream& stream, Span output, View input); template void bnll(const Stream&, Span, View); template void ceil(const Stream&, Span, View); diff --git a/modules/dnn/src/cuda/functors.hpp b/modules/dnn/src/cuda/functors.hpp index 04b545acaf..98ae175ce8 100644 --- a/modules/dnn/src/cuda/functors.hpp +++ b/modules/dnn/src/cuda/functors.hpp @@ -169,16 +169,20 @@ struct SigmoidFunctor { template struct ELUFunctor { struct Params { - CUDA4DNN_HOST_DEVICE Params() { } + CUDA4DNN_HOST_DEVICE Params() : alpha(1) { } + CUDA4DNN_HOST_DEVICE Params(T alpha_) : alpha(alpha_) { } + T alpha; }; - CUDA4DNN_DEVICE ELUFunctor() { } - CUDA4DNN_DEVICE ELUFunctor(const Params& params) { } + CUDA4DNN_DEVICE ELUFunctor() : ELUFunctor(Params{}) { } + CUDA4DNN_DEVICE ELUFunctor(const Params& params) : alpha{params.alpha} { } CUDA4DNN_DEVICE T operator()(T value) { using csl::device::expm1; - return value >= T(0) ? value : expm1(value); + return value >= T(0) ? value : alpha * expm1(value); } + + T alpha; }; template diff --git a/modules/dnn/src/cuda4dnn/kernels/activations.hpp b/modules/dnn/src/cuda4dnn/kernels/activations.hpp index 0fcf7dab8a..d7c471a5ec 100644 --- a/modules/dnn/src/cuda4dnn/kernels/activations.hpp +++ b/modules/dnn/src/cuda4dnn/kernels/activations.hpp @@ -34,7 +34,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { void sigmoid(const csl::Stream& stream, csl::Span output, csl::View input); template - void elu(const csl::Stream& stream, csl::Span output, csl::View input); + void elu(const csl::Stream& stream, csl::Span output, csl::View input, T alpha); template void abs(const csl::Stream& stream, csl::Span output, csl::View input); diff --git a/modules/dnn/src/cuda4dnn/primitives/activation.hpp b/modules/dnn/src/cuda4dnn/primitives/activation.hpp index a179db2da5..77a79703fe 100644 --- a/modules/dnn/src/cuda4dnn/primitives/activation.hpp +++ b/modules/dnn/src/cuda4dnn/primitives/activation.hpp @@ -156,15 +156,16 @@ namespace cv { namespace dnn { namespace cuda4dnn { template class ELUOp final : public BaseOp { public: - ELUOp(csl::Stream stream_) : stream(std::move(stream_)) { } + ELUOp(csl::Stream stream_, T alpha_) : stream(std::move(stream_)), alpha(alpha_) { } void calculate(csl::TensorSpan output, csl::TensorView input) const { - kernels::elu(stream, output, input); + kernels::elu(stream, output, input, alpha); } private: csl::Stream stream; + T alpha; }; template diff --git a/modules/dnn/src/layers/elementwise_layers.cpp b/modules/dnn/src/layers/elementwise_layers.cpp index 56e82cc3d1..7cec0d5f7b 100644 --- a/modules/dnn/src/layers/elementwise_layers.cpp +++ b/modules/dnn/src/layers/elementwise_layers.cpp @@ -987,6 +987,9 @@ const char* const SigmoidFunctor::BaseDefaultFunctor::ocl_kernel struct ELUFunctor : public BaseDefaultFunctor { typedef ELULayer Layer; + float alpha; + + explicit ELUFunctor(float alpha_ = 1.f) : alpha(alpha_) {} bool supportBackend(int backendId, int) { @@ -998,13 +1001,18 @@ struct ELUFunctor : public BaseDefaultFunctor inline float calculate(float x) const { - return x >= 0.f ? x : exp(x) - 1.f; + return x >= 0.f ? x : alpha * (exp(x) - 1.f); + } + + inline void setKernelParams(ocl::Kernel& kernel) const + { + kernel.set(3, alpha); } #ifdef HAVE_CUDA Ptr initCUDA(int target, csl::Stream stream) { - return make_cuda_node(target, stream); + return make_cuda_node(target, stream, alpha); } #endif @@ -1012,7 +1020,7 @@ struct ELUFunctor : public BaseDefaultFunctor void attachHalide(const Halide::Expr& input, Halide::Func& top) { Halide::Var x("x"), y("y"), c("c"), n("n"); - top(x, y, c, n) = select(input >= 0.0f, input, exp(input) - 1); + top(x, y, c, n) = select(input >= 0.0f, input, alpha * (exp(input) - 1)); } #endif // HAVE_HALIDE @@ -1026,7 +1034,7 @@ struct ELUFunctor : public BaseDefaultFunctor #ifdef HAVE_DNN_NGRAPH std::shared_ptr initNgraphAPI(const std::shared_ptr& node) { - return std::make_shared(node, 1.0); + return std::make_shared(node, alpha); } #endif // HAVE_DNN_NGRAPH @@ -1856,8 +1864,10 @@ Ptr SigmoidLayer::create(const LayerParams& params) Ptr ELULayer::create(const LayerParams& params) { - Ptr l(new ElementWiseLayer(ELUFunctor())); + float alpha = params.get("alpha", 1.0f); + Ptr l(new ElementWiseLayer(ELUFunctor(alpha))); l->setParamsFrom(params); + l->alpha = alpha; return l; } diff --git a/modules/dnn/src/opencl/activations.cl b/modules/dnn/src/opencl/activations.cl index bc2a105aba..e110160c06 100644 --- a/modules/dnn/src/opencl/activations.cl +++ b/modules/dnn/src/opencl/activations.cl @@ -131,13 +131,14 @@ __kernel void PowForward(const int n, __global const T* in, __global T* out, out[index] = pow(shift + scale * in[index], power); } -__kernel void ELUForward(const int n, __global const T* in, __global T* out) +__kernel void ELUForward(const int n, __global const T* in, __global T* out, + const KERNEL_ARG_DTYPE alpha) { int index = get_global_id(0); if (index < n) { T src = in[index]; - out[index] = (src >= 0.f) ? src : exp(src) - 1; + out[index] = (src >= 0.f) ? src : alpha * (exp(src) - 1); } } From 829410729c8fe548a1786e5b3320194648aaa1e8 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Tue, 30 Nov 2021 15:20:52 +0300 Subject: [PATCH 115/226] add new (Log)SoftMax simplification passes --- .../dnn/src/onnx/onnx_graph_simplifier.cpp | 70 ++++++++++++++++--- 1 file changed, 59 insertions(+), 11 deletions(-) diff --git a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp index 76937e08f3..e4cf73fd07 100644 --- a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp +++ b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp @@ -107,17 +107,10 @@ private: opencv_onnx::GraphProto& net; }; -class SoftMaxSubgraph : public Subgraph +class SoftMaxSubgraphBase : public Subgraph { public: - SoftMaxSubgraph() : axis(1) - { - int input = addNodeToMatch(""); - int inpExp = addNodeToMatch("Exp", input); - int sum = addNodeToMatch("ReduceSum", inpExp); - addNodeToMatch("Div", inpExp, sum); - setFusedNode("Softmax", input); - } + SoftMaxSubgraphBase() : axis(1), id(-1) {} virtual bool match(const Ptr& net, int nodeId, std::vector& matchedNodesIds, @@ -125,7 +118,8 @@ public: { if (Subgraph::match(net, nodeId, matchedNodesIds, targetNodesIds)) { - Ptr sum = net->getNode(matchedNodesIds[1]); + CV_Assert(id >= 0 && id < matchedNodesIds.size()); + Ptr sum = net->getNode(matchedNodesIds[id]); opencv_onnx::NodeProto* node = sum.dynamicCast()->node; for (int i = 0; i < node->attribute_size(); i++) @@ -153,8 +147,60 @@ public: attr->set_i(axis); } -private: +protected: int axis; + int id; +}; + +class SoftMaxSubgraph : public SoftMaxSubgraphBase +{ +public: + SoftMaxSubgraph() + { + int input = addNodeToMatch(""); + int inpExp = addNodeToMatch("Exp", input); + + int sum = addNodeToMatch("ReduceSum", inpExp); + id = 1; + + addNodeToMatch("Div", inpExp, sum); + setFusedNode("Softmax", input); + } +}; + +class SoftMaxSubgraph2 : public SoftMaxSubgraphBase { +public: + SoftMaxSubgraph2() { + int input = addNodeToMatch(""); + + int reducemax = addNodeToMatch("ReduceMax", input); + id = 0; + + int sub = addNodeToMatch("Sub", input, reducemax); + int exp = addNodeToMatch("Exp", sub); + int reducesum = addNodeToMatch("ReduceSum", exp, addNodeToMatch("")); + addNodeToMatch("Div", exp, reducesum); + setFusedNode("Softmax", input); + } +}; + +class LogSoftMaxSubgraph : public SoftMaxSubgraphBase +{ +public: + LogSoftMaxSubgraph() + { + int input = addNodeToMatch(""); + + int reducemax = addNodeToMatch("ReduceMax", input); + id = 0; + + int sub_1 = addNodeToMatch("Sub", input, reducemax); + int exp = addNodeToMatch("Exp", sub_1); + int reducesum = addNodeToMatch("ReduceSum", exp, addNodeToMatch("")); + int log = addNodeToMatch("Log", reducesum); + addNodeToMatch("Sub", sub_1, log); + setFusedNode("LogSoftmax", input); + } }; class NormalizeSubgraphBase : public Subgraph @@ -574,6 +620,8 @@ void simplifySubgraphs(opencv_onnx::GraphProto& net) subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); + subgraphs.push_back(makePtr()); + subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); From 11e6848bb97d6b6c2b1dad90f283bafed29a9316 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Tue, 30 Nov 2021 15:34:34 +0300 Subject: [PATCH 116/226] add default order to transpose --- modules/dnn/src/onnx/onnx_importer.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index d70a81b9ad..540dac3a90 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -1644,6 +1644,16 @@ void ONNXImporter::parseTranspose(LayerParams& layerParams, const opencv_onnx::N { layerParams.type = "Permute"; replaceLayerParam(layerParams, "perm", "order"); + if (!layerParams.has("order")) { + MatShape inpShape = outShapes[node_proto.input(0)]; + size_t dims = inpShape.size(); + std::vector perm(dims); + for (size_t d = 0; d < dims; ++d) + { + perm[d] = static_cast(dims - 1 - d); + } + layerParams.set("order", DictValue::arrayInt(perm.data(), perm.size())); + } CV_Assert(node_proto.input_size() == 1); if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) From 33e97e994db8e7e6e275dec6daf538f07f0e8d28 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Tue, 30 Nov 2021 15:42:20 +0300 Subject: [PATCH 117/226] add sum of 1 input --- modules/dnn/src/onnx/onnx_importer.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index d70a81b9ad..7af095e360 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -929,6 +929,14 @@ void ONNXImporter::parseBias(LayerParams& layerParams, const opencv_onnx::NodePr opencv_onnx::NodeProto node_proto = node_proto_; const std::string& layer_type = node_proto.op_type(); bool isSub = layer_type == "Sub"; + + if (layer_type == "Sum" && node_proto.input_size() == 1) + { + layerParams.type = "Identity"; + addLayer(layerParams, node_proto); + return; + } + CV_Assert((node_proto.input_size() == 2) || (layer_type == "Sum" && node_proto.input_size() > 2)); if (layer_type == "Sum" && node_proto.input_size() > 2) From 1a1a7bbbfd5b0a5cd0bd544eafae8729c9874203 Mon Sep 17 00:00:00 2001 From: Vincent Rabaud Date: Wed, 1 Dec 2021 13:46:03 +0100 Subject: [PATCH 118/226] Merge pull request #21112 from vrabaud:3.4_luv_overflow * Fix integer overflow in cv::Luv2RGBinteger::process. For LL=49, uu=205, vv=23, we end up with x=7373056 and y=458 which overflows y*x. * imgproc(test): adjust test parameters to cover SIMD code --- modules/imgproc/src/color_lab.cpp | 8 ++------ modules/imgproc/test/test_color.cpp | 25 ++++++++++++++++++++----- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/modules/imgproc/src/color_lab.cpp b/modules/imgproc/src/color_lab.cpp index a181880862..6b03be6195 100644 --- a/modules/imgproc/src/color_lab.cpp +++ b/modules/imgproc/src/color_lab.cpp @@ -3567,7 +3567,7 @@ struct Luv2RGBinteger long long int xv = ((int)up)*(long long)vp; int x = (int)(xv/BASE); - x = y*x/BASE; + x = ((long long int)y)*x/BASE; long long int vpl = LUVLUT.LvToVpl_b[LL*256+vv]; long long int zp = vpl - xv*(255/3); @@ -3689,6 +3689,7 @@ struct Luv2RGBinteger vzm[i] = zm; vx[i] = (int32_t)(xv >> base_shift); + vx[i] = (((int64_t)y_)*vx[i]) >> base_shift; } v_int32 zm[4]; for(int k = 0; k < 4; k++) @@ -3697,11 +3698,6 @@ struct Luv2RGBinteger zm[k] = vx_load_aligned(vzm + k*vsize/4); } - for(int k = 0; k < 4; k++) - { - x[k] = (y[k]*x[k]) >> base_shift; - } - // z = zm/256 + zm/65536; for (int k = 0; k < 4; k++) { diff --git a/modules/imgproc/test/test_color.cpp b/modules/imgproc/test/test_color.cpp index 3cd57c9d38..f828dac18a 100644 --- a/modules/imgproc/test/test_color.cpp +++ b/modules/imgproc/test/test_color.cpp @@ -1840,6 +1840,21 @@ TEST(Imgproc_ColorLuv, accuracy) { CV_ColorLuvTest test; test.safe_run(); } TEST(Imgproc_ColorRGB, accuracy) { CV_ColorRGBTest test; test.safe_run(); } TEST(Imgproc_ColorBayer, accuracy) { CV_ColorBayerTest test; test.safe_run(); } +TEST(Imgproc_ColorLuv, Overflow_21112) +{ + const Size sz(107, 16); // unaligned size to run both SIMD and generic code + Mat luv_init(sz, CV_8UC3, Scalar(49, 205, 23)); + Mat rgb; + cvtColor(luv_init, rgb, COLOR_Luv2RGB); + // Convert to normal Luv coordinates for floats. + Mat luv_initf(sz, CV_32FC3, Scalar(49.0f/255.f*100, 205.0f*354/255.f - 134, 23.0f*262/255.f - 140)); + Mat rgbf; + cvtColor(luv_initf, rgbf, COLOR_Luv2RGB); + Mat rgb_converted; + rgb.convertTo(rgb_converted, CV_32F); + EXPECT_LE(cvtest::norm(255.f*rgbf, rgb_converted, NORM_INF), 1e-5); +} + TEST(Imgproc_ColorBayer, regression) { cvtest::TS* ts = cvtest::TS::ptr(); @@ -2569,7 +2584,7 @@ int row8uLuv2RGB(const uchar* src_row, uchar *dst_row, int n, int cn, int blue_i long long int xv = ((int)up)*(long long)vp; int x = (int)(xv/BASE); - x = y*x/BASE; + x = ((long long int)y)*x/BASE; long long int vpl = LvToVpl_b[LL*256+vv]; long long int zp = vpl - xv*(255/3); @@ -2725,11 +2740,11 @@ TEST(Imgproc_ColorLuv_Full, bitExactness) 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0xec311a14, 0x995efefc, 0xf71b590b, 0xc1edfce7, 0x67b2b2e2, 0xe6d7f90d, 0xbcbaff5c, 0xd86ae19c, - 0x3e8e4647, 0x53f1a5e3, 0x60dfb6ca, 0xcda851fe, 0xd91084b3, 0xe361bf6f, 0x90fe66ed, 0xb19c5b89, + 0x4bff0e00, 0x76bbff01, 0x80735725, 0xb5e0f137, 0x96abb417, 0xfb2cf5cf, 0x314cf55e, 0x77bde10e, + 0x2ab24209, 0x81caa6F0, 0x3019b8eb, 0x427c505f, 0x5bba7d77, 0xf29cb4d6, 0x760f65ca, 0xf6b4536c, - 0x190508ec, 0xc7764e22, 0x19b042a8, 0x2db4c5d8, 0x6e1cfd1d, 0x39bddd51, 0x942714ed, 0x19444d39, - 0xed16e206, 0xc4102784, 0x590075fe, 0xaaef2ec6, 0xbeb84149, 0x8da31e4f, 0x7cbe7d77, 0x1c90b30a, + 0xb5cd0704, 0x82144fd4, 0x4e6f4843, 0x106bc505, 0xf587fc97, 0x3665d9a3, 0x3ea014a8, 0xec664953, + 0x6ec9e59e, 0xf9201e08, 0xf3676fb8, 0xe4e42c10, 0x92d33f64, 0x13b923f7, 0x308f7f50, 0xca98b420, }; RNG rng(0); From 369b260e1208f6f182c6c1da4dac9c6ed30950fa Mon Sep 17 00:00:00 2001 From: Anna Khakimova Date: Thu, 2 Dec 2021 00:58:30 +0300 Subject: [PATCH 119/226] Merge pull request #21158 from anna-khakimova:ak/simd_subC * GAPI Fluid: SIMD for SubC kernel. * Applied comments --- .../gapi/perf/common/gapi_core_perf_tests.hpp | 2 +- .../perf/common/gapi_core_perf_tests_inl.hpp | 16 +- .../perf/cpu/gapi_core_perf_tests_cpu.cpp | 3 +- .../perf/cpu/gapi_core_perf_tests_fluid.cpp | 11 +- .../perf/gpu/gapi_core_perf_tests_gpu.cpp | 3 +- .../gapi/src/backends/fluid/gfluidcore.cpp | 152 +++++------ .../fluid/gfluidcore_func.dispatch.cpp | 32 ++- .../src/backends/fluid/gfluidcore_func.hpp | 25 +- .../backends/fluid/gfluidcore_func.simd.hpp | 237 ++++++++++++------ 9 files changed, 311 insertions(+), 170 deletions(-) diff --git a/modules/gapi/perf/common/gapi_core_perf_tests.hpp b/modules/gapi/perf/common/gapi_core_perf_tests.hpp index 39557f1acb..97b12f86b1 100644 --- a/modules/gapi/perf/common/gapi_core_perf_tests.hpp +++ b/modules/gapi/perf/common/gapi_core_perf_tests.hpp @@ -30,7 +30,7 @@ namespace opencv_test class AddPerfTest : public TestPerfParams> {}; class AddCPerfTest : public TestPerfParams> {}; class SubPerfTest : public TestPerfParams> {}; - class SubCPerfTest : public TestPerfParams> {}; + class SubCPerfTest : public TestPerfParams> {}; class SubRCPerfTest : public TestPerfParams> {}; class MulPerfTest : public TestPerfParams> {}; class MulDoublePerfTest : public TestPerfParams> {}; diff --git a/modules/gapi/perf/common/gapi_core_perf_tests_inl.hpp b/modules/gapi/perf/common/gapi_core_perf_tests_inl.hpp index b0568f9bae..6c286a5ce2 100644 --- a/modules/gapi/perf/common/gapi_core_perf_tests_inl.hpp +++ b/modules/gapi/perf/common/gapi_core_perf_tests_inl.hpp @@ -138,10 +138,13 @@ PERF_TEST_P_(SubPerfTest, TestPerformance) PERF_TEST_P_(SubCPerfTest, TestPerformance) { - Size sz = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - int dtype = get<2>(GetParam()); - cv::GCompileArgs compile_args = get<3>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + int dtype = -1; + cv::GCompileArgs compile_args; + + std::tie(cmpF, sz, type, dtype, compile_args) = GetParam(); initMatsRandU(type, sz, dtype, false); @@ -165,8 +168,9 @@ PERF_TEST_P_(SubCPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - // FIXIT unrealiable check: EXPECT_EQ(0, cv::countNonZero(out_mat_gapi != out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } diff --git a/modules/gapi/perf/cpu/gapi_core_perf_tests_cpu.cpp b/modules/gapi/perf/cpu/gapi_core_perf_tests_cpu.cpp index d7ead88327..31e9d25610 100644 --- a/modules/gapi/perf/cpu/gapi_core_perf_tests_cpu.cpp +++ b/modules/gapi/perf/cpu/gapi_core_perf_tests_cpu.cpp @@ -35,7 +35,8 @@ INSTANTIATE_TEST_CASE_P(SubPerfTestCPU, SubPerfTest, Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(SubCPerfTestCPU, SubCPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), Values(-1, CV_8U, CV_16U, CV_32F), Values(cv::compile_args(CORE_CPU)))); diff --git a/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp b/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp index a367937520..6ebd92dc4a 100644 --- a/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp +++ b/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp @@ -31,11 +31,12 @@ INSTANTIATE_TEST_CASE_P(SubPerfTestFluid, SubPerfTest, Values(-1, CV_8U, CV_32F), Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(SubCPerfTestFluid, SubCPerfTest, -// Combine(Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), -// Values(-1, CV_8U, CV_16U, CV_32F), -// Values(cv::compile_args(CORE_FLUID)))); + INSTANTIATE_TEST_CASE_P(SubCPerfTestFluid, SubCPerfTest, + Combine(Values(Tolerance_FloatRel_IntAbs(1e-6, 1).to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(-1, CV_8U, CV_16U, CV_16S, CV_32F), + Values(cv::compile_args(CORE_FLUID)))); // INSTANTIATE_TEST_CASE_P(SubRCPerfTestFluid, SubRCPerfTest, // Combine(Values(szSmall128, szVGA, sz720p, sz1080p), diff --git a/modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp b/modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp index d1d6deff2d..b4207c266d 100644 --- a/modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp +++ b/modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp @@ -33,7 +33,8 @@ INSTANTIATE_TEST_CASE_P(SubPerfTestGPU, SubPerfTest, Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(SubCPerfTestGPU, SubCPerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), Values( -1, CV_8U, CV_16U, CV_32F ), Values(cv::compile_args(CORE_GPU)))); diff --git a/modules/gapi/src/backends/fluid/gfluidcore.cpp b/modules/gapi/src/backends/fluid/gfluidcore.cpp index 16a87ea314..a737ad627b 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore.cpp +++ b/modules/gapi/src/backends/fluid/gfluidcore.cpp @@ -844,16 +844,12 @@ GAPI_FLUID_KERNEL(GFluidAbsDiff, cv::gapi::core::GAbsDiff, false) // //-------------------------------------- -static inline v_uint16x8 v_sub_16u(const v_uint16x8 &x, const v_uint16x8 &y) { return x - y; } static inline v_uint16x8 v_subr_16u(const v_uint16x8 &x, const v_uint16x8 &y) { return y - x; } -static inline v_float32x4 v_sub_32f(const v_float32x4 &x, const v_float32x4 &y) { return x - y; } static inline v_float32x4 v_subr_32f(const v_float32x4 &x, const v_float32x4 &y) { return y - x; } -static inline int s_sub_8u(uchar x, uchar y) { return x - y; } static inline int s_subr_8u(uchar x, uchar y) { return y - x; } -static inline float s_sub_32f(float x, float y) { return x - y; } static inline float s_subr_32f(float x, float y) { return y - x; } // manual SIMD if important case 8UC3 @@ -942,21 +938,11 @@ static void run_arithm_s1(uchar out[], const float in[], int width, const float } } -static void run_arithm_s_sub3(uchar out[], const uchar in[], int width, const uchar scalar[]) -{ - run_arithm_s3(out, in, width, scalar, v_sub_16u, s_sub_8u); -} - static void run_arithm_s_subr3(uchar out[], const uchar in[], int width, const uchar scalar[]) { run_arithm_s3(out, in, width, scalar, v_subr_16u, s_subr_8u); // reverse: subr } -static void run_arithm_s_sub1(uchar out[], const float in[], int width, const float scalar[]) -{ - run_arithm_s1(out, in, width, scalar, v_sub_32f, s_sub_32f); -} - static void run_arithm_s_subr1(uchar out[], const float in[], int width, const float scalar[]) { run_arithm_s1(out, in, width, scalar, v_subr_32f, s_subr_32f); // reverse: subr @@ -1273,6 +1259,7 @@ CV_ALWAYS_INLINE void run_arithm_s(Buffer &dst, const View &src, const float sca int width = dst.length(); int chan = dst.meta().chan; + const int length = width * chan; switch (arithm) { @@ -1280,37 +1267,21 @@ CV_ALWAYS_INLINE void run_arithm_s(Buffer &dst, const View &src, const float sca { int w = 0; #if CV_SIMD - w = addc_simd(in, scalar, out, width, chan); + w = addc_simd(in, scalar, out, length, chan); #endif - - for (; w < width * chan; ++w) + for (; w < length; ++w) out[w] = add(in[w], scalar[w % chan]); break; } case ARITHM_SUBTRACT: { - // What if we cast the scalar into the SRC type? - const SRC myscal[4] = { static_cast(scalar[0]), static_cast(scalar[1]), - static_cast(scalar[2]), static_cast(scalar[3]) }; - bool usemyscal = (myscal[0] == scalar[0]) && (myscal[1] == scalar[1]) && - (myscal[2] == scalar[2]) && (myscal[3] == scalar[3]); - - if (usemyscal) - { - if (std::is_same::value && - std::is_same::value && - chan == 3) - run_arithm_s_sub3((uchar*)out, (const uchar*)in, width, (const uchar*)myscal); - else if (std::is_same::value && - std::is_same::value && - chan == 1) - run_arithm_s_sub1((uchar*)out, (const float*)in, width, (const float*)myscal); - else - run_arithm_s(out, in, width, chan, myscal, sub); - } - else - run_arithm_s(out, in, width, chan, scalar, sub); + int w = 0; +#if CV_SIMD + w = subc_simd(in, scalar, out, length, chan); +#endif + for (; w < length; ++w) + out[w] = sub(in[w], scalar[w % chan]); break; } // TODO: optimize miltiplication and division @@ -1416,6 +1387,32 @@ GAPI_FLUID_KERNEL(GFluidAbsDiffC, cv::gapi::core::GAbsDiffC, true) } }; +CV_ALWAYS_INLINE void initScratchBuffer(Buffer& scratch) +{ +#if CV_SIMD + // 512 bits / 32 bits = 16 elements of float32 can contain a AVX 512 SIMD vector. + constexpr int maxNlanes = 16; + + // +2 is offset for 3-channel case. + // Offset is need to right load coefficients from scalar array to SIMD vectors for 3-channel case. + // Scalar array looks like: scalar[] = {C1, C2, C3, C1, C2, C3, ...} + // The first scalar SIMD vector should looks like: + // C1 C2 C3 C1 + // The second: + // C2 C3 C1 C2 + // The third: + // C3 C1 C2 C3 + constexpr int offset = 2; + constexpr int buflen = maxNlanes + offset; +#else + constexpr int buflen = 4; +#endif + cv::Size bufsize(buflen, 1); + GMatDesc bufdesc = { CV_32F, 1, bufsize }; + Buffer buffer(bufdesc); + scratch = std::move(buffer); +} + GAPI_FLUID_KERNEL(GFluidAddC, cv::gapi::core::GAddC, true) { static const int Window = 1; @@ -1458,59 +1455,62 @@ GAPI_FLUID_KERNEL(GFluidAddC, cv::gapi::core::GAddC, true) static void initScratch(const GMatDesc&, const GScalarDesc&, int, Buffer& scratch) { -#if CV_SIMD - // 512 bits / 32 bits = 16 elements of float32 can contain a AVX 512 SIMD vector. - constexpr int maxNlanes = 16; - - // +2 is offset for 3-channel case. - // Offset is need to right load coefficients from scalar array to SIMD vectors for 3-channel case. - // Scalar array looks like: scalar[] = {C1, C2, C3, C1, C2, C3, ...} - // The first scalar SIMD vector should looks like: - // C1 C2 C3 C1 - // The second: - // C2 C3 C1 C2 - // The third: - // C3 C1 C2 C3 - constexpr int offset = 2; - constexpr int buflen = maxNlanes + offset; -#else - constexpr int buflen = 4; -#endif - cv::Size bufsize(buflen, 1); - GMatDesc bufdesc = { CV_32F, 1, bufsize }; - Buffer buffer(bufdesc); - scratch = std::move(buffer); + initScratchBuffer(scratch); } - static void resetScratch(Buffer& /* scratch */) + static void resetScratch(Buffer& /*scratch*/) { } }; -GAPI_FLUID_KERNEL(GFluidSubC, cv::gapi::core::GSubC, false) +GAPI_FLUID_KERNEL(GFluidSubC, cv::gapi::core::GSubC, true) { static const int Window = 1; - static void run(const View &src, const cv::Scalar &_scalar, int /*dtype*/, Buffer &dst) + static void run(const View& src, const cv::Scalar& _scalar, int /*dtype*/, Buffer& dst, Buffer& scratch) { - const float scalar[4] = { - static_cast(_scalar[0]), - static_cast(_scalar[1]), - static_cast(_scalar[2]), - static_cast(_scalar[3]) - }; + GAPI_Assert(src.meta().chan <= 4); + + if (dst.y() == 0) + { + const int chan = src.meta().chan; + float* sc = scratch.OutLine(); + + for (int i = 0; i < scratch.length(); ++i) + sc[i] = static_cast(_scalar[i % chan]); + } + + const float* scalar = scratch.OutLine(); // DST SRC OP __VA_ARGS__ - UNARY_(uchar , uchar , run_arithm_s, dst, src, scalar, ARITHM_SUBTRACT); - UNARY_(uchar , short, run_arithm_s, dst, src, scalar, ARITHM_SUBTRACT); - UNARY_(uchar , float, run_arithm_s, dst, src, scalar, ARITHM_SUBTRACT); - UNARY_( short, short, run_arithm_s, dst, src, scalar, ARITHM_SUBTRACT); - UNARY_( float, uchar , run_arithm_s, dst, src, scalar, ARITHM_SUBTRACT); - UNARY_( float, short, run_arithm_s, dst, src, scalar, ARITHM_SUBTRACT); - UNARY_( float, float, run_arithm_s, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(uchar, uchar, run_arithm_s, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(uchar, ushort, run_arithm_s, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(uchar, short, run_arithm_s, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(uchar, float, run_arithm_s, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(ushort, ushort, run_arithm_s, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(ushort, short, run_arithm_s, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(ushort, uchar, run_arithm_s, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(ushort, float, run_arithm_s, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(short, short, run_arithm_s, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(short, ushort, run_arithm_s, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(short, uchar, run_arithm_s, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(short, float, run_arithm_s, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(float, uchar , run_arithm_s, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(float, ushort, run_arithm_s, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(float, short, run_arithm_s, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(float, float, run_arithm_s, dst, src, scalar, ARITHM_SUBTRACT); CV_Error(cv::Error::StsBadArg, "unsupported combination of types"); } + + static void initScratch(const GMatDesc&, const GScalarDesc&, int, Buffer& scratch) + { + initScratchBuffer(scratch); + } + + static void resetScratch(Buffer& /*scratch*/) + { + } }; GAPI_FLUID_KERNEL(GFluidSubRC, cv::gapi::core::GSubRC, false) diff --git a/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp b/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp index b6842e2390..668ac3a4bb 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp +++ b/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp @@ -65,7 +65,6 @@ int mul_simd(const SRC in1[], const SRC in2[], DST out[], \ CV_CPU_DISPATCH_MODES_ALL); \ } - MUL_SIMD(uchar, uchar) MUL_SIMD(ushort, uchar) MUL_SIMD(short, uchar) @@ -87,9 +86,9 @@ MUL_SIMD(float, float) #define ADDC_SIMD(SRC, DST) \ int addc_simd(const SRC in[], const float scalar[], DST out[], \ - const int width, const int chan) \ + const int length, const int chan) \ { \ - CV_CPU_DISPATCH(addc_simd, (in, scalar, out, width, chan), \ + CV_CPU_DISPATCH(addc_simd, (in, scalar, out, length, chan), \ CV_CPU_DISPATCH_MODES_ALL); \ } @@ -112,6 +111,33 @@ ADDC_SIMD(float, float) #undef ADDC_SIMD +#define SUBC_SIMD(SRC, DST) \ +int subc_simd(const SRC in[], const float scalar[], DST out[], \ + const int length, const int chan) \ +{ \ + CV_CPU_DISPATCH(subc_simd, (in, scalar, out, length, chan), \ + CV_CPU_DISPATCH_MODES_ALL); \ +} + +SUBC_SIMD(uchar, uchar) +SUBC_SIMD(ushort, uchar) +SUBC_SIMD(short, uchar) +SUBC_SIMD(float, uchar) +SUBC_SIMD(short, short) +SUBC_SIMD(ushort, short) +SUBC_SIMD(uchar, short) +SUBC_SIMD(float, short) +SUBC_SIMD(ushort, ushort) +SUBC_SIMD(uchar, ushort) +SUBC_SIMD(short, ushort) +SUBC_SIMD(float, ushort) +SUBC_SIMD(uchar, float) +SUBC_SIMD(ushort, float) +SUBC_SIMD(short, float) +SUBC_SIMD(float, float) + +#undef SUBC_SIMD + } // namespace fluid } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/backends/fluid/gfluidcore_func.hpp b/modules/gapi/src/backends/fluid/gfluidcore_func.hpp index ba48f7a621..e6c0d4fe9b 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore_func.hpp +++ b/modules/gapi/src/backends/fluid/gfluidcore_func.hpp @@ -62,7 +62,7 @@ MUL_SIMD(float, float) #define ADDC_SIMD(SRC, DST) \ int addc_simd(const SRC in[], const float scalar[], DST out[], \ - const int width, const int chan); + const int length, const int chan); ADDC_SIMD(uchar, uchar) ADDC_SIMD(ushort, uchar) @@ -83,6 +83,29 @@ ADDC_SIMD(float, float) #undef ADDC_SIMD +#define SUBC_SIMD(SRC, DST) \ +int subc_simd(const SRC in[], const float scalar[], DST out[], \ + const int length, const int chan); + +SUBC_SIMD(uchar, uchar) +SUBC_SIMD(ushort, uchar) +SUBC_SIMD(short, uchar) +SUBC_SIMD(float, uchar) +SUBC_SIMD(short, short) +SUBC_SIMD(ushort, short) +SUBC_SIMD(uchar, short) +SUBC_SIMD(float, short) +SUBC_SIMD(ushort, ushort) +SUBC_SIMD(uchar, ushort) +SUBC_SIMD(short, ushort) +SUBC_SIMD(float, ushort) +SUBC_SIMD(uchar, float) +SUBC_SIMD(ushort, float) +SUBC_SIMD(short, float) +SUBC_SIMD(float, float) + +#undef SUBC_SIMD + } // namespace fluid } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp b/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp index 071c05633a..aed5359e7b 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp +++ b/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp @@ -83,7 +83,7 @@ MUL_SIMD(float, float) #define ADDC_SIMD(SRC, DST) \ int addc_simd(const SRC in[], const float scalar[], DST out[], \ - const int width, const int chan); + const int length, const int chan); ADDC_SIMD(uchar, uchar) ADDC_SIMD(ushort, uchar) @@ -104,6 +104,29 @@ ADDC_SIMD(float, float) #undef ADDC_SIMD +#define SUBC_SIMD(SRC, DST) \ +int subc_simd(const SRC in[], const float scalar[], DST out[], \ + const int length, const int chan); + +SUBC_SIMD(uchar, uchar) +SUBC_SIMD(ushort, uchar) +SUBC_SIMD(short, uchar) +SUBC_SIMD(float, uchar) +SUBC_SIMD(short, short) +SUBC_SIMD(ushort, short) +SUBC_SIMD(uchar, short) +SUBC_SIMD(float, short) +SUBC_SIMD(ushort, ushort) +SUBC_SIMD(uchar, ushort) +SUBC_SIMD(short, ushort) +SUBC_SIMD(float, ushort) +SUBC_SIMD(uchar, float) +SUBC_SIMD(ushort, float) +SUBC_SIMD(short, float) +SUBC_SIMD(float, float) + +#undef SUBC_SIMD + #ifndef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY struct scale_tag {}; @@ -851,10 +874,13 @@ MUL_SIMD(float, float) // //------------------------- -CV_ALWAYS_INLINE void addc_pack_store_c3(short* outx, const v_int32& c1, - const v_int32& c2, const v_int32& c3, - const v_int32& c4, const v_int32& c5, - const v_int32& c6) +struct add_tag {}; +struct sub_tag {}; + +CV_ALWAYS_INLINE void arithmOpScalar_pack_store_c3(short* outx, const v_int32& c1, + const v_int32& c2, const v_int32& c3, + const v_int32& c4, const v_int32& c5, + const v_int32& c6) { constexpr int nlanes = v_int16::nlanes; vx_store(outx, v_pack(c1, c2)); @@ -862,10 +888,10 @@ CV_ALWAYS_INLINE void addc_pack_store_c3(short* outx, const v_int32& c1, vx_store(&outx[2*nlanes], v_pack(c5, c6)); } -CV_ALWAYS_INLINE void addc_pack_store_c3(ushort* outx, const v_int32& c1, - const v_int32& c2, const v_int32& c3, - const v_int32& c4, const v_int32& c5, - const v_int32& c6) +CV_ALWAYS_INLINE void arithmOpScalar_pack_store_c3(ushort* outx, const v_int32& c1, + const v_int32& c2, const v_int32& c3, + const v_int32& c4, const v_int32& c5, + const v_int32& c6) { constexpr int nlanes = v_uint16::nlanes; vx_store(outx, v_pack_u(c1, c2)); @@ -873,50 +899,64 @@ CV_ALWAYS_INLINE void addc_pack_store_c3(ushort* outx, const v_int32& c1, vx_store(&outx[2*nlanes], v_pack_u(c5, c6)); } -template +CV_ALWAYS_INLINE v_float32 oper(add_tag, const v_float32& a, const v_float32& sc) +{ + return a + sc; +} + +CV_ALWAYS_INLINE v_float32 oper(sub_tag, const v_float32& a, const v_float32& sc) +{ + return a - sc; +} + +template CV_ALWAYS_INLINE typename std::enable_if<(std::is_same::value || std::is_same::value), void>::type -addc_simd_common_impl(const SRC* inx, DST* outx, const v_float32& sc, const int nlanes) +arithmOpScalar_simd_common_impl(oper_tag t, const SRC* inx, DST* outx, + const v_float32& sc, const int nlanes) { v_float32 a1 = vg_load_f32(inx); v_float32 a2 = vg_load_f32(&inx[nlanes/2]); - v_store_i16(outx, v_round(a1 + sc), v_round(a2 + sc)); + v_store_i16(outx, v_round(oper(t, a1, sc)), v_round(oper(t, a2, sc))); } //------------------------------------------------------------------------------------------------- -template -CV_ALWAYS_INLINE void addc_simd_common_impl(const SRC* inx, uchar* outx, const v_float32& sc, const int nlanes) +template +CV_ALWAYS_INLINE void arithmOpScalar_simd_common_impl(oper_tag t, const SRC* inx, + uchar* outx, const v_float32& sc, + const int nlanes) { v_float32 a1 = vg_load_f32(inx); v_float32 a2 = vg_load_f32(&inx[nlanes/4]); v_float32 a3 = vg_load_f32(&inx[nlanes/2]); v_float32 a4 = vg_load_f32(&inx[3 * nlanes/4]); - vx_store(outx, v_pack_u(v_pack(v_round(a1 + sc), - v_round(a2 + sc)), - v_pack(v_round(a3 + sc), - v_round(a4 + sc)))); + vx_store(outx, v_pack_u(v_pack(v_round(oper(t, a1, sc)), + v_round(oper(t, a2, sc))), + v_pack(v_round(oper(t, a3, sc)), + v_round(oper(t, a4, sc))))); } //------------------------------------------------------------------------------------------------- -template -CV_ALWAYS_INLINE void addc_simd_common_impl(const SRC* inx, float* outx, const v_float32& sc, const int) +template +CV_ALWAYS_INLINE void arithmOpScalar_simd_common_impl(oper_tag t, const SRC* inx, + float* outx, const v_float32& sc, const int) { v_float32 a1 = vg_load_f32(inx); - vx_store(outx, a1 + sc); + vx_store(outx, oper(t, a1, sc)); } //------------------------------------------------------------------------------------------------- -template +template CV_ALWAYS_INLINE typename std::enable_if::value || std::is_same::value, void>::type -addc_simd_c3_impl(const SRC* inx, DST* outx, const v_float32& s1, const v_float32& s2, +arithmOpScalar_simd_c3_impl(oper_tag t, const SRC* inx, DST* outx, const v_float32& s1, const v_float32& s2, const v_float32& s3, const int nlanes) { v_float32 a1 = vg_load_f32(inx); @@ -926,60 +966,62 @@ addc_simd_c3_impl(const SRC* inx, DST* outx, const v_float32& s1, const v_float3 v_float32 a5 = vg_load_f32(&inx[2 * nlanes]); v_float32 a6 = vg_load_f32(&inx[5 * nlanes / 2]); - addc_pack_store_c3(outx, v_round(a1 + s1), - v_round(a2 + s2), - v_round(a3 + s3), - v_round(a4 + s1), - v_round(a5 + s2), - v_round(a6 + s3)); + arithmOpScalar_pack_store_c3(outx, v_round(oper(t, a1, s1)), + v_round(oper(t, a2, s2)), + v_round(oper(t, a3, s3)), + v_round(oper(t, a4, s1)), + v_round(oper(t, a5, s2)), + v_round(oper(t, a6, s3))); } //------------------------------------------------------------------------------------------------- -template -CV_ALWAYS_INLINE void addc_simd_c3_impl(const SRC* inx, uchar* outx, - const v_float32& s1, const v_float32& s2, - const v_float32& s3, const int nlanes) +template +CV_ALWAYS_INLINE void arithmOpScalar_simd_c3_impl(oper_tag t, const SRC* inx, uchar* outx, + const v_float32& s1, const v_float32& s2, + const v_float32& s3, const int nlanes) { vx_store(outx, - v_pack_u(v_pack(v_round(vg_load_f32(inx) + s1), - v_round(vg_load_f32(&inx[nlanes/4]) + s2)), - v_pack(v_round(vg_load_f32(&inx[nlanes/2]) + s3), - v_round(vg_load_f32(&inx[3*nlanes/4]) + s1)))); + v_pack_u(v_pack(v_round(oper(t, vg_load_f32(inx), s1)), + v_round(oper(t, vg_load_f32(&inx[nlanes/4]), s2))), + v_pack(v_round(oper(t, vg_load_f32(&inx[nlanes/2]), s3)), + v_round(oper(t, vg_load_f32(&inx[3*nlanes/4]), s1))))); vx_store(&outx[nlanes], - v_pack_u(v_pack(v_round(vg_load_f32(&inx[nlanes]) + s2), - v_round(vg_load_f32(&inx[5*nlanes/4]) + s3)), - v_pack(v_round(vg_load_f32(&inx[3*nlanes/2]) + s1), - v_round(vg_load_f32(&inx[7*nlanes/4]) + s2)))); + v_pack_u(v_pack(v_round(oper(t, vg_load_f32(&inx[nlanes]), s2)), + v_round(oper(t, vg_load_f32(&inx[5*nlanes/4]), s3))), + v_pack(v_round(oper(t, vg_load_f32(&inx[3*nlanes/2]), s1)), + v_round(oper(t, vg_load_f32(&inx[7*nlanes/4]), s2))))); vx_store(&outx[2 * nlanes], - v_pack_u(v_pack(v_round(vg_load_f32(&inx[2*nlanes]) + s3), - v_round(vg_load_f32(&inx[9*nlanes/4]) + s1)), - v_pack(v_round(vg_load_f32(&inx[5*nlanes/2]) + s2), - v_round(vg_load_f32(&inx[11*nlanes/4]) + s3)))); + v_pack_u(v_pack(v_round(oper(t, vg_load_f32(&inx[2*nlanes]), s3)), + v_round(oper(t, vg_load_f32(&inx[9*nlanes/4]), s1))), + v_pack(v_round(oper(t, vg_load_f32(&inx[5*nlanes/2]), s2)), + v_round(oper(t, vg_load_f32(&inx[11*nlanes/4]), s3))))); } //------------------------------------------------------------------------------------------------- -template -CV_ALWAYS_INLINE void addc_simd_c3_impl(const SRC* in, float* out, - const v_float32& s1, const v_float32& s2, - const v_float32& s3, const int nlanes) +template +CV_ALWAYS_INLINE void arithmOpScalar_simd_c3_impl(oper_tag t, const SRC* in, float* out, + const v_float32& s1, const v_float32& s2, + const v_float32& s3, const int nlanes) { v_float32 a1 = vg_load_f32(in); v_float32 a2 = vg_load_f32(&in[nlanes]); v_float32 a3 = vg_load_f32(&in[2*nlanes]); - vx_store(out, a1 + s1); - vx_store(&out[nlanes], a2 + s2); - vx_store(&out[2*nlanes], a3 + s3); + vx_store(out, oper(t, a1, s1)); + vx_store(&out[nlanes], oper(t, a2, s2)); + vx_store(&out[2*nlanes], oper(t, a3, s3)); } //------------------------------------------------------------------------------------------------- -template -CV_ALWAYS_INLINE int addc_simd_c3(const SRC in[], const float scalar[], DST out[], const int length) +template +CV_ALWAYS_INLINE int arithmOpScalar_simd_c3(oper_tag t, const SRC in[], + const float scalar[], DST out[], + const int length) { constexpr int chan = 3; constexpr int nlanes = vector_type_of_t::nlanes; @@ -1002,7 +1044,7 @@ CV_ALWAYS_INLINE int addc_simd_c3(const SRC in[], const float scalar[], DST out[ { for (; x <= length - lanes; x += lanes) { - addc_simd_c3_impl(&in[x], &out[x], s1, s2, s3, nlanes); + arithmOpScalar_simd_c3_impl(t, &in[x], &out[x], s1, s2, s3, nlanes); } if (x < length) @@ -1015,8 +1057,12 @@ CV_ALWAYS_INLINE int addc_simd_c3(const SRC in[], const float scalar[], DST out[ return x; } -template -CV_ALWAYS_INLINE int addc_simd_common(const SRC in[], const float scalar[], DST out[], const int length) +//------------------------------------------------------------------------------------------------- + +template +CV_ALWAYS_INLINE int arithmOpScalar_simd_common(oper_tag t, const SRC in[], + const float scalar[], DST out[], + const int length) { constexpr int nlanes = vector_type_of_t::nlanes; @@ -1030,7 +1076,7 @@ CV_ALWAYS_INLINE int addc_simd_common(const SRC in[], const float scalar[], DST { for (; x <= length - nlanes; x += nlanes) { - addc_simd_common_impl(&in[x], &out[x], sc, nlanes); + arithmOpScalar_simd_common_impl(t, &in[x], &out[x], sc, nlanes); } if (x < length) @@ -1043,24 +1089,25 @@ CV_ALWAYS_INLINE int addc_simd_common(const SRC in[], const float scalar[], DST return x; } -#define ADDC_SIMD(SRC, DST) \ -int addc_simd(const SRC in[], const float scalar[], DST out[], \ - const int width, const int chan) \ -{ \ - const int length = width * chan; \ - switch (chan) \ - { \ - case 1: \ - case 2: \ - case 4: \ - return addc_simd_common(in, scalar, out, length); \ - case 3: \ - return addc_simd_c3(in, scalar, out, length); \ - default: \ - GAPI_Assert(chan <= 4); \ - break; \ - } \ - return 0; \ + + +#define ADDC_SIMD(SRC, DST) \ +int addc_simd(const SRC in[], const float scalar[], DST out[], \ + const int length, const int chan) \ +{ \ + switch (chan) \ + { \ + case 1: \ + case 2: \ + case 4: \ + return arithmOpScalar_simd_common(add_tag{}, in, scalar, out, length); \ + case 3: \ + return arithmOpScalar_simd_c3(add_tag{}, in, scalar, out, length); \ + default: \ + GAPI_Assert(chan <= 4); \ + break; \ + } \ + return 0; \ } ADDC_SIMD(uchar, uchar) @@ -1082,6 +1129,44 @@ ADDC_SIMD(float, float) #undef ADDC_SIMD +#define SUBC_SIMD(SRC, DST) \ +int subc_simd(const SRC in[], const float scalar[], DST out[], \ + const int length, const int chan) \ +{ \ + switch (chan) \ + { \ + case 1: \ + case 2: \ + case 4: \ + return arithmOpScalar_simd_common(sub_tag{}, in, scalar, out, length); \ + case 3: \ + return arithmOpScalar_simd_c3(sub_tag{}, in, scalar, out, length); \ + default: \ + GAPI_Assert(chan <= 4); \ + break; \ + } \ + return 0; \ +} + +SUBC_SIMD(uchar, uchar) +SUBC_SIMD(ushort, uchar) +SUBC_SIMD(short, uchar) +SUBC_SIMD(float, uchar) +SUBC_SIMD(short, short) +SUBC_SIMD(ushort, short) +SUBC_SIMD(uchar, short) +SUBC_SIMD(float, short) +SUBC_SIMD(ushort, ushort) +SUBC_SIMD(uchar, ushort) +SUBC_SIMD(short, ushort) +SUBC_SIMD(float, ushort) +SUBC_SIMD(uchar, float) +SUBC_SIMD(ushort, float) +SUBC_SIMD(short, float) +SUBC_SIMD(float, float) + +#undef SUBC_SIMD + #endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY CV_CPU_OPTIMIZATION_NAMESPACE_END From f55c9ed1ba5f40a67014bd030463f80bece351a0 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 2 Dec 2021 05:52:12 +0000 Subject: [PATCH 120/226] dnn(test): drop non OCV/CPU cases for Int8 - zero code coverage and up to x3-x8 tests slowdown - implementation executes OCV/CPU in all cases - wrong skip conditions --- modules/dnn/test/test_int8_layers.cpp | 202 ++------------------------ 1 file changed, 11 insertions(+), 191 deletions(-) diff --git a/modules/dnn/test/test_int8_layers.cpp b/modules/dnn/test/test_int8_layers.cpp index 85c30a4271..c181dfa5eb 100644 --- a/modules/dnn/test/test_int8_layers.cpp +++ b/modules/dnn/test/test_int8_layers.cpp @@ -8,6 +8,13 @@ #include namespace opencv_test { namespace { +testing::internal::ParamGenerator< tuple > dnnBackendsAndTargetsInt8() +{ + std::vector< tuple > targets; + targets.push_back(make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)); + return testing::ValuesIn(targets); +} + template static std::string _tf(TString filename) { @@ -341,7 +348,7 @@ TEST_P(Test_Int8_layers, Eltwise) testLayer("split_max", "ONNX", 0.004, 0.012); } -INSTANTIATE_TEST_CASE_P(/**/, Test_Int8_layers, dnnBackendsAndTargets()); +INSTANTIATE_TEST_CASE_P(/**/, Test_Int8_layers, dnnBackendsAndTargetsInt8()); class Test_Int8_nets : public DNNTestLayer { @@ -657,11 +664,6 @@ TEST_P(Test_Int8_nets, CaffeNet) if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019030000) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD - && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif float l1 = 4e-5, lInf = 0.0025; testONNXNet("caffenet", l1, lInf); } @@ -679,11 +681,6 @@ TEST_P(Test_Int8_nets, RCNN_ILSVRC13) if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019030000) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD - && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif float l1 = 0.02, lInf = 0.042; testONNXNet("rcnn_ilsvrc13", l1, lInf); } @@ -715,12 +712,6 @@ TEST_P(Test_Int8_nets, Shufflenet) if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) - { - if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); - if (target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); - if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); - } testONNXNet("shufflenet", default_l1, default_lInf); } @@ -767,12 +758,6 @@ TEST_P(Test_Int8_nets, MobileNet_v1_SSD_PPN) if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) - applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, - CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif - Net net = readNetFromTensorflow(findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pb", false), findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pbtxt")); @@ -792,11 +777,6 @@ TEST_P(Test_Int8_nets, Inception_v2_SSD) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2019010000) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD && - getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif Net net = readNetFromTensorflow(findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pb", false), findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pbtxt")); @@ -875,25 +855,9 @@ TEST_P(Test_Int8_nets, FasterRCNN_resnet50) if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); -#ifdef INF_ENGINE_RELEASE - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && - (INF_ENGINE_VER_MAJOR_LT(2019020000) || target != DNN_TARGET_CPU)) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); - - if (INF_ENGINE_VER_MAJOR_GT(2019030000) && - backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); -#endif - - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); - if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (backend == DNN_BACKEND_CUDA && target == DNN_TARGET_CUDA_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA_FP16); - Net net = readNetFromTensorflow(findDataFile("dnn/faster_rcnn_resnet50_coco_2018_01_28.pb", false), findDataFile("dnn/faster_rcnn_resnet50_coco_2018_01_28.pbtxt")); @@ -918,25 +882,9 @@ TEST_P(Test_Int8_nets, FasterRCNN_inceptionv2) if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); -#ifdef INF_ENGINE_RELEASE - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && - (INF_ENGINE_VER_MAJOR_LT(2019020000) || target != DNN_TARGET_CPU)) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); - - if (INF_ENGINE_VER_MAJOR_GT(2019030000) && - backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); -#endif - - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); - if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (backend == DNN_BACKEND_CUDA && target == DNN_TARGET_CUDA_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA_FP16); - Net net = readNetFromTensorflow(findDataFile("dnn/faster_rcnn_inception_v2_coco_2018_01_28.pb", false), findDataFile("dnn/faster_rcnn_inception_v2_coco_2018_01_28.pbtxt")); @@ -965,17 +913,6 @@ TEST_P(Test_Int8_nets, FasterRCNN_vgg16) if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); -#if defined(INF_ENGINE_RELEASE) - if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) - applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); - - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); - - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); -#endif - Net net = readNetFromCaffe(findDataFile("dnn/faster_rcnn_vgg16.prototxt"), findDataFile("dnn/VGG16_faster_rcnn_final.caffemodel", false)); @@ -1003,17 +940,6 @@ TEST_P(Test_Int8_nets, FasterRCNN_zf) if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || - backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); - - if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || - backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); - - if (target == DNN_TARGET_CUDA_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA_FP16); - Net net = readNetFromCaffe(findDataFile("dnn/faster_rcnn_zf.prototxt"), findDataFile("dnn/ZF_faster_rcnn_final.caffemodel", false)); @@ -1038,14 +964,6 @@ TEST_P(Test_Int8_nets, RFCN) if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || - backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); - - if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || - backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); - Net net = readNetFromCaffe(findDataFile("dnn/rfcn_pascal_voc_resnet50.prototxt"), findDataFile("dnn/resnet50_rfcn_final.caffemodel", false)); @@ -1072,22 +990,6 @@ TEST_P(Test_Int8_nets, YoloVoc) if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); -#endif -#if defined(INF_ENGINE_RELEASE) - if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && - target == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); -#endif - Mat ref = (Mat_(6, 7) << 0, 6, 0.750469f, 0.577374f, 0.127391f, 0.902949f, 0.300809f, 0, 1, 0.780879f, 0.270762f, 0.264102f, 0.732475f, 0.745412f, 0, 11, 0.901615f, 0.1386f, 0.338509f, 0.421337f, 0.938789f, @@ -1119,18 +1021,6 @@ TEST_P(Test_Int8_nets, TinyYoloVoc) if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif -#if defined(INF_ENGINE_RELEASE) - if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && - target == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); -#endif - Mat ref = (Mat_(4, 7) << 0, 6, 0.761967f, 0.579042f, 0.159161f, 0.894482f, 0.31994f, 0, 11, 0.780595f, 0.129696f, 0.386467f, 0.445275f, 0.920994f, 1, 6, 0.651450f, 0.460526f, 0.458019f, 0.522527f, 0.5341f, @@ -1160,16 +1050,6 @@ TEST_P(Test_Int8_nets, YOLOv3) if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif - - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); - const int N0 = 3; const int N1 = 6; static const float ref_[/* (N0 + N1) * 7 */] = { @@ -1195,19 +1075,6 @@ TEST_P(Test_Int8_nets, YOLOv3) testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff, confThreshold); } -#if defined(INF_ENGINE_RELEASE) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) - { - if (target == DNN_TARGET_OPENCL) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); - else if (target == DNN_TARGET_OPENCL_FP16 && INF_ENGINE_VER_MAJOR_LE(202010000)) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); - else if (target == DNN_TARGET_MYRIAD && - getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); - } -#endif - { SCOPED_TRACE("batch size 2"); testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff, confThreshold); @@ -1223,17 +1090,6 @@ TEST_P(Test_Int8_nets, YOLOv4) if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif -#if defined(INF_ENGINE_RELEASE) - if (target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif - const int N0 = 3; const int N1 = 7; static const float ref_[/* (N0 + N1) * 7 */] = { @@ -1262,19 +1118,6 @@ TEST_P(Test_Int8_nets, YOLOv4) { SCOPED_TRACE("batch size 2"); -#if defined(INF_ENGINE_RELEASE) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) - { - if (target == DNN_TARGET_OPENCL) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); - else if (target == DNN_TARGET_OPENCL_FP16 && INF_ENGINE_VER_MAJOR_LE(202010000)) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); - else if (target == DNN_TARGET_MYRIAD && - getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); - } -#endif - testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff); } } @@ -1290,11 +1133,6 @@ TEST_P(Test_Int8_nets, YOLOv4_tiny) if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2021010000) - if (target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif - const float confThreshold = 0.6; const int N0 = 2; @@ -1314,38 +1152,20 @@ TEST_P(Test_Int8_nets, YOLOv4_tiny) double scoreDiff = 0.12; double iouDiff = target == DNN_TARGET_OPENCL_FP16 ? 0.2 : 0.082; -#if defined(INF_ENGINE_RELEASE) - if (target == DNN_TARGET_MYRIAD) // bad accuracy - iouDiff = std::numeric_limits::quiet_NaN(); - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_OPENCL) - iouDiff = std::numeric_limits::quiet_NaN(); - if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || - backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16) - iouDiff = std::numeric_limits::quiet_NaN(); -#endif - { SCOPED_TRACE("batch size 1"); testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff, confThreshold); } + throw SkipTestException("batch2: bad accuracy on second image"); /* bad accuracy on second image { SCOPED_TRACE("batch size 2"); testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff, confThreshold); } */ - -#if defined(INF_ENGINE_RELEASE) - if (target == DNN_TARGET_MYRIAD) // bad accuracy - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION); - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_OPENCL) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); - if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || - backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif } -INSTANTIATE_TEST_CASE_P(/**/, Test_Int8_nets, dnnBackendsAndTargets()); +INSTANTIATE_TEST_CASE_P(/**/, Test_Int8_nets, dnnBackendsAndTargetsInt8()); + }} // namespace From bd396e1fd5eb2b31fb55daf3b022ecbcf6f9510f Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 30 Nov 2021 12:08:35 +0000 Subject: [PATCH 121/226] dnn(test): re-enable tests which works with OpenVINO 2021.4.x (3.4) --- modules/dnn/test/test_backends.cpp | 2 +- modules/dnn/test/test_caffe_importer.cpp | 30 +++- modules/dnn/test/test_darknet_importer.cpp | 72 +++++++-- modules/dnn/test/test_halide_layers.cpp | 26 ++- modules/dnn/test/test_layers.cpp | 12 ++ modules/dnn/test/test_misc.cpp | 3 +- modules/dnn/test/test_onnx_importer.cpp | 176 +++++++++++++++++---- modules/dnn/test/test_tf_importer.cpp | 97 ++++++++++-- modules/dnn/test/test_torch_importer.cpp | 11 +- 9 files changed, 357 insertions(+), 72 deletions(-) diff --git a/modules/dnn/test/test_backends.cpp b/modules/dnn/test/test_backends.cpp index 19e0729abb..d1df4c35aa 100644 --- a/modules/dnn/test/test_backends.cpp +++ b/modules/dnn/test/test_backends.cpp @@ -248,7 +248,7 @@ TEST_P(DNNTestNetwork, MobileNet_SSD_v1_TensorFlow_Different_Width_Height) { if (backend == DNN_BACKEND_HALIDE) applyTestTag(CV_TEST_TAG_DNN_SKIP_HALIDE); -#if defined(INF_ENGINE_RELEASE) +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); diff --git a/modules/dnn/test/test_caffe_importer.cpp b/modules/dnn/test/test_caffe_importer.cpp index 18b8d5ad82..7249fb4e9f 100644 --- a/modules/dnn/test/test_caffe_importer.cpp +++ b/modules/dnn/test/test_caffe_importer.cpp @@ -112,10 +112,12 @@ TEST(Test_Caffe, read_googlenet) TEST_P(Test_Caffe_nets, Axpy) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif String proto = _tf("axpy.prototxt"); Net net = readNetFromCaffe(proto); @@ -150,8 +152,17 @@ TEST_P(Test_Caffe_nets, Axpy) } } } - float l1 = (target == DNN_TARGET_OPENCL_FP16) ? 2e-4 : 1e-5; - float lInf = (target == DNN_TARGET_OPENCL_FP16) ? 1e-3 : 1e-4; + float l1 = 1e-5, lInf = 1e-4; + if (target == DNN_TARGET_OPENCL_FP16) + { + l1 = 2e-4; + lInf = 1e-3; + } + if (target == DNN_TARGET_MYRIAD) + { + l1 = 0.001; + lInf = 0.001; + } normAssert(ref, out, "", l1, lInf); } @@ -657,7 +668,7 @@ TEST_P(Test_Caffe_nets, FasterRCNN_vgg16) CV_TEST_TAG_DEBUG_VERYLONG ); -#if defined(INF_ENGINE_RELEASE) +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); @@ -668,6 +679,19 @@ TEST_P(Test_Caffe_nets, FasterRCNN_vgg16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); #endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // IE exception: Ngraph operation Reshape with name rpn_cls_score_reshape has dynamic output shape on 0 port, but CPU plug-in supports only static shape + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); + // Check 'backward_compatible_check || in_out_elements_equal' failed at core/src/op/reshape.cpp:390: + // While validating node 'v1::Reshape bbox_pred_reshape (bbox_pred[0]:f32{1,84}, Constant_241202[0]:i64{4}) -> (f32{?,?,?,?})' with friendly_name 'bbox_pred_reshape': + // Requested output shape {1,6300,4,1} is incompatible with input shape Shape{1, 84} + if (target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + static Mat ref = (Mat_(3, 7) << 0, 2, 0.949398, 99.2454, 210.141, 601.205, 462.849, 0, 7, 0.997022, 481.841, 92.3218, 722.685, 175.953, 0, 12, 0.993028, 133.221, 189.377, 350.994, 563.166); diff --git a/modules/dnn/test/test_darknet_importer.cpp b/modules/dnn/test/test_darknet_importer.cpp index 885bd6eb6b..f069ad190d 100644 --- a/modules/dnn/test/test_darknet_importer.cpp +++ b/modules/dnn/test/test_darknet_importer.cpp @@ -121,7 +121,7 @@ public: { SCOPED_TRACE("batch size 2"); -#if defined(INF_ENGINE_RELEASE) +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (target == DNN_TARGET_MYRIAD && name == "shortcut") applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); #endif @@ -429,22 +429,31 @@ TEST_P(Test_Darknet_nets_async, Accuracy) { Backend backendId = get<0>(get<1>(GetParam())); Target targetId = get<1>(get<1>(GetParam())); + std::string prefix = get<0>(GetParam()); + applyTestTag(CV_TEST_TAG_MEMORY_512MB); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (INF_ENGINE_VER_MAJOR_LT(2019020000) && backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); - applyTestTag(CV_TEST_TAG_MEMORY_512MB); if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); - - std::string prefix = get<0>(GetParam()); - - if (targetId == DNN_TARGET_MYRIAD && prefix == "yolov4") // NC_OUT_OF_MEMORY - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif if (backendId != DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && backendId != DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) throw SkipTestException("No support for async forward"); +#if defined(INF_ENGINE_RELEASE) +#if INF_ENGINE_VER_MAJOR_GE(2021040000) + if (targetId == DNN_TARGET_MYRIAD && prefix == "yolov3") // NC_OUT_OF_MEMORY + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#else + if (targetId == DNN_TARGET_MYRIAD && prefix == "yolov4") // NC_OUT_OF_MEMORY + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif +#endif + const int numInputs = 2; std::vector inputs(numInputs); int blobSize[] = {1, 3, 416, 416}; @@ -472,6 +481,34 @@ TEST_P(Test_Darknet_nets_async, Accuracy) netAsync.setPreferableBackend(backendId); netAsync.setPreferableTarget(targetId); + double l1 = 0.0; + double lInf = 0.0; +#if defined(INF_ENGINE_RELEASE) + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + if (targetId == DNN_TARGET_MYRIAD && prefix == "yolo-voc") + { + l1 = 0.02; + lInf = 0.15; + } + if (targetId == DNN_TARGET_OPENCL_FP16 && prefix == "yolo-voc") + { + l1 = 0.02; + lInf = 0.1; + } + if (targetId == DNN_TARGET_OPENCL_FP16 && prefix == "yolov3") + { + l1 = 0.001; + lInf = 0.007; + } + if (targetId == DNN_TARGET_OPENCL_FP16 && prefix == "yolov4") + { + l1 = 0.001; + lInf = 0.005; + } + } +#endif + // Run asynchronously. To make test more robust, process inputs in the reversed order. for (int i = numInputs - 1; i >= 0; --i) { @@ -481,7 +518,7 @@ TEST_P(Test_Darknet_nets_async, Accuracy) ASSERT_TRUE(out.valid()); Mat result; EXPECT_TRUE(out.get(result, async_timeout)); - normAssert(refs[i], result, format("Index: %d", i).c_str(), 0, 0); + normAssert(refs[i], result, format("Index: %d", i).c_str(), l1, lInf); } } @@ -836,10 +873,23 @@ TEST_P(Test_Darknet_layers, avgpool_softmax) TEST_P(Test_Darknet_layers, region) { -#if defined(INF_ENGINE_RELEASE) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && INF_ENGINE_VER_MAJOR_GE(2020020000)) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && INF_ENGINE_VER_MAJOR_GE(2020020000)) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); #endif + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // accuracy on CPU, OpenCL + // Expected: (normInf) <= (lInf), actual: 0.763223 vs 0.0001 + // |ref| = 1.207319974899292 + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_CPU) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_CPU, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); +#endif + testDarknetLayer("region"); } diff --git a/modules/dnn/test/test_halide_layers.cpp b/modules/dnn/test/test_halide_layers.cpp index b24968b8e8..405587eec7 100644 --- a/modules/dnn/test/test_halide_layers.cpp +++ b/modules/dnn/test/test_halide_layers.cpp @@ -240,9 +240,11 @@ TEST_P(LRN, Accuracy) Backend backendId = get<0>(get<5>(GetParam())); Target targetId = get<1>(get<5>(GetParam())); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if ((inSize.width == 5 || inSize.height == 5) && targetId == DNN_TARGET_MYRIAD && nrmType == "ACROSS_CHANNELS") applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); +#endif LayerParams lp; lp.set("norm_region", nrmType); @@ -407,12 +409,14 @@ TEST_P(FullyConnected, Accuracy) bool hasBias = get<3>(GetParam()); Backend backendId = get<0>(get<4>(GetParam())); Target targetId = get<1>(get<4>(GetParam())); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if ((backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && (targetId == DNN_TARGET_OPENCL_FP16 || (targetId == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X))) { applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); } +#endif Mat weights(outChannels, inChannels * inSize.height * inSize.width, CV_32F); randu(weights, -1.0f, 1.0f); @@ -430,7 +434,21 @@ TEST_P(FullyConnected, Accuracy) int sz[] = {1, inChannels, inSize.height, inSize.width}; Mat input(4, &sz[0], CV_32F); - test(lp, input, backendId, targetId); + + double l1 = 0.0; + double lInf = 0.0; +#if defined(INF_ENGINE_RELEASE) + if (targetId == DNN_TARGET_MYRIAD) + { + l1 = 0.015; + lInf = 0.025; + } + else if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && targetId == DNN_TARGET_OPENCL_FP16) + { + l1 = 0.01; + } +#endif + test(lp, input, backendId, targetId, false, l1, lInf); } INSTANTIATE_TEST_CASE_P(Layer_Test_Halide, FullyConnected, Combine( @@ -812,18 +830,18 @@ TEST_P(Eltwise, Accuracy) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); #endif -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000) +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && numConv > 1) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); #endif -#if defined(INF_ENGINE_RELEASE) +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && targetId == DNN_TARGET_OPENCL && op == "sum" && numConv == 1 && !weighted) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); #endif -#if defined(INF_ENGINE_RELEASE) +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && numConv > 1) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); #endif diff --git a/modules/dnn/test/test_layers.cpp b/modules/dnn/test/test_layers.cpp index e61fd733f9..2d4f78c88c 100644 --- a/modules/dnn/test/test_layers.cpp +++ b/modules/dnn/test/test_layers.cpp @@ -194,13 +194,23 @@ TEST_P(Test_Caffe_layers, DeConvolution) TEST_P(Test_Caffe_layers, InnerProduct) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // IE exception: Ngraph operation Reshape with name Reshape_4219609 has dynamic output shape on 0 port, but CPU plug-in supports only static shape + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); +#endif if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + testLayerUsingCaffeModels("layer_inner_product", true); } @@ -295,10 +305,12 @@ TEST_P(Test_Caffe_layers, Concat) CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); #endif +#if INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif #endif testLayerUsingCaffeModels("layer_concat"); diff --git a/modules/dnn/test/test_misc.cpp b/modules/dnn/test/test_misc.cpp index 9971450478..0de503ac7d 100644 --- a/modules/dnn/test/test_misc.cpp +++ b/modules/dnn/test/test_misc.cpp @@ -578,7 +578,8 @@ TEST_P(Async, create_layer_pipeline_set_and_forward_all) if (backendId != DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && backendId != DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) throw SkipTestException("No support for async forward"); - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + // Exception: Default implementation fallbacks in asynchronous mode + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && dtype == CV_8U) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 0fd5d08258..a432f2251e 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -185,17 +185,11 @@ TEST_P(Test_ONNX_layers, Gather) TEST_P(Test_ONNX_layers, Convolution3D) { -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif testONNXModels("conv3d"); } TEST_P(Test_ONNX_layers, Convolution3D_bias) { -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif testONNXModels("conv3d_bias"); } @@ -222,14 +216,73 @@ TEST_P(Test_ONNX_layers, Deconvolution) TEST_P(Test_ONNX_layers, Deconvolution3D) { -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + // [ GENERAL_ERROR ] vpu/graph_transformer/src/frontend/frontend.cpp:439 Failed to compile layer "2": + // [ GENERAL_ERROR ] vpu/graph_transformer/src/model/model.cpp:198 duplicateData error: while duplicating 2@weights Const data got different desc and content byte sizes (162 and 486 respectively) + if (target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + } #endif - if (backend == DNN_BACKEND_OPENCV || target != DNN_TARGET_CPU) - throw SkipTestException("Only DLIE backend on CPU is supported"); + + if (backend == DNN_BACKEND_OPENCV) + throw SkipTestException("OpenCV backend is not supported"); // FIXIT use tags + testONNXModels("deconv3d"); +} + +TEST_P(Test_ONNX_layers, Deconvolution3D_bias) +{ +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + // [ GENERAL_ERROR ] vpu/graph_transformer/src/frontend/frontend.cpp:439 Failed to compile layer "2": + // [ GENERAL_ERROR ] vpu/graph_transformer/src/model/model.cpp:198 duplicateData error: while duplicating 2@weights Const data got different desc and content byte sizes (162 and 486 respectively) + if (target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + } +#endif + + if (backend == DNN_BACKEND_OPENCV) + throw SkipTestException("OpenCV backend is not supported"); // FIXIT use tags + testONNXModels("deconv3d_bias"); +} + +TEST_P(Test_ONNX_layers, Deconvolution3D_pad) +{ +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + // [ GENERAL_ERROR ] vpu/graph_transformer/src/frontend/frontend.cpp:439 Failed to compile layer "2": + // [ GENERAL_ERROR ] vpu/graph_transformer/src/model/model.cpp:198 duplicateData error: while duplicating 2@weights Const data got different desc and content byte sizes (162 and 486 respectively) + if (target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + } +#endif + + if (backend == DNN_BACKEND_OPENCV) + throw SkipTestException("OpenCV backend is not supported"); // FIXIT use tags + testONNXModels("deconv3d_pad"); +} + +TEST_P(Test_ONNX_layers, Deconvolution3D_adjpad) +{ +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + // [ GENERAL_ERROR ] vpu/graph_transformer/src/frontend/frontend.cpp:439 Failed to compile layer "2": + // [ GENERAL_ERROR ] vpu/graph_transformer/src/model/model.cpp:198 duplicateData error: while duplicating 2@weights Const data got different desc and content byte sizes (162 and 486 respectively) + if (target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + } +#endif + + if (backend == DNN_BACKEND_OPENCV) + throw SkipTestException("OpenCV backend is not supported"); // FIXIT use tags + testONNXModels("deconv3d_adjpad"); } @@ -295,12 +348,15 @@ TEST_P(Test_ONNX_layers, Scale) TEST_P(Test_ONNX_layers, ReduceMean3D) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target != DNN_TARGET_CPU) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); // Only CPU on DLIE backend is supported if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target != DNN_TARGET_CPU) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); // Only CPU on DLIE backend is supported - if (target != DNN_TARGET_CPU) - throw SkipTestException("Only CPU is supported"); +#endif + if (backend == DNN_BACKEND_OPENCV && target != DNN_TARGET_CPU) + throw SkipTestException("Only CPU is supported"); // FIXIT use tags + testONNXModels("reduce_mean3d"); } @@ -340,13 +396,12 @@ TEST_P(Test_ONNX_layers, Concatenation) TEST_P(Test_ONNX_layers, Eltwise3D) { -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target != DNN_TARGET_CPU) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); // Only CPU on DLIE backend is supported if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target != DNN_TARGET_CPU) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); // Only CPU on DLIE backend is supported +#endif testONNXModels("eltwise3d"); } @@ -357,43 +412,56 @@ TEST_P(Test_ONNX_layers, AveragePooling) TEST_P(Test_ONNX_layers, MaxPooling3D) { -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + // accuracy + if (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); + // IE exception: [ GENERAL_ERROR ] AssertionFailed: !expired() + if (target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + } #endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target != DNN_TARGET_CPU) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); // Only CPU on DLIE backend is supported if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target != DNN_TARGET_CPU) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); // Only CPU on DLIE backend is supported - if (target != DNN_TARGET_CPU) - throw SkipTestException("Only CPU is supported"); +#endif + if (backend == DNN_BACKEND_OPENCV && target != DNN_TARGET_CPU) + throw SkipTestException("Only CPU is supported"); // FIXIT use tags + testONNXModels("max_pool3d", npy, 0, 0, false, false); } TEST_P(Test_ONNX_layers, AvePooling3D) { -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target != DNN_TARGET_CPU) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); // Only CPU on DLIE backend is supported if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target != DNN_TARGET_CPU) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); // Only CPU on DLIE backend is supported - if (target != DNN_TARGET_CPU) - throw SkipTestException("Only CPU is supported"); +#endif + if (backend == DNN_BACKEND_OPENCV && target != DNN_TARGET_CPU) + throw SkipTestException("Only CPU is supported"); // FIXIT use tags + testONNXModels("ave_pool3d"); } TEST_P(Test_ONNX_layers, PoolConv3D) { -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target != DNN_TARGET_CPU) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); // Only CPU on DLIE backend is supported if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target != DNN_TARGET_CPU) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); // Only CPU on DLIE backend is supported - if (target != DNN_TARGET_CPU) - throw SkipTestException("Only CPU is supported"); +#endif + if (backend == DNN_BACKEND_OPENCV && target != DNN_TARGET_CPU) + throw SkipTestException("Only CPU is supported"); // FIXIT use tags + testONNXModels("pool_conv_3d"); } @@ -875,6 +943,7 @@ TEST_P(Test_ONNX_layers, DynamicAxes) TEST_P(Test_ONNX_layers, MaxPool1d) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) { if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); @@ -883,11 +952,20 @@ TEST_P(Test_ONNX_layers, MaxPool1d) { if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); } +#endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2021040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) + { + // 2021.4: [ GENERAL_ERROR ] AssertionFailed: !expired() + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + } +#endif testONNXModels("maxpooling_1d"); } TEST_P(Test_ONNX_layers, MaxPoolSigmoid1d) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) { if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); @@ -896,11 +974,13 @@ TEST_P(Test_ONNX_layers, MaxPoolSigmoid1d) { if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); } +#endif testONNXModels("maxpooling_sigmoid_1d"); } TEST_P(Test_ONNX_layers, MaxPool1d_Twise) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) { if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); @@ -909,11 +989,13 @@ TEST_P(Test_ONNX_layers, MaxPool1d_Twise) { if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); } +#endif testONNXModels("two_maxpooling_1d"); } TEST_P(Test_ONNX_layers, AvePool1d) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) { if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); @@ -922,11 +1004,13 @@ TEST_P(Test_ONNX_layers, AvePool1d) { if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); } +#endif testONNXModels("average_pooling_1d"); } TEST_P(Test_ONNX_layers, PoolConv1d) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) { if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); @@ -935,6 +1019,7 @@ TEST_P(Test_ONNX_layers, PoolConv1d) { if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); } +#endif testONNXModels("pool_conv_1d"); } @@ -1014,11 +1099,18 @@ TEST_P(Test_ONNX_nets, Squeezenet) TEST_P(Test_ONNX_nets, Googlenet) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // accuracy + if (target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif const String model = _tf("models/googlenet.onnx", false); @@ -1264,7 +1356,7 @@ TEST_P(Test_ONNX_nets, DenseNet121) TEST_P(Test_ONNX_nets, Inception_v1) { -#if defined(INF_ENGINE_RELEASE) +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); @@ -1274,26 +1366,35 @@ TEST_P(Test_ONNX_nets, Inception_v1) TEST_P(Test_ONNX_nets, Shufflenet) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) { if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); if (target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); } +#endif testONNXModels("shufflenet", pb); } TEST_P(Test_ONNX_nets, Resnet34_kinetics) { -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target != DNN_TARGET_CPU) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); // Only CPU on DLIE backend is supported if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target != DNN_TARGET_CPU) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); // Only CPU on DLIE backend is supported - if (target != DNN_TARGET_CPU) - throw SkipTestException("Only CPU is supported"); +#endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + // IE exception: Function contains several inputs and outputs with one friendly name! + if (target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + } +#endif + if (backend == DNN_BACKEND_OPENCV && target != DNN_TARGET_CPU) + throw SkipTestException("Only CPU is supported"); // FIXIT use tags String onnxmodel = findDataFile("dnn/resnet-34_kinetics.onnx", false); Mat image0 = imread(findDataFile("dnn/dog416.png")); @@ -1334,6 +1435,11 @@ TEST_P(Test_ONNX_nets, Resnet34_kinetics) // output range [-5, 11] float l1 = 0.0013; float lInf = 0.009; + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + { + l1 = 0.02; + lInf = 0.07; + } checkBackend(&input0, &ref0); net.setInput(input0); diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index d02b7136d6..073a2f3395 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -215,13 +215,12 @@ TEST_P(Test_TensorFlow_layers, conv_pool_nchw) TEST_P(Test_TensorFlow_layers, Convolution3D) { -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target != DNN_TARGET_CPU) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); // Only CPU on DLIE backend is supported if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target != DNN_TARGET_CPU) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); // Only CPU on DLIE backend is supported +#endif runTensorFlowNet("conv3d"); } @@ -230,7 +229,7 @@ TEST_P(Test_TensorFlow_layers, padding) runTensorFlowNet("padding_valid"); runTensorFlowNet("spatial_padding"); runTensorFlowNet("mirror_pad"); -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019020000) +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019020000) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (target == DNN_TARGET_MYRIAD) { if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) @@ -343,6 +342,7 @@ TEST_P(Test_TensorFlow_layers, concat_axis_1) TEST_P(Test_TensorFlow_layers, concat_3d) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_OPENCV && target != DNN_TARGET_CPU) { if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); @@ -352,6 +352,7 @@ TEST_P(Test_TensorFlow_layers, concat_3d) if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH || backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) && target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); +#endif runTensorFlowNet("concat_3d"); } @@ -429,13 +430,27 @@ TEST_P(Test_TensorFlow_layers, batch_norm3D) TEST_P(Test_TensorFlow_layers, slim_batch_norm) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif // Output values range: [-40.0597, 207.827] - double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.041 : default_l1; - double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.33 : default_lInf; + double l1 = default_l1; + double lInf = default_lInf; + if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) + { + l1 = 0.041; + lInf = 0.33; + } +#if defined(INF_ENGINE_RELEASE) + else if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_CPU) + { + lInf = 0.0002; + } +#endif + runTensorFlowNet("slim_batch_norm", false, l1, lInf); } @@ -562,7 +577,7 @@ TEST_P(Test_TensorFlow_layers, max_pool_grad) TEST_P(Test_TensorFlow_layers, ave_pool_same) { // Reference output values are in range [-0.519531, 0.112976] -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000) +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (target == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) { if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) @@ -576,29 +591,42 @@ TEST_P(Test_TensorFlow_layers, ave_pool_same) TEST_P(Test_TensorFlow_layers, MaxPooling3D) { -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + // accuracy + if (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); + // IE exception: [ GENERAL_ERROR ] AssertionFailed: !expired() + if (target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + } #endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target != DNN_TARGET_CPU) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); // Only CPU on DLIE backend is supported if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target != DNN_TARGET_CPU) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); // Only CPU on DLIE backend is supported - if (target != DNN_TARGET_CPU) - throw SkipTestException("Only CPU is supported"); +#endif + if (backend == DNN_BACKEND_OPENCV && target != DNN_TARGET_CPU) + throw SkipTestException("Only CPU is supported"); // FIXIT use tags + runTensorFlowNet("max_pool3d"); } TEST_P(Test_TensorFlow_layers, AvePooling3D) { -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target != DNN_TARGET_CPU) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); // Only CPU on DLIE backend is supported if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target != DNN_TARGET_CPU) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); // Only CPU on DLIE backend is supported - if (target != DNN_TARGET_CPU) - throw SkipTestException("Only CPU is supported"); +#endif + if (backend == DNN_BACKEND_OPENCV && target != DNN_TARGET_CPU) + throw SkipTestException("Only CPU is supported"); // FIXIT use tags + runTensorFlowNet("ave_pool3d"); } @@ -628,10 +656,12 @@ TEST_P(Test_TensorFlow_layers, matmul) TEST_P(Test_TensorFlow_layers, reshape) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif runTensorFlowNet("shift_reshape_no_reorder"); runTensorFlowNet("reshape_no_reorder"); runTensorFlowNet("reshape_reduce"); @@ -1188,18 +1218,35 @@ TEST_P(Test_TensorFlow_layers, quantized) TEST_P(Test_TensorFlow_layers, lstm) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + // Exception: Ngraph operation Reshape with name Reshape has dynamic output shape on 0 port, but CPU plug-in supports only static shape + if (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); + // Xlink + if (target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + } +#endif if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + runTensorFlowNet("lstm", true); runTensorFlowNet("lstm", true, 0.0, 0.0, true); } TEST_P(Test_TensorFlow_layers, split) { + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) @@ -1229,8 +1276,10 @@ TEST_P(Test_TensorFlow_layers, resize_nearest_neighbor_align_corners) TEST_P(Test_TensorFlow_layers, resize_nearest_neighbor_half_pixel) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif runTensorFlowNet("resize_nearest_neighbor", false, 0.0, 0.0, false, "_half_pixel"); } @@ -1369,10 +1418,26 @@ TEST_P(Test_TensorFlow_layers, clip_by_value) TEST_P(Test_TensorFlow_layers, tf2_prelu) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + // IE exception: Input prelu:StatefulPartitionedCall/StatefulPartitionedCall/sequential/p_re_lu/add hasn't been found in primitiveIDs map + if (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); + // IE exception: Eltwise node with name `StatefulPartitionedCall/StatefulPartitionedCall/sequential/p_re_lu/add` has invalid input/output dims configuration + if (target == DNN_TARGET_CPU) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_CPU, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + } +#endif + runTensorFlowNet("tf2_prelu"); } diff --git a/modules/dnn/test/test_torch_importer.cpp b/modules/dnn/test/test_torch_importer.cpp index 96e8ac85a8..fdd5a4b923 100644 --- a/modules/dnn/test/test_torch_importer.cpp +++ b/modules/dnn/test/test_torch_importer.cpp @@ -211,23 +211,32 @@ TEST_P(Test_Torch_layers, net_lp_pooling_square) } TEST_P(Test_Torch_layers, net_lp_pooling_power) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif runTorchNet("net_lp_pooling_power", "", false, true); } TEST_P(Test_Torch_layers, net_conv_gemm_lrn) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif double l1 = 0.0, lInf = 0.0; if (target == DNN_TARGET_OPENCL_FP16) { l1 = 0.046; lInf = 0.023; } + else if (target == DNN_TARGET_MYRIAD) + { + l1 = 0.02; + lInf = 0.05; + } // The OpenCL kernels use the native_ math functions which have // implementation defined accuracy, so we use relaxed thresholds. See // https://github.com/opencv/opencv/issues/9821 for more details. From 2da1f9181a0c3a5f3b67a29670080a515bf1cf7d Mon Sep 17 00:00:00 2001 From: Jong Sin Kim Date: Thu, 2 Dec 2021 22:48:11 +0900 Subject: [PATCH 122/226] Merge pull request #21170 from JJJoonngg:4.x Check buffer size when frameWidth * frameHeight bigger than allocated buffer size --- modules/videoio/src/cap_android_mediandk.cpp | 21 +++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/modules/videoio/src/cap_android_mediandk.cpp b/modules/videoio/src/cap_android_mediandk.cpp index 4fb4a82c2f..c7f2855502 100644 --- a/modules/videoio/src/cap_android_mediandk.cpp +++ b/modules/videoio/src/cap_android_mediandk.cpp @@ -97,10 +97,29 @@ public: LOGV("buffer size: %zu", bufferSize); LOGV("width (frame): %d", frameWidth); LOGV("height (frame): %d", frameHeight); - if (info.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) { + if (info.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) + { LOGV("output EOS"); sawOutputEOS = true; } + if ((size_t)frameWidth * frameHeight * 3 / 2 > bufferSize) + { + if (bufferSize == 3110400 && frameWidth == 1920 && frameHeight == 1088) + { + frameHeight = 1080; + LOGV("Buffer size is too small, force using height = %d", frameHeight); + } + else if(bufferSize == 3110400 && frameWidth == 1088 && frameHeight == 1920) + { + frameWidth = 1080; + LOGV("Buffer size is too small, force using width = %d", frameWidth); + } + else + { + LOGE("Buffer size is too small. Frame is ignored. Enable verbose logging to see actual values of parameters"); + return false; + } + } AMediaCodec_releaseOutputBuffer(mediaCodec.get(), bufferIndex, info.size != 0); return true; } else if (bufferIndex == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED) { From 37b1876807430148aba9208fceaeb3a1cb2c356a Mon Sep 17 00:00:00 2001 From: APrigarina Date: Thu, 2 Dec 2021 15:04:04 +0300 Subject: [PATCH 123/226] qr encoder: fix memory and unused variables issues --- modules/objdetect/src/qrcode_encoder.cpp | 47 ++++++++++-------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/modules/objdetect/src/qrcode_encoder.cpp b/modules/objdetect/src/qrcode_encoder.cpp index 8329c1c890..ae15c64f42 100644 --- a/modules/objdetect/src/qrcode_encoder.cpp +++ b/modules/objdetect/src/qrcode_encoder.cpp @@ -308,18 +308,7 @@ int QRCodeEncoderImpl::versionAuto(const std::string& input_str) for(size_t i = 0; i < possible_version.size(); i++) { int version_range_index = possible_version[i]; - if (version_range_index == 1) - { - tmp_version = 1; - } - else if (version_range_index == 2) - { - tmp_version = 10; - } - else - { - tmp_version = 27; - } + encodeAuto(input_str, payload_tmp); tmp_version = findVersionCapacity((int)payload_tmp.size(), ecc_level, version_range[version_range_index], version_range[version_range_index + 1]); @@ -351,10 +340,11 @@ void QRCodeEncoderImpl::generateQR(const std::string &input) int segment_begin = i * segment_len; int segemnt_end = min((i + 1) * segment_len, (int) input.length()) - 1; std::string input_info = input.substr(segment_begin, segemnt_end - segment_begin + 1); - int v = versionAuto(input_info); + int detected_version = versionAuto(input_info); + CV_Assert(detected_version != -1); if (version_level == 0) - version_level = v; - else if (version_level < v) + version_level = detected_version; + else if (version_level < detected_version) CV_Error(Error::StsBadArg, "The given version is not suitable for the given input string length "); payload.clear(); @@ -752,12 +742,14 @@ void QRCodeEncoderImpl::eccGenerate(vector > &data_blocks, vecto void QRCodeEncoderImpl::rearrangeBlocks(const vector > &data_blocks, const vector > &ecc_blocks) { rearranged_data.clear(); - rearranged_data.reserve(MAX_PAYLOAD_LEN); int blocks = cur_ecc_params->num_blocks_in_G2 + cur_ecc_params->num_blocks_in_G1; int col_border = max(cur_ecc_params->data_codewords_in_G2, cur_ecc_params->data_codewords_in_G1); int total_codeword_num = version_info->total_codewords; int is_not_equal = cur_ecc_params->data_codewords_in_G2 - cur_ecc_params->data_codewords_in_G1; - for (int i = 0; i < total_codeword_num; i++) + int add_steps = cur_ecc_params->data_codewords_in_G2 > cur_ecc_params->data_codewords_in_G1 ? + (cur_ecc_params->data_codewords_in_G2 - cur_ecc_params->data_codewords_in_G1) * cur_ecc_params->num_blocks_in_G1 : 0; + rearranged_data.reserve(total_codeword_num + add_steps); + for (int i = 0; i < total_codeword_num + add_steps; i++) { int cur_col = i / blocks; int cur_row = i % blocks; @@ -783,16 +775,6 @@ void QRCodeEncoderImpl::rearrangeBlocks(const vector > &data_blo } rearranged_data.push_back(tmp); } - const int remainder_len []= {0, - 0, 7, 7, 7, 7, 7, 0, 0, 0, 0, - 0, 0, 0, 3, 3, 3, 3, 3, 3, 3, - 4, 4, 4, 4, 4, 4, 4, 3, 3, 3, - 3, 3, 3, 3, 0, 0, 0, 0, 0, 0}; - int cur_remainder_len = remainder_len[version_level]; - if (cur_remainder_len != 0) - { - rearranged_data.push_back(0); - } } void QRCodeEncoderImpl::findAutoMaskType() @@ -1078,6 +1060,8 @@ void QRCodeEncoderImpl::writeData() int dir = -1; int count = 0; int codeword_value = rearranged_data[0]; + const int limit_bits = (int)rearranged_data.size() * 8; + bool limit_reached = false; while (x > 0) { if (x == 6) @@ -1093,11 +1077,20 @@ void QRCodeEncoderImpl::writeData() continue; } count++; + if (count == limit_bits) + { + limit_reached = true; + break; + } if (count % 8 == 0) { codeword_value = rearranged_data[count / 8]; } } + if (limit_reached) + { + break; + } y += dir; if (y < 0 || y >= version_size) { From 1613d30544742b646e015660df76325740fcb9c2 Mon Sep 17 00:00:00 2001 From: rogday Date: Thu, 2 Dec 2021 20:11:11 +0300 Subject: [PATCH 124/226] Merge pull request #21159 from rogday:ceil_mode fix ceil_mode for Average/MaxPooling * fix ceil_mode * add a comment --- modules/dnn/src/onnx/onnx_importer.cpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 9f7895c7dd..f776bdc5da 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -612,11 +612,24 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto) } } +void setCeilMode(LayerParams& layerParams) +{ + // auto_pad attribute is deprecated and uses ceil + if (layerParams.has("pad_mode")) + { + layerParams.set("ceil_mode", true); + } + else if (!layerParams.has("ceil_mode")) + { + layerParams.set("ceil_mode", false); + } +} + void ONNXImporter::parseMaxPool(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { layerParams.type = "Pooling"; layerParams.set("pool", "MAX"); - layerParams.set("ceil_mode", layerParams.has("pad_mode")); + setCeilMode(layerParams); addLayer(layerParams, node_proto); } @@ -624,7 +637,7 @@ void ONNXImporter::parseAveragePool(LayerParams& layerParams, const opencv_onnx: { layerParams.type = "Pooling"; layerParams.set("pool", "AVE"); - layerParams.set("ceil_mode", layerParams.has("pad_mode")); + setCeilMode(layerParams); layerParams.set("ave_pool_padded_area", framework_name == "pytorch"); addLayer(layerParams, node_proto); } From c5b8b5687f2ec7fe73d6e2265eba46804114dcd9 Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Fri, 3 Dec 2021 15:30:05 +0300 Subject: [PATCH 125/226] Merge pull request #21041 from sivanov-work:gin_gout_concept G-API: GAPI introduce compile guard for some types for gin/gout params passing * Initial for taged solution * Move out tags to gtags.hpp & add protection for own::Mat * Add compile guard to proper place * Fix MACRO concat * Add unit tests * Remove class MACRO injection due to Python3 * Revert back unproper changes * Apply comments: reuse shape from traits * Throw away unused gtags * Apply comments * Handle own::* * Fix test * Fix test(1) * Fix unix build * Try on type list * Apply comments * Apply comments * Fix warning --- .../include/opencv2/gapi/gtype_traits.hpp | 20 ++++++++++ .../include/opencv2/gapi/opencv_includes.hpp | 8 ++++ .../gapi/include/opencv2/gapi/own/types.hpp | 1 + .../gapi/include/opencv2/gapi/util/util.hpp | 7 ++++ modules/gapi/test/gapi_util_tests.cpp | 40 +++++++++++++++++-- .../test/internal/gapi_int_gmetaarg_test.cpp | 16 ++------ .../test/internal/gapi_int_proto_tests.cpp | 1 - 7 files changed, 76 insertions(+), 17 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/gtype_traits.hpp b/modules/gapi/include/opencv2/gapi/gtype_traits.hpp index 2e8dcb1aec..2b43421907 100644 --- a/modules/gapi/include/opencv2/gapi/gtype_traits.hpp +++ b/modules/gapi/include/opencv2/gapi/gtype_traits.hpp @@ -19,11 +19,25 @@ #include #include #include +#include +#include namespace cv { namespace detail { + template + struct contains_shape_field : std::false_type {}; + + template + struct contains_shape_field> : + std::is_same::type, GShape> + {}; + + template + struct has_gshape : contains_shape_field {}; + // FIXME: These traits and enum and possible numerous switch(kind) // block may be replaced with a special Handler object or with // a double dispatch @@ -181,10 +195,16 @@ namespace detail } template static auto wrap_in (const U &u) -> typename GTypeTraits::strip_type { + static_assert(!(cv::detail::has_gshape>::value + || cv::detail::contains::type, GAPI_OWN_TYPES_LIST>::value), + "gin/gout must not be used with G* classes or cv::gapi::own::*"); return GTypeTraits::wrap_in(u); } template static auto wrap_out(U &u) -> typename GTypeTraits::strip_type { + static_assert(!(cv::detail::has_gshape>::value + || cv::detail::contains::type, GAPI_OWN_TYPES_LIST>::value), + "gin/gout must not be used with G* classses or cv::gapi::own::*"); return GTypeTraits::wrap_out(u); } }; diff --git a/modules/gapi/include/opencv2/gapi/opencv_includes.hpp b/modules/gapi/include/opencv2/gapi/opencv_includes.hpp index 08b2d6ed02..25a67d6da6 100644 --- a/modules/gapi/include/opencv2/gapi/opencv_includes.hpp +++ b/modules/gapi/include/opencv2/gapi/opencv_includes.hpp @@ -14,6 +14,12 @@ # include # include # include +#define GAPI_OWN_TYPES_LIST cv::gapi::own::Rect, \ + cv::gapi::own::Size, \ + cv::gapi::own::Point, \ + cv::gapi::own::Point2f, \ + cv::gapi::own::Scalar, \ + cv::gapi::own::Mat #else // Without OpenCV # include # include // cv::gapi::own::Rect/Size/Point @@ -28,6 +34,8 @@ namespace cv { using Scalar = gapi::own::Scalar; using Mat = gapi::own::Mat; } // namespace cv +#define GAPI_OWN_TYPES_LIST cv::gapi::own::VoidType + #endif // !defined(GAPI_STANDALONE) #endif // OPENCV_GAPI_OPENCV_INCLUDES_HPP diff --git a/modules/gapi/include/opencv2/gapi/own/types.hpp b/modules/gapi/include/opencv2/gapi/own/types.hpp index 6bd6880982..38143660bc 100644 --- a/modules/gapi/include/opencv2/gapi/own/types.hpp +++ b/modules/gapi/include/opencv2/gapi/own/types.hpp @@ -143,6 +143,7 @@ inline std::ostream& operator<<(std::ostream& o, const Size& s) return o; } +struct VoidType {}; } // namespace own } // namespace gapi } // namespace cv diff --git a/modules/gapi/include/opencv2/gapi/util/util.hpp b/modules/gapi/include/opencv2/gapi/util/util.hpp index eb435a3eef..3be46d7ec2 100644 --- a/modules/gapi/include/opencv2/gapi/util/util.hpp +++ b/modules/gapi/include/opencv2/gapi/util/util.hpp @@ -116,6 +116,13 @@ namespace detail using type = std::tuple; static type get(std::tuple&& objs) { return std::forward>(objs); } }; + + template + struct make_void { typedef void type;}; + + template + using void_t = typename make_void::type; + } // namespace detail namespace util diff --git a/modules/gapi/test/gapi_util_tests.cpp b/modules/gapi/test/gapi_util_tests.cpp index e080f544f9..b95c12f639 100644 --- a/modules/gapi/test/gapi_util_tests.cpp +++ b/modules/gapi/test/gapi_util_tests.cpp @@ -4,13 +4,32 @@ // // Copyright (C) 2018 Intel Corporation - -#include "test_precomp.hpp" - #include - +#include "test_precomp.hpp" +#include #include +namespace cv +{ +struct Own {}; +namespace gapi +{ +namespace own +{ +struct ConvertibleToOwn{}; +struct NotConvertibleToOwn{}; +} // own +} // gapi +} // cv + +struct NoGhape {}; +struct HasGShape { + static constexpr cv::GShape shape = cv::GShape::GFRAME; +}; +struct MimicGShape { + static constexpr int shape = 0; +}; +#define DISALLOWED_LIST cv::gapi::own::NotConvertibleToOwn namespace opencv_test { @@ -40,4 +59,17 @@ TEST(GAPIUtil, AllButLast) "[int, float] are NOT all integral types"); } +TEST(GAPIUtil, GShaped) +{ + static_assert(!cv::detail::has_gshape::value, "NoGhape hasn't got any shape"); + static_assert(cv::detail::has_gshape::value, "HasGShape has got GShape shape"); + static_assert(!cv::detail::has_gshape::value, "MimicGShape hasn't got right shape"); +} + +TEST(GAPIUtil, GTypeList) +{ + static_assert(cv::detail::contains::value, "NotConvertibleToOwn is in denial list"); + static_assert(!cv::detail::contains::value, "ConvertibleToOwn is not in empty denial list"); + static_assert(!cv::detail::contains::value, "ConvertibleToOwn is not in denial list"); +} } // namespace opencv_test diff --git a/modules/gapi/test/internal/gapi_int_gmetaarg_test.cpp b/modules/gapi/test/internal/gapi_int_gmetaarg_test.cpp index 78048cfa3f..bcf014ac82 100644 --- a/modules/gapi/test/internal/gapi_int_gmetaarg_test.cpp +++ b/modules/gapi/test/internal/gapi_int_gmetaarg_test.cpp @@ -138,22 +138,18 @@ TEST(GMetaArg, Can_Describe_RunArg) cv::Mat m(3, 3, CV_8UC3); cv::UMat um(3, 3, CV_8UC3); cv::Scalar s; - constexpr int w = 3, h = 3, c = 3; - uchar data[w*h*c]; - cv::gapi::own::Mat om(h, w, CV_8UC3, data); cv::Scalar os; std::vector v; GMetaArgs metas = {GMetaArg(descr_of(m)), GMetaArg(descr_of(um)), GMetaArg(descr_of(s)), - GMetaArg(descr_of(om)), GMetaArg(descr_of(os)), GMetaArg(descr_of(v))}; - auto in_run_args = cv::gin(m, um, s, om, os, v); + auto in_run_args = cv::gin(m, um, s, os, v); - for (int i = 0; i < 3; i++) { + for (size_t i = 0; i < metas.size(); i++) { EXPECT_TRUE(can_describe(metas[i], in_run_args[i])); } } @@ -178,22 +174,18 @@ TEST(GMetaArg, Can_Describe_RunArgP) cv::Mat m(3, 3, CV_8UC3); cv::UMat um(3, 3, CV_8UC3); cv::Scalar s; - constexpr int w = 3, h = 3, c = 3; - uchar data[w*h*c]; - cv::gapi::own::Mat om(h, w, CV_8UC3, data); cv::Scalar os; std::vector v; GMetaArgs metas = {GMetaArg(descr_of(m)), GMetaArg(descr_of(um)), GMetaArg(descr_of(s)), - GMetaArg(descr_of(om)), GMetaArg(descr_of(os)), GMetaArg(descr_of(v))}; - auto out_run_args = cv::gout(m, um, s, om, os, v); + auto out_run_args = cv::gout(m, um, s, os, v); - for (int i = 0; i < 3; i++) { + for (size_t i = 0; i < metas.size(); i++) { EXPECT_TRUE(can_describe(metas[i], out_run_args[i])); } } diff --git a/modules/gapi/test/internal/gapi_int_proto_tests.cpp b/modules/gapi/test/internal/gapi_int_proto_tests.cpp index 3dd7221198..67b9fa4913 100644 --- a/modules/gapi/test/internal/gapi_int_proto_tests.cpp +++ b/modules/gapi/test/internal/gapi_int_proto_tests.cpp @@ -15,7 +15,6 @@ struct ProtoPtrTest : public ::testing::Test { using Type = T; }; using ProtoPtrTestTypes = ::testing::Types< cv::Mat , cv::UMat - , cv::gapi::own::Mat , cv::RMat , cv::Scalar , std::vector From c3910807c5a4ad0b32dac5881bc8054036a9225f Mon Sep 17 00:00:00 2001 From: Anna Khakimova Date: Fri, 3 Dec 2021 15:30:39 +0300 Subject: [PATCH 126/226] Merge pull request #21177 from anna-khakimova:ak/simd_mulc * GAPI Fluid: SIMD for MulC kernel. * Changes for MulDouble kernel. --- .../gapi/perf/common/gapi_core_perf_tests.hpp | 4 +- .../perf/common/gapi_core_perf_tests_inl.hpp | 38 ++- .../perf/cpu/gapi_core_perf_tests_cpu.cpp | 12 +- .../perf/cpu/gapi_core_perf_tests_fluid.cpp | 22 +- .../perf/gpu/gapi_core_perf_tests_gpu.cpp | 6 +- .../gapi/src/backends/fluid/gfluidcore.cpp | 120 +++++--- .../fluid/gfluidcore_func.dispatch.cpp | 27 ++ .../src/backends/fluid/gfluidcore_func.hpp | 23 ++ .../backends/fluid/gfluidcore_func.simd.hpp | 289 +++++++++++++++++- 9 files changed, 466 insertions(+), 75 deletions(-) diff --git a/modules/gapi/perf/common/gapi_core_perf_tests.hpp b/modules/gapi/perf/common/gapi_core_perf_tests.hpp index 97b12f86b1..4084ed3e88 100644 --- a/modules/gapi/perf/common/gapi_core_perf_tests.hpp +++ b/modules/gapi/perf/common/gapi_core_perf_tests.hpp @@ -33,8 +33,8 @@ namespace opencv_test class SubCPerfTest : public TestPerfParams> {}; class SubRCPerfTest : public TestPerfParams> {}; class MulPerfTest : public TestPerfParams> {}; - class MulDoublePerfTest : public TestPerfParams> {}; - class MulCPerfTest : public TestPerfParams> {}; + class MulDoublePerfTest : public TestPerfParams> {}; + class MulCPerfTest : public TestPerfParams> {}; class DivPerfTest : public TestPerfParams> {}; class DivCPerfTest : public TestPerfParams> {}; class DivRCPerfTest : public TestPerfParams> {}; diff --git a/modules/gapi/perf/common/gapi_core_perf_tests_inl.hpp b/modules/gapi/perf/common/gapi_core_perf_tests_inl.hpp index 6c286a5ce2..d4144cd71a 100644 --- a/modules/gapi/perf/common/gapi_core_perf_tests_inl.hpp +++ b/modules/gapi/perf/common/gapi_core_perf_tests_inl.hpp @@ -257,17 +257,21 @@ PERF_TEST_P_(MulPerfTest, TestPerformance) PERF_TEST_P_(MulDoublePerfTest, TestPerformance) { - Size sz = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - int dtype = get<2>(GetParam()); - cv::GCompileArgs compile_args = get<3>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + int dtype = -1; + double scale = 1.0; + cv::GCompileArgs compile_args; + + std::tie(cmpF, sz, type, dtype, compile_args) = GetParam(); auto& rng = cv::theRNG(); double d = rng.uniform(0.0, 10.0); initMatrixRandU(type, sz, dtype, false); // OpenCV code /////////////////////////////////////////////////////////// - cv::multiply(in_mat1, d, out_mat_ocv, 1, dtype); + cv::multiply(in_mat1, d, out_mat_ocv, scale, dtype); // G-API code //////////////////////////////////////////////////////////// cv::GMat in1, out; @@ -285,8 +289,9 @@ PERF_TEST_P_(MulDoublePerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - // FIXIT unrealiable check: EXPECT_EQ(0, cv::countNonZero(out_mat_gapi != out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -295,15 +300,19 @@ PERF_TEST_P_(MulDoublePerfTest, TestPerformance) PERF_TEST_P_(MulCPerfTest, TestPerformance) { - Size sz = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - int dtype = get<2>(GetParam()); - cv::GCompileArgs compile_args = get<3>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + int dtype = -1; + double scale = 1.0; + cv::GCompileArgs compile_args; + + std::tie(cmpF, sz, type, dtype, compile_args) = GetParam(); initMatsRandU(type, sz, dtype, false); // OpenCV code /////////////////////////////////////////////////////////// - cv::multiply(in_mat1, sc, out_mat_ocv, 1, dtype); + cv::multiply(in_mat1, sc, out_mat_ocv, scale, dtype); // G-API code //////////////////////////////////////////////////////////// cv::GMat in1, out; @@ -322,8 +331,9 @@ PERF_TEST_P_(MulCPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - // FIXIT unrealiable check: EXPECT_EQ(0, cv::countNonZero(out_mat_gapi != out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } diff --git a/modules/gapi/perf/cpu/gapi_core_perf_tests_cpu.cpp b/modules/gapi/perf/cpu/gapi_core_perf_tests_cpu.cpp index 31e9d25610..1255f5ca52 100644 --- a/modules/gapi/perf/cpu/gapi_core_perf_tests_cpu.cpp +++ b/modules/gapi/perf/cpu/gapi_core_perf_tests_cpu.cpp @@ -56,13 +56,15 @@ INSTANTIATE_TEST_CASE_P(MulPerfTestCPU, MulPerfTest, Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(MulDoublePerfTestCPU, MulDoublePerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(-1, CV_8U, CV_16U, CV_32F), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(-1, CV_8U, CV_16U, CV_32F), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(MulCPerfTestCPU, MulCPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), Values(-1, CV_8U, CV_16U, CV_32F), Values(cv::compile_args(CORE_CPU)))); diff --git a/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp b/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp index 6ebd92dc4a..058cff69ac 100644 --- a/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp +++ b/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp @@ -52,17 +52,19 @@ INSTANTIATE_TEST_CASE_P(SubPerfTestFluid, SubPerfTest, Values(2.0), Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(MulDoublePerfTestFluid, MulDoublePerfTest, -// Combine(Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), -// Values(-1, CV_8U, CV_16U, CV_32F), -// Values(cv::compile_args(CORE_FLUID)))); + INSTANTIATE_TEST_CASE_P(MulDoublePerfTestFluid, MulDoublePerfTest, + Combine(Values(Tolerance_FloatRel_IntAbs(1e-6, 1).to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16SC1, CV_32FC1), + Values(-1, CV_8U, CV_32F), + Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(MulCPerfTestFluid, MulCPerfTest, -// Combine(Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), -// Values(-1, CV_8U, CV_16U, CV_32F), -// Values(cv::compile_args(CORE_FLUID)))); + INSTANTIATE_TEST_CASE_P(MulCPerfTestFluid, MulCPerfTest, + Combine(Values(Tolerance_FloatRel_IntAbs(1e-6, 1).to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(-1, CV_8U, CV_16U, CV_16S, CV_32F), + Values(cv::compile_args(CORE_FLUID)))); INSTANTIATE_TEST_CASE_P(DivPerfTestFluid, DivPerfTest, Combine(Values(AbsExact().to_compare_f()), diff --git a/modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp b/modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp index b4207c266d..025ea5331d 100644 --- a/modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp +++ b/modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp @@ -54,13 +54,15 @@ INSTANTIATE_TEST_CASE_P(MulPerfTestGPU, MulPerfTest, Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(MulDoublePerfTestGPU, MulDoublePerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), Values( -1, CV_8U, CV_16U, CV_32F ), Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(MulCPerfTestGPU, MulCPerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), Values( -1, CV_8U, CV_16U, CV_32F ), Values(cv::compile_args(CORE_GPU)))); diff --git a/modules/gapi/src/backends/fluid/gfluidcore.cpp b/modules/gapi/src/backends/fluid/gfluidcore.cpp index a737ad627b..a0513a09cd 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore.cpp +++ b/modules/gapi/src/backends/fluid/gfluidcore.cpp @@ -1265,12 +1265,12 @@ CV_ALWAYS_INLINE void run_arithm_s(Buffer &dst, const View &src, const float sca { case ARITHM_ADD: { - int w = 0; + int w = 0; #if CV_SIMD - w = addc_simd(in, scalar, out, length, chan); + w = addc_simd(in, scalar, out, length, chan); #endif - for (; w < length; ++w) - out[w] = add(in[w], scalar[w % chan]); + for (; w < length; ++w) + out[w] = add(in[w], scalar[w % chan]); break; } @@ -1284,12 +1284,17 @@ CV_ALWAYS_INLINE void run_arithm_s(Buffer &dst, const View &src, const float sca out[w] = sub(in[w], scalar[w % chan]); break; } - // TODO: optimize miltiplication and division case ARITHM_MULTIPLY: - for (int w=0; w < width; w++) - for (int c=0; c < chan; c++) - out[chan*w + c] = mul(in[chan*w + c], scalar[c], scale); + { + int w = 0; +#if CV_SIMD + w = mulc_simd(in, scalar, out, length, chan, scale); +#endif + for (; w < width; ++w) + for (int c = 0; c < chan; ++c) + out[chan * w + c] = mul(in[chan * w + c], scalar[c], scale); break; + } case ARITHM_DIVIDE: for (int w=0; w < width; w++) for (int c=0; c < chan; c++) @@ -1539,18 +1544,73 @@ GAPI_FLUID_KERNEL(GFluidSubRC, cv::gapi::core::GSubRC, false) } }; -GAPI_FLUID_KERNEL(GFluidMulC, cv::gapi::core::GMulC, false) +GAPI_FLUID_KERNEL(GFluidMulC, cv::gapi::core::GMulC, true) { static const int Window = 1; - static void run(const View &src, const cv::Scalar &_scalar, int /*dtype*/, Buffer &dst) + static void run(const View& src, const cv::Scalar& _scalar, int /*dtype*/, + Buffer& dst, Buffer& scratch) { - const float scalar[4] = { - static_cast(_scalar[0]), - static_cast(_scalar[1]), - static_cast(_scalar[2]), - static_cast(_scalar[3]) - }; + GAPI_Assert(src.meta().chan <= 4); + + if (dst.y() == 0) + { + const int chan = src.meta().chan; + float* sc = scratch.OutLine(); + + for (int i = 0; i < scratch.length(); ++i) + sc[i] = static_cast(_scalar[i % chan]); + } + const float* scalar = scratch.OutLine(); + const float scale = 1.0; + + // DST SRC OP __VA_ARGS__ + UNARY_(uchar, uchar, run_arithm_s, dst, src, scalar, ARITHM_MULTIPLY, scale); + UNARY_(uchar, ushort, run_arithm_s, dst, src, scalar, ARITHM_MULTIPLY, scale); + UNARY_(uchar, short, run_arithm_s, dst, src, scalar, ARITHM_MULTIPLY, scale); + UNARY_(uchar, float, run_arithm_s, dst, src, scalar, ARITHM_MULTIPLY, scale); + UNARY_(ushort, ushort, run_arithm_s, dst, src, scalar, ARITHM_MULTIPLY, scale); + UNARY_(ushort, short, run_arithm_s, dst, src, scalar, ARITHM_MULTIPLY, scale); + UNARY_(ushort, uchar, run_arithm_s, dst, src, scalar, ARITHM_MULTIPLY, scale); + UNARY_(ushort, float, run_arithm_s, dst, src, scalar, ARITHM_MULTIPLY, scale); + UNARY_(short, short, run_arithm_s, dst, src, scalar, ARITHM_MULTIPLY, scale); + UNARY_(short, ushort, run_arithm_s, dst, src, scalar, ARITHM_MULTIPLY, scale); + UNARY_(short, uchar, run_arithm_s, dst, src, scalar, ARITHM_MULTIPLY, scale); + UNARY_(short, float, run_arithm_s, dst, src, scalar, ARITHM_MULTIPLY, scale); + UNARY_(float, uchar, run_arithm_s, dst, src, scalar, ARITHM_MULTIPLY, scale); + UNARY_(float, ushort, run_arithm_s, dst, src, scalar, ARITHM_MULTIPLY, scale); + UNARY_(float, short, run_arithm_s, dst, src, scalar, ARITHM_MULTIPLY, scale); + UNARY_(float, float, run_arithm_s, dst, src, scalar, ARITHM_MULTIPLY, scale); + + CV_Error(cv::Error::StsBadArg, "unsupported combination of types"); + } + + static void initScratch(const GMatDesc&, const GScalarDesc&, int, Buffer& scratch) + { + initScratchBuffer(scratch); + } + + static void resetScratch(Buffer& /*scratch*/) + { + } +}; + +GAPI_FLUID_KERNEL(GFluidMulCOld, cv::gapi::core::GMulCOld, true) +{ + static const int Window = 1; + + static void run(const View &src, double _scalar, int /*dtype*/, Buffer &dst, Buffer& scratch) + { + GAPI_Assert(src.meta().chan <= 4); + + if (dst.y() == 0) + { + float* sc = scratch.OutLine(); + + for (int i = 0; i < scratch.length(); ++i) + sc[i] = static_cast(_scalar); + } + const float* scalar = scratch.OutLine(); const float scale = 1.f; // DST SRC OP __VA_ARGS__ @@ -1564,32 +1624,14 @@ GAPI_FLUID_KERNEL(GFluidMulC, cv::gapi::core::GMulC, false) CV_Error(cv::Error::StsBadArg, "unsupported combination of types"); } -}; -GAPI_FLUID_KERNEL(GFluidMulCOld, cv::gapi::core::GMulCOld, false) -{ - static const int Window = 1; - - static void run(const View &src, double _scalar, int /*dtype*/, Buffer &dst) + static void initScratch(const GMatDesc&, double, int, Buffer& scratch) { - const float scalar[4] = { - static_cast(_scalar), - static_cast(_scalar), - static_cast(_scalar), - static_cast(_scalar) - }; - const float scale = 1.f; + initScratchBuffer(scratch); + } - // DST SRC OP __VA_ARGS__ - UNARY_(uchar , uchar , run_arithm_s, dst, src, scalar, ARITHM_MULTIPLY, scale); - UNARY_(uchar , short, run_arithm_s, dst, src, scalar, ARITHM_MULTIPLY, scale); - UNARY_(uchar , float, run_arithm_s, dst, src, scalar, ARITHM_MULTIPLY, scale); - UNARY_( short, short, run_arithm_s, dst, src, scalar, ARITHM_MULTIPLY, scale); - UNARY_( float, uchar , run_arithm_s, dst, src, scalar, ARITHM_MULTIPLY, scale); - UNARY_( float, short, run_arithm_s, dst, src, scalar, ARITHM_MULTIPLY, scale); - UNARY_( float, float, run_arithm_s, dst, src, scalar, ARITHM_MULTIPLY, scale); - - CV_Error(cv::Error::StsBadArg, "unsupported combination of types"); + static void resetScratch(Buffer& /*scratch*/) + { } }; diff --git a/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp b/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp index 668ac3a4bb..f596779286 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp +++ b/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp @@ -138,6 +138,33 @@ SUBC_SIMD(float, float) #undef SUBC_SIMD +#define MULC_SIMD(SRC, DST) \ +int mulc_simd(const SRC in[], const float scalar[], DST out[], \ + const int length, const int chan, const float scale) \ +{ \ + CV_CPU_DISPATCH(mulc_simd, (in, scalar, out, length, chan, scale), \ + CV_CPU_DISPATCH_MODES_ALL); \ +} + +MULC_SIMD(uchar, uchar) +MULC_SIMD(ushort, uchar) +MULC_SIMD(short, uchar) +MULC_SIMD(float, uchar) +MULC_SIMD(short, short) +MULC_SIMD(ushort, short) +MULC_SIMD(uchar, short) +MULC_SIMD(float, short) +MULC_SIMD(ushort, ushort) +MULC_SIMD(uchar, ushort) +MULC_SIMD(short, ushort) +MULC_SIMD(float, ushort) +MULC_SIMD(uchar, float) +MULC_SIMD(ushort, float) +MULC_SIMD(short, float) +MULC_SIMD(float, float) + +#undef MULC_SIMD + } // namespace fluid } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/backends/fluid/gfluidcore_func.hpp b/modules/gapi/src/backends/fluid/gfluidcore_func.hpp index e6c0d4fe9b..541870e548 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore_func.hpp +++ b/modules/gapi/src/backends/fluid/gfluidcore_func.hpp @@ -106,6 +106,29 @@ SUBC_SIMD(float, float) #undef SUBC_SIMD +#define MULC_SIMD(SRC, DST) \ +int mulc_simd(const SRC in[], const float scalar[], DST out[], \ + const int length, const int chan, const float scale); + +MULC_SIMD(uchar, uchar) +MULC_SIMD(ushort, uchar) +MULC_SIMD(short, uchar) +MULC_SIMD(float, uchar) +MULC_SIMD(short, short) +MULC_SIMD(ushort, short) +MULC_SIMD(uchar, short) +MULC_SIMD(float, short) +MULC_SIMD(ushort, ushort) +MULC_SIMD(uchar, ushort) +MULC_SIMD(short, ushort) +MULC_SIMD(float, ushort) +MULC_SIMD(uchar, float) +MULC_SIMD(ushort, float) +MULC_SIMD(short, float) +MULC_SIMD(float, float) + +#undef MULC_SIMD + } // namespace fluid } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp b/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp index aed5359e7b..45974131c3 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp +++ b/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp @@ -127,6 +127,30 @@ SUBC_SIMD(float, float) #undef SUBC_SIMD + +#define MULC_SIMD(SRC, DST) \ +int mulc_simd(const SRC in[], const float scalar[], DST out[], \ + const int length, const int chan, const float scale); + +MULC_SIMD(uchar, uchar) +MULC_SIMD(ushort, uchar) +MULC_SIMD(short, uchar) +MULC_SIMD(float, uchar) +MULC_SIMD(short, short) +MULC_SIMD(ushort, short) +MULC_SIMD(uchar, short) +MULC_SIMD(float, short) +MULC_SIMD(ushort, ushort) +MULC_SIMD(uchar, ushort) +MULC_SIMD(short, ushort) +MULC_SIMD(float, ushort) +MULC_SIMD(uchar, float) +MULC_SIMD(ushort, float) +MULC_SIMD(short, float) +MULC_SIMD(float, float) + +#undef MULC_SIMD + #ifndef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY struct scale_tag {}; @@ -870,12 +894,13 @@ MUL_SIMD(float, float) //------------------------- // -// Fluid kernels: AddC +// Fluid kernels: AddC, SubC // //------------------------- struct add_tag {}; struct sub_tag {}; +struct mul_tag {}; CV_ALWAYS_INLINE void arithmOpScalar_pack_store_c3(short* outx, const v_int32& c1, const v_int32& c2, const v_int32& c3, @@ -909,6 +934,12 @@ CV_ALWAYS_INLINE v_float32 oper(sub_tag, const v_float32& a, const v_float32& sc return a - sc; } +CV_ALWAYS_INLINE v_float32 oper(mul_tag, const v_float32& a, const v_float32& sc) +{ + return a * sc; +} +//------------------------------------------------------------------------------------------------- + template CV_ALWAYS_INLINE typename std::enable_if<(std::is_same::value || @@ -957,7 +988,7 @@ CV_ALWAYS_INLINE typename std::enable_if::value || std::is_same::value, void>::type arithmOpScalar_simd_c3_impl(oper_tag t, const SRC* inx, DST* outx, const v_float32& s1, const v_float32& s2, - const v_float32& s3, const int nlanes) + const v_float32& s3, const int nlanes) { v_float32 a1 = vg_load_f32(inx); v_float32 a2 = vg_load_f32(&inx[nlanes / 2]); @@ -1089,7 +1120,7 @@ CV_ALWAYS_INLINE int arithmOpScalar_simd_common(oper_tag t, const SRC in[], return x; } - +//------------------------------------------------------------------------------------------------- #define ADDC_SIMD(SRC, DST) \ int addc_simd(const SRC in[], const float scalar[], DST out[], \ @@ -1129,6 +1160,8 @@ ADDC_SIMD(float, float) #undef ADDC_SIMD +//------------------------------------------------------------------------------------------------- + #define SUBC_SIMD(SRC, DST) \ int subc_simd(const SRC in[], const float scalar[], DST out[], \ const int length, const int chan) \ @@ -1167,6 +1200,256 @@ SUBC_SIMD(float, float) #undef SUBC_SIMD +//------------------------- +// +// Fluid kernels: MulC +// +//------------------------- + +template +CV_ALWAYS_INLINE +typename std::enable_if::value || + std::is_same::value, void>::type +mulc_scale_simd_c3_impl(const SRC* inx, DST* outx, const v_float32& s1, const v_float32& s2, + const v_float32& s3, const v_float32& scale, const int nlanes) +{ + v_float32 a1 = vg_load_f32(inx); + v_float32 a2 = vg_load_f32(&inx[nlanes / 2]); + v_float32 a3 = vg_load_f32(&inx[nlanes]); + v_float32 a4 = vg_load_f32(&inx[3 * nlanes / 2]); + v_float32 a5 = vg_load_f32(&inx[2 * nlanes]); + v_float32 a6 = vg_load_f32(&inx[5 * nlanes / 2]); + + arithmOpScalar_pack_store_c3(outx, v_round(scale*a1*s1), + v_round(scale*a2*s2), + v_round(scale*a3*s3), + v_round(scale*a4*s1), + v_round(scale*a5*s2), + v_round(scale*a6*s3)); +} + +//------------------------------------------------------------------------------------------------- + +template +CV_ALWAYS_INLINE void mulc_scale_simd_c3_impl(const SRC* inx, uchar* outx, + const v_float32& s1, const v_float32& s2, + const v_float32& s3, const v_float32& scale, const int nlanes) +{ + vx_store(outx, + v_pack_u(v_pack(v_round(scale * vg_load_f32(inx)* s1), + v_round(scale * vg_load_f32(&inx[nlanes/4])* s2)), + v_pack(v_round(scale * vg_load_f32(&inx[nlanes/2])* s3), + v_round(scale * vg_load_f32(&inx[3*nlanes/4])* s1)))); + + vx_store(&outx[nlanes], + v_pack_u(v_pack(v_round(scale * vg_load_f32(&inx[nlanes])* s2), + v_round(scale * vg_load_f32(&inx[5*nlanes/4])* s3)), + v_pack(v_round(scale * vg_load_f32(&inx[3*nlanes/2])* s1), + v_round(scale * vg_load_f32(&inx[7*nlanes/4])* s2)))); + + vx_store(&outx[2 * nlanes], + v_pack_u(v_pack(v_round(scale * vg_load_f32(&inx[2*nlanes])* s3), + v_round(scale * vg_load_f32(&inx[9*nlanes/4])* s1)), + v_pack(v_round(scale * vg_load_f32(&inx[5*nlanes/2])* s2), + v_round(scale * vg_load_f32(&inx[11*nlanes/4])* s3)))); +} + +//------------------------------------------------------------------------------------------------- + +template +CV_ALWAYS_INLINE void mulc_scale_simd_c3_impl(const SRC* in, float* out, + const v_float32& s1, const v_float32& s2, + const v_float32& s3, const v_float32& scale, const int nlanes) +{ + v_float32 a1 = vg_load_f32(in); + v_float32 a2 = vg_load_f32(&in[nlanes]); + v_float32 a3 = vg_load_f32(&in[2*nlanes]); + + vx_store(out, scale * a1* s1); + vx_store(&out[nlanes], scale * a2* s2); + vx_store(&out[2*nlanes], scale * a3* s3); +} + +//------------------------------------------------------------------------------------------------- + +template +CV_ALWAYS_INLINE int mulc_scale_simd_c3(const SRC in[], + const float scalar[], DST out[], + const int length, const float _scale) +{ + constexpr int chan = 3; + constexpr int nlanes = vector_type_of_t::nlanes; + constexpr int lanes = chan * nlanes; + + if (length < lanes) + return 0; + + v_float32 scale = vx_setall_f32(_scale); + + v_float32 s1 = vx_load(scalar); +#if CV_SIMD_WIDTH == 32 + v_float32 s2 = vx_load(&scalar[2]); + v_float32 s3 = vx_load(&scalar[1]); +#else + v_float32 s2 = vx_load(&scalar[1]); + v_float32 s3 = vx_load(&scalar[2]); +#endif + + int x = 0; + for (;;) + { + for (; x <= length - lanes; x += lanes) + { + mulc_scale_simd_c3_impl(&in[x], &out[x], s1, s2, s3, scale, nlanes); + } + + if (x < length) + { + x = length - lanes; + continue; // process unaligned tail + } + break; + } + return x; +} + +//------------------------------------------------------------------------------------------------- + +template +CV_ALWAYS_INLINE +typename std::enable_if<(std::is_same::value || + std::is_same::value), void>::type +mulc_scale_simd_common_impl(const SRC* inx, DST* outx, + const v_float32& sc, const v_float32& scale, + const int nlanes) +{ + v_float32 a1 = vg_load_f32(inx); + v_float32 a2 = vg_load_f32(&inx[nlanes/2]); + + v_store_i16(outx, v_round(scale * a1* sc), v_round(scale * a2* sc)); +} + +//------------------------------------------------------------------------------------------------- + +template +CV_ALWAYS_INLINE void mulc_scale_simd_common_impl(const SRC* inx, + uchar* outx, const v_float32& sc, + const v_float32& scale, const int nlanes) +{ + v_float32 a1 = vg_load_f32(inx); + v_float32 a2 = vg_load_f32(&inx[nlanes/4]); + v_float32 a3 = vg_load_f32(&inx[nlanes/2]); + v_float32 a4 = vg_load_f32(&inx[3 * nlanes/4]); + + vx_store(outx, v_pack_u(v_pack(v_round(scale * a1* sc), + v_round(scale * a2* sc)), + v_pack(v_round(scale * a3* sc), + v_round(scale * a4* sc)))); +} + +//------------------------------------------------------------------------------------------------- + +template +CV_ALWAYS_INLINE void mulc_scale_simd_common_impl(const SRC* inx, + float* outx, const v_float32& sc, + const v_float32& scale, const int) +{ + v_float32 a1 = vg_load_f32(inx); + vx_store(outx, scale * a1* sc); +} + +//------------------------------------------------------------------------------------------------- + +template +CV_ALWAYS_INLINE int mulc_scale_simd_common(const SRC in[], + const float scalar[], DST out[], + const int length, const float _scale) +{ + constexpr int nlanes = vector_type_of_t::nlanes; + + if (length < nlanes) + return 0; + + v_float32 _scalar = vx_load(scalar); + v_float32 scale = vx_setall_f32(_scale); + + int x = 0; + for (;;) + { + for (; x <= length - nlanes; x += nlanes) + { + mulc_scale_simd_common_impl(&in[x], &out[x], _scalar, scale, nlanes); + } + + if (x < length) + { + x = length - nlanes; + continue; // process unaligned tail + } + break; + } + return x; +} + +#define MULC_SIMD(SRC, DST) \ +int mulc_simd(const SRC in[], const float scalar[], DST out[], \ + const int length, const int chan, const float scale) \ +{ \ + mul_tag op_t; \ + switch (chan) \ + { \ + case 1: \ + case 2: \ + case 4: \ + { \ + if (std::fabs(scale - 1.0f) <= FLT_EPSILON) \ + { \ + return arithmOpScalar_simd_common(op_t, in, scalar, \ + out, length); \ + } \ + else \ + { \ + return mulc_scale_simd_common(in, scalar, out, length, scale); \ + } \ + } \ + case 3: \ + { \ + if (std::fabs(scale - 1.0f) <= FLT_EPSILON) \ + { \ + return arithmOpScalar_simd_c3(op_t, in, scalar, \ + out, length); \ + } \ + else \ + { \ + return mulc_scale_simd_c3(in, scalar, out, length, scale); \ + } \ + } \ + default: \ + GAPI_Assert(chan <= 4); \ + break; \ + } \ + return 0; \ +} + +MULC_SIMD(uchar, uchar) +MULC_SIMD(ushort, uchar) +MULC_SIMD(short, uchar) +MULC_SIMD(float, uchar) +MULC_SIMD(short, short) +MULC_SIMD(ushort, short) +MULC_SIMD(uchar, short) +MULC_SIMD(float, short) +MULC_SIMD(ushort, ushort) +MULC_SIMD(uchar, ushort) +MULC_SIMD(short, ushort) +MULC_SIMD(float, ushort) +MULC_SIMD(uchar, float) +MULC_SIMD(ushort, float) +MULC_SIMD(short, float) +MULC_SIMD(float, float) + +#undef MULC_SIMD + #endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY CV_CPU_OPTIMIZATION_NAMESPACE_END From 0835611d3af16739c2c99670a21aef271842fc7f Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 30 Nov 2021 12:08:35 +0000 Subject: [PATCH 127/226] dnn(test): re-enable tests which works with OpenVINO 2021.4.x --- modules/dnn/test/test_onnx_importer.cpp | 138 +++++++++++++++++++++++- 1 file changed, 136 insertions(+), 2 deletions(-) diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 4859432f57..f2deaf1a4e 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -402,77 +402,211 @@ TEST_P(Test_ONNX_layers, Exp) TEST_P(Test_ONNX_layers, Elementwise_Ceil) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif testONNXModels("ceil"); } TEST_P(Test_ONNX_layers, Elementwise_Floor) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif testONNXModels("floor"); } TEST_P(Test_ONNX_layers, Elementwise_Log) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif testONNXModels("log"); } TEST_P(Test_ONNX_layers, Elementwise_Round) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif testONNXModels("round"); } TEST_P(Test_ONNX_layers, Elementwise_Sqrt) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); testONNXModels("sqrt"); +#endif } TEST_P(Test_ONNX_layers, Elementwise_not) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif testONNXModels("not"); } -TEST_P(Test_ONNX_layers, Compare) +TEST_P(Test_ONNX_layers, Compare_EQ) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + // IE exception: Function contains several inputs and outputs with one friendly name! + if (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); + // IE exception: Function contains several inputs and outputs with one friendly name! + if (target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + } +#endif testONNXModels("equal"); +} + +TEST_P(Test_ONNX_layers, Compare_GT) +{ +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + // IE exception: Function contains several inputs and outputs with one friendly name! + if (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); + // IE exception: Function contains several inputs and outputs with one friendly name! + if (target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + } +#endif testONNXModels("greater"); +} + +TEST_P(Test_ONNX_layers, Compare_LT) +{ +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + // IE exception: Function contains several inputs and outputs with one friendly name! + if (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); + // IE exception: Function contains several inputs and outputs with one friendly name! + if (target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + } +#endif testONNXModels("less"); } -TEST_P(Test_ONNX_layers, CompareSameDims) +TEST_P(Test_ONNX_layers, CompareSameDims_EQ) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + // IE exception: Function contains several inputs and outputs with one friendly name! + if (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); + // IE exception: Function contains several inputs and outputs with one friendly name! + if (target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + } +#endif testONNXModels("equal_same_dims", npy, 0, 0, false, true, 2); +} + +TEST_P(Test_ONNX_layers, CompareSameDims_GT) +{ +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + // IE exception: Function contains several inputs and outputs with one friendly name! + if (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); + // IE exception: Function contains several inputs and outputs with one friendly name! + if (target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + } +#endif testONNXModels("greater_same_dims", npy, 0, 0, false, true, 2); +} + +TEST_P(Test_ONNX_layers, CompareSameDims_LT) +{ +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + // IE exception: Function contains several inputs and outputs with one friendly name! + if (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); + // IE exception: Function contains several inputs and outputs with one friendly name! + if (target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + } +#endif testONNXModels("less_same_dims", npy, 0, 0, false, true, 2); } From 4935b145394d76dbd358ce85f7bde86a84b4aab0 Mon Sep 17 00:00:00 2001 From: HAN Liutong Date: Fri, 3 Dec 2021 23:13:24 +0800 Subject: [PATCH 128/226] Merge pull request #21012 from hanliutong:rvv_clang Update RVV backend for using Clang. * Update cmake file of clang. * Modify the RVV optimization on DNN to adapt to clang. * Modify intrin_rvv: Disable some existing types. * Modify intrin_rvv: Reinterpret instead of load&cast. * Modify intrin_rvv: Update load&store without cast. * Modify intrin_rvv: Rename vfredsum to fredosum. * Modify intrin_rvv: Rewrite Check all/any by using vpopc. * Modify intrin_rvv: Use reinterpret instead of c-style casting. * Remove all macros which is not used in v_reinterpret * Rename vpopc to vcpop according to spec. --- .../include/opencv2/core/hal/intrin_rvv.hpp | 220 +++++++++++++----- modules/dnn/src/layers/layers_common.simd.hpp | 128 +++++----- platforms/linux/riscv64-clang.toolchain.cmake | 7 +- 3 files changed, 237 insertions(+), 118 deletions(-) diff --git a/modules/core/include/opencv2/core/hal/intrin_rvv.hpp b/modules/core/include/opencv2/core/hal/intrin_rvv.hpp index 55e0d88f59..fe6c077639 100644 --- a/modules/core/include/opencv2/core/hal/intrin_rvv.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_rvv.hpp @@ -19,6 +19,8 @@ CV_CPU_OPTIMIZATION_HAL_NAMESPACE_BEGIN #define CV_SIMD128_64F 1 //////////// Unsupported native intrinsics in C++ //////////// +// The following types have been defined in clang, but not in GCC yet. +#ifndef __clang__ struct vuint8mf2_t { @@ -224,6 +226,7 @@ inline vint16mf2_t vwcvt_x_x_v_i16mf2 (vint8mf4_t src, size_t vl) } return vle16_v_i16mf2(tmp, vl); } +#endif //////////// Types //////////// @@ -584,63 +587,84 @@ OPENCV_HAL_IMPL_RVV_SELF_REINTERPRET(int64x2, s64) OPENCV_HAL_IMPL_RVV_SELF_REINTERPRET(float64x2, f64) #endif -#define OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(_Tpvec1, _Tpvec2, _nTpvec1, _nTpvec2, suffix1, suffix2, nsuffix1, nsuffix2, width1, width2, vl1, vl2) \ +#define OPENCV_HAL_IMPL_RVV_NATIVE_REINTERPRET(_Tpvec1, _Tpvec2, suffix1, suffix2, nsuffix1, nsuffix2) \ inline v_##_Tpvec1 v_reinterpret_as_##suffix1(const v_##_Tpvec2& v) \ { \ - return v_##_Tpvec1((_nTpvec1)vle##width2##_v_##nsuffix2##m1(v.val, vl2)); \ + return v_##_Tpvec1(vreinterpret_v_##nsuffix2##m1_##nsuffix1##m1(v));\ } \ inline v_##_Tpvec2 v_reinterpret_as_##suffix2(const v_##_Tpvec1& v) \ { \ - return v_##_Tpvec2((_nTpvec2)vle##width1##_v_##nsuffix1##m1(v.val, vl1)); \ + return v_##_Tpvec2(vreinterpret_v_##nsuffix1##m1_##nsuffix2##m1(v));\ } -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int8x16, vuint8m1_t, vint8m1_t, u8, s8, u8, i8, 8, 8, 16, 16) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int16x8, vuint16m1_t, vint16m1_t, u16, s16, u16, i16, 16, 16, 8, 8) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int32x4, vuint32m1_t, vint32m1_t, u32, s32, u32, i32, 32, 32, 4, 4) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, float32x4, vuint32m1_t, vfloat32m1_t, u32, f32, u32, f32, 32, 32, 4, 4) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int32x4, float32x4, vint32m1_t, vfloat32m1_t, s32, f32, i32, f32, 32, 32, 4, 4) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int64x2, vuint64m1_t, vint64m1_t, u64, s64, u64, i64, 64, 64, 2, 2) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, uint16x8, vuint8m1_t, vuint16m1_t, u8, u16, u8, u16, 8, 16, 16, 8) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, uint32x4, vuint8m1_t, vuint32m1_t, u8, u32, u8, u32, 8, 32, 16, 4) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, uint64x2, vuint8m1_t, vuint64m1_t, u8, u64, u8, u64, 8, 64, 16, 2) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, uint32x4, vuint16m1_t, vuint32m1_t, u16, u32, u16, u32, 16, 32, 8, 4) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, uint64x2, vuint16m1_t, vuint64m1_t, u16, u64, u16, u64, 16, 64, 8, 2) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, uint64x2, vuint32m1_t, vuint64m1_t, u32, u64, u32, u64, 32, 64, 4, 2) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, int16x8, vint8m1_t, vint16m1_t, s8, s16, i8, i16, 8, 16, 16, 8) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, int32x4, vint8m1_t, vint32m1_t, s8, s32, i8, i32, 8, 32, 16, 4) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, int64x2, vint8m1_t, vint64m1_t, s8, s64, i8, i64, 8, 64, 16, 2) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, int32x4, vint16m1_t, vint32m1_t, s16, s32, i16, i32, 16, 32, 8, 4) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, int64x2, vint16m1_t, vint64m1_t, s16, s64, i16, i64, 16, 64, 8, 2) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int32x4, int64x2, vint32m1_t, vint64m1_t, s32, s64, i32, i64, 32, 64, 4, 2) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int16x8, vuint8m1_t, vint16m1_t, u8, s16, u8, i16, 8, 16, 16, 8) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int32x4, vuint8m1_t, vint32m1_t, u8, s32, u8, i32, 8, 32, 16, 4) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int64x2, vuint8m1_t, vint64m1_t, u8, s64, u8, i64, 8, 64, 16, 2) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int8x16, vuint16m1_t, vint8m1_t, u16, s8, u16, i8, 16, 8, 8, 16) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int32x4, vuint16m1_t, vint32m1_t, u16, s32, u16, i32, 16, 32, 8, 4) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int64x2, vuint16m1_t, vint64m1_t, u16, s64, u16, i64, 16, 64, 8, 2) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int8x16, vuint32m1_t, vint8m1_t, u32, s8, u32, i8, 32, 8, 4, 16) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int16x8, vuint32m1_t, vint16m1_t, u32, s16, u32, i16, 32, 16, 4, 8) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int64x2, vuint32m1_t, vint64m1_t, u32, s64, u32, i64, 32, 64, 4, 2) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int8x16, vuint64m1_t, vint8m1_t, u64, s8, u64, i8, 64, 8, 2, 16) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int16x8, vuint64m1_t, vint16m1_t, u64, s16, u64, i16, 64, 16, 2, 8) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int32x4, vuint64m1_t, vint32m1_t, u64, s32, u64, i32, 64, 32, 2, 4) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, float32x4, vuint8m1_t, vfloat32m1_t, u8, f32, u8, f32, 8, 32, 16, 4) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, float32x4, vuint16m1_t, vfloat32m1_t, u16, f32, u16, f32, 16, 32, 8, 4) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, float32x4, vuint64m1_t, vfloat32m1_t, u64, f32, u64, f32, 64, 32, 2, 4) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, float32x4, vint8m1_t, vfloat32m1_t, s8, f32, i8, f32, 8, 32, 16, 4) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, float32x4, vint16m1_t, vfloat32m1_t, s16, f32, i16, f32, 16, 32, 8, 4) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int64x2, float32x4, vint64m1_t, vfloat32m1_t, s64, f32, i64, f32, 64, 32, 2, 4) +OPENCV_HAL_IMPL_RVV_NATIVE_REINTERPRET(uint8x16, int8x16, u8, s8, u8, i8) +OPENCV_HAL_IMPL_RVV_NATIVE_REINTERPRET(uint16x8, int16x8, u16, s16, u16, i16) +OPENCV_HAL_IMPL_RVV_NATIVE_REINTERPRET(uint32x4, int32x4, u32, s32, u32, i32) +OPENCV_HAL_IMPL_RVV_NATIVE_REINTERPRET(uint32x4, float32x4, u32, f32, u32, f32) +OPENCV_HAL_IMPL_RVV_NATIVE_REINTERPRET(int32x4, float32x4, s32, f32, i32, f32) +OPENCV_HAL_IMPL_RVV_NATIVE_REINTERPRET(uint64x2, int64x2, u64, s64, u64, i64) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, float64x2, vuint64m1_t, vfloat64m1_t, u64, f64, u64, f64, 64, 64, 2, 2) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int64x2, float64x2, vint64m1_t, vfloat64m1_t, s64, f64, i64, f64, 64, 64, 2, 2) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, float64x2, vuint8m1_t, vfloat64m1_t, u8, f64, u8, f64, 8, 64, 16, 2) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, float64x2, vuint16m1_t, vfloat64m1_t, u16, f64, u16, f64, 16, 64, 6, 2) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, float64x2, vuint32m1_t, vfloat64m1_t, u32, f64, u32, f64, 32, 64, 4, 2) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, float64x2, vint8m1_t, vfloat64m1_t, s8, f64, i8, f64, 8, 64, 16, 2) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, float64x2, vint16m1_t, vfloat64m1_t, s16, f64, i16, f64, 16, 64, 8, 2) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int32x4, float64x2, vint32m1_t, vfloat64m1_t, s32, f64, i32, f64, 32, 64, 4, 2) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(float32x4, float64x2, vfloat32m1_t, vfloat64m1_t, f32, f64, f32, f64, 32, 64, 4, 2) +OPENCV_HAL_IMPL_RVV_NATIVE_REINTERPRET(uint64x2, float64x2, u64, f64, u64, f64) +OPENCV_HAL_IMPL_RVV_NATIVE_REINTERPRET(int64x2, float64x2, s64, f64, i64, f64) #endif +OPENCV_HAL_IMPL_RVV_NATIVE_REINTERPRET(uint8x16, uint16x8, u8, u16, u8, u16) +OPENCV_HAL_IMPL_RVV_NATIVE_REINTERPRET(uint8x16, uint32x4, u8, u32, u8, u32) +OPENCV_HAL_IMPL_RVV_NATIVE_REINTERPRET(uint8x16, uint64x2, u8, u64, u8, u64) +OPENCV_HAL_IMPL_RVV_NATIVE_REINTERPRET(uint16x8, uint32x4, u16, u32, u16, u32) +OPENCV_HAL_IMPL_RVV_NATIVE_REINTERPRET(uint16x8, uint64x2, u16, u64, u16, u64) +OPENCV_HAL_IMPL_RVV_NATIVE_REINTERPRET(uint32x4, uint64x2, u32, u64, u32, u64) +OPENCV_HAL_IMPL_RVV_NATIVE_REINTERPRET(int8x16, int16x8, s8, s16, i8, i16) +OPENCV_HAL_IMPL_RVV_NATIVE_REINTERPRET(int8x16, int32x4, s8, s32, i8, i32) +OPENCV_HAL_IMPL_RVV_NATIVE_REINTERPRET(int8x16, int64x2, s8, s64, i8, i64) +OPENCV_HAL_IMPL_RVV_NATIVE_REINTERPRET(int16x8, int32x4, s16, s32, i16, i32) +OPENCV_HAL_IMPL_RVV_NATIVE_REINTERPRET(int16x8, int64x2, s16, s64, i16, i64) +OPENCV_HAL_IMPL_RVV_NATIVE_REINTERPRET(int32x4, int64x2, s32, s64, i32, i64) + + +#define OPENCV_HAL_IMPL_RVV_TWO_TIMES_REINTERPRET(_Tpvec1, _Tpvec2, suffix1, suffix2, nsuffix1, nsuffix2, width1, width2) \ +inline v_##_Tpvec1 v_reinterpret_as_##suffix1(const v_##_Tpvec2& v) \ +{ \ + return v_##_Tpvec1(vreinterpret_v_##nsuffix1##width2##m1_##nsuffix1##width1##m1(vreinterpret_v_##nsuffix2##width2##m1_##nsuffix1##width2##m1(v)));\ +} \ +inline v_##_Tpvec2 v_reinterpret_as_##suffix2(const v_##_Tpvec1& v) \ +{ \ + return v_##_Tpvec2(vreinterpret_v_##nsuffix1##width2##m1_##nsuffix2##width2##m1(vreinterpret_v_##nsuffix1##width1##m1_##nsuffix1##width2##m1(v)));\ +} + +OPENCV_HAL_IMPL_RVV_TWO_TIMES_REINTERPRET(uint8x16, int16x8, u8, s16, u, i, 8, 16) +OPENCV_HAL_IMPL_RVV_TWO_TIMES_REINTERPRET(uint8x16, int32x4, u8, s32, u, i, 8, 32) +OPENCV_HAL_IMPL_RVV_TWO_TIMES_REINTERPRET(uint8x16, int64x2, u8, s64, u, i, 8, 64) +OPENCV_HAL_IMPL_RVV_TWO_TIMES_REINTERPRET(uint16x8, int8x16, u16, s8, u, i, 16, 8) +OPENCV_HAL_IMPL_RVV_TWO_TIMES_REINTERPRET(uint16x8, int32x4, u16, s32, u, i, 16, 32) +OPENCV_HAL_IMPL_RVV_TWO_TIMES_REINTERPRET(uint16x8, int64x2, u16, s64, u, i, 16, 64) +OPENCV_HAL_IMPL_RVV_TWO_TIMES_REINTERPRET(uint32x4, int8x16, u32, s8, u, i, 32, 8) +OPENCV_HAL_IMPL_RVV_TWO_TIMES_REINTERPRET(uint32x4, int16x8, u32, s16, u, i, 32, 16) +OPENCV_HAL_IMPL_RVV_TWO_TIMES_REINTERPRET(uint32x4, int64x2, u32, s64, u, i, 32, 64) +OPENCV_HAL_IMPL_RVV_TWO_TIMES_REINTERPRET(uint64x2, int8x16, u64, s8, u, i, 64, 8) +OPENCV_HAL_IMPL_RVV_TWO_TIMES_REINTERPRET(uint64x2, int16x8, u64, s16, u, i, 64, 16) +OPENCV_HAL_IMPL_RVV_TWO_TIMES_REINTERPRET(uint64x2, int32x4, u64, s32, u, i, 64, 32) +OPENCV_HAL_IMPL_RVV_TWO_TIMES_REINTERPRET(uint8x16, float32x4, u8, f32, u, f, 8, 32) +OPENCV_HAL_IMPL_RVV_TWO_TIMES_REINTERPRET(uint16x8, float32x4, u16, f32, u, f, 16, 32) +OPENCV_HAL_IMPL_RVV_TWO_TIMES_REINTERPRET(uint64x2, float32x4, u64, f32, u, f, 64, 32) +OPENCV_HAL_IMPL_RVV_TWO_TIMES_REINTERPRET(int8x16, float32x4, s8, f32, i, f, 8, 32) +OPENCV_HAL_IMPL_RVV_TWO_TIMES_REINTERPRET(int16x8, float32x4, s16, f32, i, f, 16, 32) +OPENCV_HAL_IMPL_RVV_TWO_TIMES_REINTERPRET(int64x2, float32x4, s64, f32, i, f, 64, 32) +OPENCV_HAL_IMPL_RVV_TWO_TIMES_REINTERPRET(uint8x16, float64x2, u8, f64, u, f, 8, 64) +OPENCV_HAL_IMPL_RVV_TWO_TIMES_REINTERPRET(uint16x8, float64x2, u16, f64, u, f, 16, 64) +OPENCV_HAL_IMPL_RVV_TWO_TIMES_REINTERPRET(uint32x4, float64x2, u32, f64, u, f, 32, 64) +OPENCV_HAL_IMPL_RVV_TWO_TIMES_REINTERPRET(int8x16, float64x2, s8, f64, i, f, 8, 64) +OPENCV_HAL_IMPL_RVV_TWO_TIMES_REINTERPRET(int16x8, float64x2, s16, f64, i, f, 16, 64) +OPENCV_HAL_IMPL_RVV_TWO_TIMES_REINTERPRET(int32x4, float64x2, s32, f64, i, f, 32, 64) + +// Three times reinterpret +inline v_float32x4 v_reinterpret_as_f32(const v_float64x2& v) \ +{ \ + return v_float32x4(vreinterpret_v_u32m1_f32m1(vreinterpret_v_u64m1_u32m1(vreinterpret_v_f64m1_u64m1(v))));\ +} \ +inline v_float64x2 v_reinterpret_as_f64(const v_float32x4& v) \ +{ \ + return v_float64x2(vreinterpret_v_u64m1_f64m1(vreinterpret_v_u32m1_u64m1(vreinterpret_v_f32m1_u32m1(v))));\ +} ////////////// Extract ////////////// @@ -686,7 +710,7 @@ OPENCV_HAL_IMPL_RVV_EXTRACT_FP(v_float64x2, double, f64, vfmv_f_s_f64m1_f64, 2) #define OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(_Tpvec, _nTpvec, _Tp, hvl, vl, width, suffix, vmv) \ inline _Tpvec v_load(const _Tp* ptr) \ { \ - return _Tpvec((_nTpvec)vle8_v_u8m1((uchar*)ptr, 16)); \ + return _Tpvec(vle##width##_v_##suffix##m1(ptr, vl)); \ } \ inline _Tpvec v_load_aligned(const _Tp* ptr) \ { \ @@ -699,7 +723,7 @@ inline _Tpvec v_load_low(const _Tp* ptr) \ } \ inline void v_store(_Tp* ptr, const _Tpvec& a) \ { \ - vse8_v_u8m1((uchar*)ptr, vle8_v_u8m1((uchar*)a.val, 16), 16); \ + vse##width##_v_##suffix##m1(ptr, a, vl); \ } \ inline void v_store_aligned(_Tp* ptr, const _Tpvec& a) \ { \ @@ -1411,9 +1435,10 @@ inline scalartype v_reduce_sum(const _Tpvec& a) \ return (scalartype)(_wTpvec(res).get0()); \ } -OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(v_float32x4, v_float32x4, vfloat32m1_t, float, f32, f32, 4, fredsum) +// vfredsum for float has renamed to fredosum, also updated in GNU. +OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(v_float32x4, v_float32x4, vfloat32m1_t, float, f32, f32, 4, fredosum) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(v_float64x2, v_float64x2, vfloat64m1_t, double, f64, f64, 2, fredsum) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(v_float64x2, v_float64x2, vfloat64m1_t, double, f64, f64, 2, fredosum) #endif @@ -1538,6 +1563,8 @@ inline v_float64x2 v_muladd(const v_float64x2& a, const v_float64x2& b, const v_ ////////////// Check all/any ////////////// +// use overloaded vcpop in clang, no casting like (vuint64m1_t) is needed. +#ifndef __clang__ #define OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(_Tpvec, suffix, shift, vl) \ inline bool v_check_all(const _Tpvec& a) \ { \ @@ -1587,7 +1614,55 @@ inline bool v_check_all(const v_float64x2& a) inline bool v_check_any(const v_float64x2& a) { return v_check_any(v_reinterpret_as_u64(a)); } #endif +#else +#define OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(_Tpvec, vl) \ +inline bool v_check_all(const _Tpvec& a) \ +{ \ + return vcpop(vmslt(a, 0, vl), vl) == vl; \ +} \ +inline bool v_check_any(const _Tpvec& a) \ +{ \ + return vcpop(vmslt(a, 0, vl), vl) != 0; \ +} +OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_int8x16, 16) +OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_int16x8, 8) +OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_int32x4, 4) +OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_int64x2, 2) + + +inline bool v_check_all(const v_uint8x16& a) +{ return v_check_all(v_reinterpret_as_s8(a)); } +inline bool v_check_any(const v_uint8x16& a) +{ return v_check_any(v_reinterpret_as_s8(a)); } + +inline bool v_check_all(const v_uint16x8& a) +{ return v_check_all(v_reinterpret_as_s16(a)); } +inline bool v_check_any(const v_uint16x8& a) +{ return v_check_any(v_reinterpret_as_s16(a)); } + +inline bool v_check_all(const v_uint32x4& a) +{ return v_check_all(v_reinterpret_as_s32(a)); } +inline bool v_check_any(const v_uint32x4& a) +{ return v_check_any(v_reinterpret_as_s32(a)); } + +inline bool v_check_all(const v_float32x4& a) +{ return v_check_all(v_reinterpret_as_s32(a)); } +inline bool v_check_any(const v_float32x4& a) +{ return v_check_any(v_reinterpret_as_s32(a)); } + +inline bool v_check_all(const v_uint64x2& a) +{ return v_check_all(v_reinterpret_as_s64(a)); } +inline bool v_check_any(const v_uint64x2& a) +{ return v_check_any(v_reinterpret_as_s64(a)); } + +#if CV_SIMD128_64F +inline bool v_check_all(const v_float64x2& a) +{ return v_check_all(v_reinterpret_as_s64(a)); } +inline bool v_check_any(const v_float64x2& a) +{ return v_check_any(v_reinterpret_as_s64(a)); } +#endif +#endif ////////////// abs ////////////// #define OPENCV_HAL_IMPL_RVV_ABSDIFF(_Tpvec, abs) \ @@ -1606,6 +1681,8 @@ OPENCV_HAL_IMPL_RVV_ABSDIFF(v_float64x2, absdiff) OPENCV_HAL_IMPL_RVV_ABSDIFF(v_int8x16, absdiffs) OPENCV_HAL_IMPL_RVV_ABSDIFF(v_int16x8, absdiffs) +// use reinterpret instead of c-style casting. +#ifndef __clang__ #define OPENCV_HAL_IMPL_RVV_ABSDIFF_S(_Tpvec, _rTpvec, _nwTpvec, sub, rshr, vl) \ inline _rTpvec v_absdiff(const _Tpvec& a, const _Tpvec& b) \ { \ @@ -1615,7 +1692,17 @@ inline _rTpvec v_absdiff(const _Tpvec& a, const _Tpvec& b) \ OPENCV_HAL_IMPL_RVV_ABSDIFF_S(v_int8x16, v_uint8x16, vuint16m2_t, vwsub_vv_i16m2, vnclipu_wx_u8m1, 16) OPENCV_HAL_IMPL_RVV_ABSDIFF_S(v_int16x8, v_uint16x8, vuint32m2_t, vwsub_vv_i32m2, vnclipu_wx_u16m1, 8) OPENCV_HAL_IMPL_RVV_ABSDIFF_S(v_int32x4, v_uint32x4, vuint64m2_t, vwsub_vv_i64m2, vnclipu_wx_u32m1, 4) +#else +#define OPENCV_HAL_IMPL_RVV_ABSDIFF_S(_Tpvec, _rTpvec, _nwTpvec, sub, rshr, width, vl) \ +inline _rTpvec v_absdiff(const _Tpvec& a, const _Tpvec& b) \ +{ \ + return _rTpvec(rshr(vreinterpret_u##width##m2(sub(v_max(a, b), v_min(a, b), vl)), 0, vl)); \ +} +OPENCV_HAL_IMPL_RVV_ABSDIFF_S(v_int8x16, v_uint8x16, vuint16m2_t, vwsub_vv_i16m2, vnclipu_wx_u8m1, 16, 16) +OPENCV_HAL_IMPL_RVV_ABSDIFF_S(v_int16x8, v_uint16x8, vuint32m2_t, vwsub_vv_i32m2, vnclipu_wx_u16m1, 32, 8) +OPENCV_HAL_IMPL_RVV_ABSDIFF_S(v_int32x4, v_uint32x4, vuint64m2_t, vwsub_vv_i64m2, vnclipu_wx_u32m1, 64, 4) +#endif #define OPENCV_HAL_IMPL_RVV_ABS(_Tprvec, _Tpvec, suffix) \ inline _Tprvec v_abs(const _Tpvec& a) \ { \ @@ -2323,6 +2410,8 @@ OPENCV_HAL_IMPL_RVV_SCAN_FORWOARD_OP(v_float64x2, double, f64) //////////// Pack triplets //////////// +// use reinterpret instead of c-style casting. +#ifndef __clang__ inline v_int8x16 v_pack_triplets(const v_int8x16& vec) { uint64 ptr[2] = {0x0908060504020100, 0xFFFFFFFF0E0D0C0A}; @@ -2347,6 +2436,33 @@ inline v_int32x4 v_pack_triplets(const v_int32x4& vec) { return vec; } inline v_uint32x4 v_pack_triplets(const v_uint32x4& vec) { return vec; } inline v_float32x4 v_pack_triplets(const v_float32x4& vec) { return vec; } +#else + +inline v_int8x16 v_pack_triplets(const v_int8x16& vec) +{ + uint64 ptr[2] = {0x0908060504020100, 0xFFFFFFFF0E0D0C0A}; + return v_int8x16(vreinterpret_i8m1(vrgather_vv_u8m1(v_reinterpret_as_u8(vec), vreinterpret_u8m1(vle64_v_u64m1(ptr, 2)), 16))); +} +inline v_uint8x16 v_pack_triplets(const v_uint8x16& vec) +{ + return v_reinterpret_as_u8(v_pack_triplets(v_reinterpret_as_s8(vec))); +} + +inline v_int16x8 v_pack_triplets(const v_int16x8& vec) +{ + uint64 ptr[2] = {0x0908060504020100, 0xFFFFFFFF0E0D0C0A}; + return v_int16x8(v_reinterpret_as_s16(v_uint8x16(vrgather_vv_u8m1(v_reinterpret_as_u8(vec), vreinterpret_u8m1(vle64_v_u64m1(ptr, 2)), 16)))); +} +inline v_uint16x8 v_pack_triplets(const v_uint16x8& vec) +{ + return v_reinterpret_as_u16(v_pack_triplets(v_reinterpret_as_s16(vec))); +} + +inline v_int32x4 v_pack_triplets(const v_int32x4& vec) { return vec; } +inline v_uint32x4 v_pack_triplets(const v_uint32x4& vec) { return vec; } +inline v_float32x4 v_pack_triplets(const v_float32x4& vec) { return vec; } + +#endif ////// FP16 support /////// diff --git a/modules/dnn/src/layers/layers_common.simd.hpp b/modules/dnn/src/layers/layers_common.simd.hpp index eba779f7f2..66cfe3e0ff 100644 --- a/modules/dnn/src/layers/layers_common.simd.hpp +++ b/modules/dnn/src/layers/layers_common.simd.hpp @@ -819,13 +819,13 @@ void fastGEMM( const float* aptr, size_t astep, const float* bptr, for( int k = 0; k < na; k++ ) { - float32_t a0 = aptr0[k]; - float32_t a1 = aptr1[k]; - float32_t a2 = aptr2[k]; - float32_t a3 = aptr3[k]; - float32_t a4 = aptr4[k]; - float32_t a5 = aptr5[k]; - float32_t a6 = aptr6[k]; + float a0 = aptr0[k]; + float a1 = aptr1[k]; + float a2 = aptr2[k]; + float a3 = aptr3[k]; + float a4 = aptr4[k]; + float a5 = aptr5[k]; + float a6 = aptr6[k]; vfloat32m4_t b = vle32_v_f32m4(bptr + k*bstep + n, mvl); d0 = vfmacc_vf_f32m4(d0, a0, b, mvl); @@ -904,23 +904,23 @@ void fastGEMM1T( const float* vec, const float* weights, } // Calculate the sum of each vector - float32_t sum[15]; + float sum[15]; vfloat32m1_t zero = vfmv_v_f_f32m1(0, vlm2); - sum[0] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs0, zero, vlm2)); - sum[1] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs1, zero, vlm2)); - sum[2] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs2, zero, vlm2)); - sum[3] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs3, zero, vlm2)); - sum[4] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs4, zero, vlm2)); - sum[5] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs5, zero, vlm2)); - sum[6] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs6, zero, vlm2)); - sum[7] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs7, zero, vlm2)); - sum[8] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs8, zero, vlm2)); - sum[9] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs9, zero, vlm2)); - sum[10] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs10, zero, vlm2)); - sum[11] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs11, zero, vlm2)); - sum[12] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs12, zero, vlm2)); - sum[13] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs13, zero, vlm2)); - sum[14] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs14, zero, vlm2)); + sum[0] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs0, zero, vlm2)); + sum[1] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs1, zero, vlm2)); + sum[2] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs2, zero, vlm2)); + sum[3] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs3, zero, vlm2)); + sum[4] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs4, zero, vlm2)); + sum[5] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs5, zero, vlm2)); + sum[6] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs6, zero, vlm2)); + sum[7] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs7, zero, vlm2)); + sum[8] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs8, zero, vlm2)); + sum[9] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs9, zero, vlm2)); + sum[10] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs10, zero, vlm2)); + sum[11] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs11, zero, vlm2)); + sum[12] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs12, zero, vlm2)); + sum[13] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs13, zero, vlm2)); + sum[14] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs14, zero, vlm2)); vfloat32m4_t s0 = vfadd_vv_f32m4(vle32_v_f32m4(sum, 15), vle32_v_f32m4(bias + i, 15), 15); vse32_v_f32m4(dst + i, s0, 15); @@ -973,22 +973,22 @@ void fastGEMM1T( const float* vec, const float* weights, vs13 = vfmacc_vv_f32m2(vs13, vle32_v_f32m2(wptr + wstep*std::min(13, mvl-1), kvl), v, kvl); } // Calculate the sum of each vector - float32_t sum[14]; + float sum[14]; vfloat32m1_t zero = vfmv_v_f_f32m1(0, vlm2); - sum[0] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs0, zero, vlm2)); - sum[1] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs1, zero, vlm2)); - sum[2] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs2, zero, vlm2)); - sum[3] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs3, zero, vlm2)); - sum[4] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs4, zero, vlm2)); - sum[5] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs5, zero, vlm2)); - sum[6] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs6, zero, vlm2)); - sum[7] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs7, zero, vlm2)); - sum[8] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs8, zero, vlm2)); - sum[9] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs9, zero, vlm2)); - sum[10] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs10, zero, vlm2)); - sum[11] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs11, zero, vlm2)); - sum[12] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs12, zero, vlm2)); - sum[13] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs13, zero, vlm2)); + sum[0] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs0, zero, vlm2)); + sum[1] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs1, zero, vlm2)); + sum[2] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs2, zero, vlm2)); + sum[3] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs3, zero, vlm2)); + sum[4] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs4, zero, vlm2)); + sum[5] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs5, zero, vlm2)); + sum[6] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs6, zero, vlm2)); + sum[7] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs7, zero, vlm2)); + sum[8] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs8, zero, vlm2)); + sum[9] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs9, zero, vlm2)); + sum[10] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs10, zero, vlm2)); + sum[11] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs11, zero, vlm2)); + sum[12] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs12, zero, vlm2)); + sum[13] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs13, zero, vlm2)); vfloat32m4_t s0 = vfadd_vv_f32m4(vle32_v_f32m4(sum, mvl), vle32_v_f32m4(bias + i, mvl), mvl); vse32_v_f32m4(dst + i, s0, mvl); @@ -1110,31 +1110,31 @@ void fastConv( const float* weights, size_t wstep, const float* bias, // compute sum of each vs vfloat32m1_t zero = vfmv_v_f_f32m1(0, vlm1Max); // vl is required here to be at least FASCONV_BASE_VECSZ, aka 8. - float32_t sum0[FASCONV_BASE_VECSZ], sum1[FASCONV_BASE_VECSZ], sum2[FASCONV_BASE_VECSZ]; - sum0[0] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs00, zero, vlm1Max)); - sum0[1] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs01, zero, vlm1Max)); - sum0[2] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs02, zero, vlm1Max)); - sum0[3] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs03, zero, vlm1Max)); - sum0[4] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs04, zero, vlm1Max)); - sum0[5] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs05, zero, vlm1Max)); - sum0[6] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs06, zero, vlm1Max)); - sum0[7] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs07, zero, vlm1Max)); - sum1[0] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs10, zero, vlm1Max)); - sum1[1] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs11, zero, vlm1Max)); - sum1[2] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs12, zero, vlm1Max)); - sum1[3] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs13, zero, vlm1Max)); - sum1[4] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs14, zero, vlm1Max)); - sum1[5] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs15, zero, vlm1Max)); - sum1[6] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs16, zero, vlm1Max)); - sum1[7] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs17, zero, vlm1Max)); - sum2[0] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs20, zero, vlm1Max)); - sum2[1] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs21, zero, vlm1Max)); - sum2[2] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs22, zero, vlm1Max)); - sum2[3] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs23, zero, vlm1Max)); - sum2[4] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs24, zero, vlm1Max)); - sum2[5] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs25, zero, vlm1Max)); - sum2[6] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs26, zero, vlm1Max)); - sum2[7] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs27, zero, vlm1Max)); + float sum0[FASCONV_BASE_VECSZ], sum1[FASCONV_BASE_VECSZ], sum2[FASCONV_BASE_VECSZ]; + sum0[0] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs00, zero, vlm1Max)); + sum0[1] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs01, zero, vlm1Max)); + sum0[2] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs02, zero, vlm1Max)); + sum0[3] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs03, zero, vlm1Max)); + sum0[4] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs04, zero, vlm1Max)); + sum0[5] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs05, zero, vlm1Max)); + sum0[6] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs06, zero, vlm1Max)); + sum0[7] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs07, zero, vlm1Max)); + sum1[0] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs10, zero, vlm1Max)); + sum1[1] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs11, zero, vlm1Max)); + sum1[2] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs12, zero, vlm1Max)); + sum1[3] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs13, zero, vlm1Max)); + sum1[4] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs14, zero, vlm1Max)); + sum1[5] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs15, zero, vlm1Max)); + sum1[6] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs16, zero, vlm1Max)); + sum1[7] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs17, zero, vlm1Max)); + sum2[0] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs20, zero, vlm1Max)); + sum2[1] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs21, zero, vlm1Max)); + sum2[2] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs22, zero, vlm1Max)); + sum2[3] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs23, zero, vlm1Max)); + sum2[4] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs24, zero, vlm1Max)); + sum2[5] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs25, zero, vlm1Max)); + sum2[6] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs26, zero, vlm1Max)); + sum2[7] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs27, zero, vlm1Max)); // if VLEN = 128, so LMUL = 2 for vl = 8. // otherwise, VLEN >=256, we only use fist 8 element of the vReg. @@ -1217,7 +1217,7 @@ static inline void vfloat32m2_load_deinterleave(const float* ptr, vfloat32m2_t& // a = vlmul_trunc_v_f32m4_f32m2(tempa); // b = vlmul_trunc_v_f32m4_f32m2(tempb); */ - cv::AutoBuffer cvBuffer(sizeof(float32_t)*vl*2); + cv::AutoBuffer cvBuffer(sizeof(float)*vl*2); float* buffer = (float*)cvBuffer.data(); vse32_v_f32m4(buffer, tempa, vl); a = vle32_v_f32m2(buffer, vl); diff --git a/platforms/linux/riscv64-clang.toolchain.cmake b/platforms/linux/riscv64-clang.toolchain.cmake index c1c74ab9df..f19c244f7b 100644 --- a/platforms/linux/riscv64-clang.toolchain.cmake +++ b/platforms/linux/riscv64-clang.toolchain.cmake @@ -17,8 +17,11 @@ set(CMAKE_ASM_COMPILER_TARGET ${CLANG_TARGET_TRIPLE}) # Don't run the linker on compiler check set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) -set(CMAKE_C_FLAGS "-march=rv64gcv0p9 -menable-experimental-extensions --gcc-toolchain=${RISCV_GCC_INSTALL_ROOT} -w ${CMAKE_C_FLAGS}") -set(CMAKE_CXX_FLAGS "-march=rv64gcv0p9 -menable-experimental-extensions --gcc-toolchain=${RISCV_GCC_INSTALL_ROOT} -w ${CXX_FLAGS}") +set(CMAKE_C_FLAGS "-march=rv64gcv0p10 -menable-experimental-extensions --gcc-toolchain=${RISCV_GCC_INSTALL_ROOT} -w ${CMAKE_C_FLAGS}") +set(CMAKE_CXX_FLAGS "-march=rv64gcv0p10 -menable-experimental-extensions --gcc-toolchain=${RISCV_GCC_INSTALL_ROOT} -w ${CXX_FLAGS}") + +set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O2") +set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O2") set(CMAKE_FIND_ROOT_PATH ${CMAKE_SYSROOT}) set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) From d2f87ca76cdbe61afa14142dc7011694e01b2ae3 Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Fri, 3 Dec 2021 08:40:49 -0800 Subject: [PATCH 129/226] Merge pull request #21147 from mjmdavis:4.x * remove tickmarks on NSSlider --- modules/highgui/src/window_cocoa.mm | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/highgui/src/window_cocoa.mm b/modules/highgui/src/window_cocoa.mm index e8e9034406..97a6831f1a 100644 --- a/modules/highgui/src/window_cocoa.mm +++ b/modules/highgui/src/window_cocoa.mm @@ -944,8 +944,6 @@ static NSSize constrainAspectRatio(NSSize base, NSSize constraint) { [[slider name] setStringValue:cvname]; [[slider slider] setMaxValue:max]; [[slider slider] setMinValue:0]; - [[slider slider] setNumberOfTickMarks:(max+1)]; - [[slider slider] setAllowsTickMarkValuesOnly:YES]; if(value) { [[slider slider] setIntValue:*value]; From 0c0b1ec9ae3f8e50470a9fa689e86c7dbf219fb1 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 4 Dec 2021 04:36:01 +0000 Subject: [PATCH 130/226] imgproc(test): add bit-exact tests for YUV cvtColor conversions --- modules/imgproc/test/test_color.cpp | 77 +++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/modules/imgproc/test/test_color.cpp b/modules/imgproc/test/test_color.cpp index f828dac18a..d0dfe21f34 100644 --- a/modules/imgproc/test/test_color.cpp +++ b/modules/imgproc/test/test_color.cpp @@ -2812,6 +2812,83 @@ TEST(Imgproc_ColorLuv_Full, bitExactness) } } + +static +void runCvtColorBitExactCheck(ColorConversionCodes code, int inputType, uint32_t hash, Size sz = Size(263, 255), int rngSeed = 0) +{ + RNG rng(rngSeed); + + Mat src(sz, inputType, Scalar::all(0)); + Mat dst; + rng.fill(src, RNG::UNIFORM, 0, 255, true); + + cv::cvtColor(src, dst, code); + + uint32_t dst_hash = adler32(dst); + + EXPECT_EQ(hash, dst_hash) << cv::format("0x%08llx", (long long int)dst_hash); + + if (cvtest::debugLevel > 0) + { + const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info(); + CV_Assert(test_info); + std::string name = (std::string(test_info->test_case_name()) + "--" + test_info->name() + ".xml"); + cv::FileStorage fs(name, cv::FileStorage::WRITE); + fs << "dst" << dst; + } +} + +TEST(Imgproc_cvtColor_BE, COLOR_BGR2YUV) { runCvtColorBitExactCheck(COLOR_BGR2YUV, CV_8UC3, 0xc2cbcfda); } +TEST(Imgproc_cvtColor_BE, COLOR_RGB2YUV) { runCvtColorBitExactCheck(COLOR_RGB2YUV, CV_8UC3, 0x4e98e757); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2BGR) { runCvtColorBitExactCheck(COLOR_YUV2BGR, CV_8UC3, 0xb2c62a3f); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2RGB) { runCvtColorBitExactCheck(COLOR_YUV2RGB, CV_8UC3, 0x6d242a3f); } + +// packed input +TEST(Imgproc_cvtColor_BE, COLOR_YUV2RGB_NV12) { runCvtColorBitExactCheck(COLOR_YUV2RGB_NV12, CV_8UC1, 0x46a1bb76, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2BGR_NV12) { runCvtColorBitExactCheck(COLOR_YUV2BGR_NV12, CV_8UC1, 0x3843bb76, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2RGB_NV21) { runCvtColorBitExactCheck(COLOR_YUV2RGB_NV21, CV_8UC1, 0xf3fdf2ea, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2BGR_NV21) { runCvtColorBitExactCheck(COLOR_YUV2BGR_NV21, CV_8UC1, 0x6e84f2ea, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2RGBA_NV12) { runCvtColorBitExactCheck(COLOR_YUV2RGBA_NV12, CV_8UC1, 0xb6a16bd3, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2BGRA_NV12) { runCvtColorBitExactCheck(COLOR_YUV2BGRA_NV12, CV_8UC1, 0xa8436bd3, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2RGBA_NV21) { runCvtColorBitExactCheck(COLOR_YUV2RGBA_NV21, CV_8UC1, 0x1c7fa347, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2BGRA_NV21) { runCvtColorBitExactCheck(COLOR_YUV2BGRA_NV21, CV_8UC1, 0x96f7a347, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2RGB_YV12) { runCvtColorBitExactCheck(COLOR_YUV2RGB_YV12, CV_8UC1, 0xc5da1651, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2BGR_YV12) { runCvtColorBitExactCheck(COLOR_YUV2BGR_YV12, CV_8UC1, 0x12161651, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2RGB_IYUV) { runCvtColorBitExactCheck(COLOR_YUV2RGB_IYUV, CV_8UC1, 0xb4e62ea5, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2BGR_IYUV) { runCvtColorBitExactCheck(COLOR_YUV2BGR_IYUV, CV_8UC1, 0xfa632ea5, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2RGBA_YV12) { runCvtColorBitExactCheck(COLOR_YUV2RGBA_YV12, CV_8UC1, 0x0db4c69f, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2BGRA_YV12) { runCvtColorBitExactCheck(COLOR_YUV2BGRA_YV12, CV_8UC1, 0x59e1c69f, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2RGBA_IYUV) { runCvtColorBitExactCheck(COLOR_YUV2RGBA_IYUV, CV_8UC1, 0xfe09def3, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2BGRA_IYUV) { runCvtColorBitExactCheck(COLOR_YUV2BGRA_IYUV, CV_8UC1, 0x4395def3, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2GRAY_420) { runCvtColorBitExactCheck(COLOR_YUV2GRAY_420, CV_8UC1, 0xf672b440, Size(262, 510)); } + +TEST(Imgproc_cvtColor_BE, COLOR_YUV2RGB_UYVY) { runCvtColorBitExactCheck(COLOR_YUV2RGB_UYVY, CV_8UC2, 0x69bea2c1, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2BGR_UYVY) { runCvtColorBitExactCheck(COLOR_YUV2BGR_UYVY, CV_8UC2, 0xdc51a2c1, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2RGBA_UYVY) { runCvtColorBitExactCheck(COLOR_YUV2RGBA_UYVY, CV_8UC2, 0x851eab45, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2BGRA_UYVY) { runCvtColorBitExactCheck(COLOR_YUV2BGRA_UYVY, CV_8UC2, 0xf7b1ab45, Size(262, 510)); } + +TEST(Imgproc_cvtColor_BE, COLOR_YUV2RGB_YUY2) { runCvtColorBitExactCheck(COLOR_YUV2RGB_YUY2, CV_8UC2, 0x607e8889, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2BGR_YUY2) { runCvtColorBitExactCheck(COLOR_YUV2BGR_YUY2, CV_8UC2, 0xfb148889, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2RGB_YVYU) { runCvtColorBitExactCheck(COLOR_YUV2RGB_YVYU, CV_8UC2, 0x239b13d4, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2BGR_YVYU) { runCvtColorBitExactCheck(COLOR_YUV2BGR_YVYU, CV_8UC2, 0x402b13d4, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2RGBA_YUY2) { runCvtColorBitExactCheck(COLOR_YUV2RGBA_YUY2, CV_8UC2, 0xf6af910d, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2BGRA_YUY2) { runCvtColorBitExactCheck(COLOR_YUV2BGRA_YUY2, CV_8UC2, 0x9154910d, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2RGBA_YVYU) { runCvtColorBitExactCheck(COLOR_YUV2RGBA_YVYU, CV_8UC2, 0x14481c58, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2BGRA_YVYU) { runCvtColorBitExactCheck(COLOR_YUV2BGRA_YVYU, CV_8UC2, 0x30d81c58, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2GRAY_UYVY) { runCvtColorBitExactCheck(COLOR_YUV2GRAY_UYVY, CV_8UC2, 0x228e669c, Size(262, 510)); } +TEST(Imgproc_cvtColor_BE, COLOR_YUV2GRAY_YUY2) { runCvtColorBitExactCheck(COLOR_YUV2GRAY_YUY2, CV_8UC2, 0x125c62fd, Size(262, 510)); } + +TEST(Imgproc_cvtColor_BE, COLOR_RGB2YUV_I420) { runCvtColorBitExactCheck(COLOR_RGB2YUV_I420, CV_8UC3, 0x44bb076a, Size(262, 254)); } +TEST(Imgproc_cvtColor_BE, COLOR_BGR2YUV_I420) { runCvtColorBitExactCheck(COLOR_BGR2YUV_I420, CV_8UC3, 0xf908ff52, Size(262, 254)); } +TEST(Imgproc_cvtColor_BE, COLOR_RGBA2YUV_I420) { runCvtColorBitExactCheck(COLOR_RGBA2YUV_I420, CV_8UC3, 0x44bb076a, Size(262, 254)); } +TEST(Imgproc_cvtColor_BE, COLOR_BGRA2YUV_I420) { runCvtColorBitExactCheck(COLOR_BGRA2YUV_I420, CV_8UC3, 0xf908ff52, Size(262, 254)); } + +TEST(Imgproc_cvtColor_BE, COLOR_RGB2YUV_YV12) { runCvtColorBitExactCheck(COLOR_RGB2YUV_YV12, CV_8UC3, 0x1b0d076a, Size(262, 254)); } +TEST(Imgproc_cvtColor_BE, COLOR_BGR2YUV_YV12) { runCvtColorBitExactCheck(COLOR_BGR2YUV_YV12, CV_8UC3, 0xda8aff52, Size(262, 254)); } +TEST(Imgproc_cvtColor_BE, COLOR_RGBA2YUV_YV12) { runCvtColorBitExactCheck(COLOR_RGBA2YUV_YV12, CV_8UC3, 0x1b0d076a, Size(262, 254)); } +TEST(Imgproc_cvtColor_BE, COLOR_BGRA2YUV_YV12) { runCvtColorBitExactCheck(COLOR_BGRA2YUV_YV12, CV_8UC3, 0xda8aff52, Size(262, 254)); } + + static void test_Bayer2RGB_EdgeAware_8u(const Mat& src, Mat& dst, int code) { if (dst.empty()) From a3287b85c5090a231473120cd7e8e6871bc5a849 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 4 Dec 2021 07:32:57 +0000 Subject: [PATCH 131/226] carotene: disable YUV color conversions (bit-exact issue) --- 3rdparty/carotene/hal/tegra_hal.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/3rdparty/carotene/hal/tegra_hal.hpp b/3rdparty/carotene/hal/tegra_hal.hpp index 7fcca975fe..c2ae0c0d87 100644 --- a/3rdparty/carotene/hal/tegra_hal.hpp +++ b/3rdparty/carotene/hal/tegra_hal.hpp @@ -1844,14 +1844,18 @@ TegraCvtColor_Invoker(bgrx2hsvf, bgrx2hsv, src_data + static_cast(range. #define cv_hal_cvtBGRtoGray TEGRA_CVTBGRTOGRAY #undef cv_hal_cvtGraytoBGR #define cv_hal_cvtGraytoBGR TEGRA_CVTGRAYTOBGR +#if 0 // bit-exact tests are failed #undef cv_hal_cvtBGRtoYUV #define cv_hal_cvtBGRtoYUV TEGRA_CVTBGRTOYUV +#endif #undef cv_hal_cvtBGRtoHSV #define cv_hal_cvtBGRtoHSV TEGRA_CVTBGRTOHSV +#if 0 // bit-exact tests are failed #undef cv_hal_cvtTwoPlaneYUVtoBGR #define cv_hal_cvtTwoPlaneYUVtoBGR TEGRA_CVT2PYUVTOBGR #undef cv_hal_cvtTwoPlaneYUVtoBGREx #define cv_hal_cvtTwoPlaneYUVtoBGREx TEGRA_CVT2PYUVTOBGR_EX +#endif #endif // OPENCV_IMGPROC_HAL_INTERFACE_H From cb08c15616e69168de0e3ff9ca63ad464dcf3cae Mon Sep 17 00:00:00 2001 From: MaximMilashchenko <67949029+MaximMilashchenko@users.noreply.github.com> Date: Sat, 4 Dec 2021 13:37:10 +0300 Subject: [PATCH 132/226] Merge pull request #21145 from MaximMilashchenko:AudioUpdate Audio MSMF: added the ability to set sample per second * Audio MSMF: added the ability to set sample per second * changed the valid sampling rate check * fixed docs * add test * fixed warning * fixed error * fixed error --- modules/videoio/include/opencv2/videoio.hpp | 2 +- modules/videoio/src/cap_msmf.cpp | 72 ++++++++++++++++++--- modules/videoio/test/test_audio.cpp | 11 ++++ 3 files changed, 74 insertions(+), 11 deletions(-) diff --git a/modules/videoio/include/opencv2/videoio.hpp b/modules/videoio/include/opencv2/videoio.hpp index 8d97c31282..93ea8cdddc 100644 --- a/modules/videoio/include/opencv2/videoio.hpp +++ b/modules/videoio/include/opencv2/videoio.hpp @@ -195,7 +195,7 @@ enum VideoCaptureProperties { CAP_PROP_AUDIO_POS = 59, //!< (read-only) Audio position is measured in samples. Accurate audio sample timestamp of previous grabbed fragment. See CAP_PROP_AUDIO_SAMPLES_PER_SECOND and CAP_PROP_AUDIO_SHIFT_NSEC. CAP_PROP_AUDIO_SHIFT_NSEC = 60, //!< (read only) Contains the time difference between the start of the audio stream and the video stream in nanoseconds. Positive value means that audio is started after the first video frame. Negative value means that audio is started before the first video frame. CAP_PROP_AUDIO_DATA_DEPTH = 61, //!< (open, read) Alternative definition to bits-per-sample, but with clear handling of 32F / 32S - CAP_PROP_AUDIO_SAMPLES_PER_SECOND = 62, //!< (read-only) determined from file/codec input. If not specified, then selected audio sample rate is 44100 + CAP_PROP_AUDIO_SAMPLES_PER_SECOND = 62, //!< (open, read) determined from file/codec input. If not specified, then selected audio sample rate is 44100 CAP_PROP_AUDIO_BASE_INDEX = 63, //!< (read-only) Index of the first audio channel for .retrieve() calls. That audio channel number continues enumeration after video channels. CAP_PROP_AUDIO_TOTAL_CHANNELS = 64, //!< (read-only) Number of audio channels in the selected audio stream (mono, stereo, etc) CAP_PROP_AUDIO_TOTAL_STREAMS = 65, //!< (read-only) Number of audio streams. diff --git a/modules/videoio/src/cap_msmf.cpp b/modules/videoio/src/cap_msmf.cpp index 39f191e642..68171ea7af 100644 --- a/modules/videoio/src/cap_msmf.cpp +++ b/modules/videoio/src/cap_msmf.cpp @@ -727,6 +727,7 @@ protected: bool configureHW(bool enable); bool configureStreams(const cv::VideoCaptureParameters&); bool setAudioProperties(const cv::VideoCaptureParameters&); + bool checkAudioProperties(); template bool readComplexPropery(long prop, long& val) const; @@ -766,6 +767,7 @@ protected: unsigned int audioBaseIndex; int outputVideoFormat; int outputAudioFormat; + UINT32 audioSamplesPerSecond; bool convertFormat; MFTIME duration; LONGLONG frameStep; @@ -818,6 +820,7 @@ CvCapture_MSMF::CvCapture_MSMF(): audioBaseIndex(1), outputVideoFormat(CV_CAP_MODE_BGR), outputAudioFormat(CV_16S), + audioSamplesPerSecond(0), convertFormat(true), duration(0), frameStep(0), @@ -1047,7 +1050,7 @@ bool CvCapture_MSMF::configureAudioOutput(MediaType newType) MediaType newFormat = bestMatch.second; newFormat.majorType = MFMediaType_Audio; - newFormat.nSamplesPerSec = 44100; + newFormat.nSamplesPerSec = (audioSamplesPerSecond == 0) ? 44100 : audioSamplesPerSecond; switch (outputAudioFormat) { case CV_8S: @@ -1147,7 +1150,8 @@ bool CvCapture_MSMF::open(int index, const cv::VideoCaptureParameters* params) if (params) { configureHW(*params); - configureStreams(*params); + if (!(configureStreams(*params) && setAudioProperties(*params))) + return false; } if (videoStream != -1 && audioStream != -1 || videoStream == -1 && audioStream == -1) { @@ -1189,6 +1193,12 @@ bool CvCapture_MSMF::open(int index, const cv::VideoCaptureParameters* params) close(); return false; } + if (isOpen) + { + if (audioStream != -1) + if (!checkAudioProperties()) + return false; + } return isOpen; } @@ -1202,8 +1212,8 @@ bool CvCapture_MSMF::open(const cv::String& _filename, const cv::VideoCapturePar if (params) { configureHW(*params); - configureStreams(*params); - setAudioProperties(*params); + if (!(configureStreams(*params) && setAudioProperties(*params))) + return false; } // Set source reader parameters _ComPtr attr = getDefaultSourceConfig(); @@ -1235,12 +1245,19 @@ bool CvCapture_MSMF::open(const cv::String& _filename, const cv::VideoCapturePar return false; } if (isOpen) - if (audioStream != -1 && videoStream != -1) + { + if (audioStream != -1) { - isOpen = grabFrame(); - if (isOpen) - grabIsDone = true; + if (!checkAudioProperties()) + return false; + if (videoStream != -1) + { + isOpen = grabFrame(); + if (isOpen) + grabIsDone = true; + } } + } return isOpen; } @@ -1318,14 +1335,49 @@ bool CvCapture_MSMF::setAudioProperties(const cv::VideoCaptureParameters& params outputAudioFormat = value; } } + if (params.has(CAP_PROP_AUDIO_SAMPLES_PER_SECOND)) + { + int value = static_cast(params.get(CAP_PROP_AUDIO_SAMPLES_PER_SECOND)); + if (value < 0) + { + CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: CAP_PROP_AUDIO_SAMPLES_PER_SECOND parameter can't be negative: " << value); + return false; + } + else + { + audioSamplesPerSecond = value; + } + } if (params.has(CAP_PROP_AUDIO_SYNCHRONIZE)) { - int value = static_cast(params.get(CAP_PROP_AUDIO_SYNCHRONIZE)); + int value = static_cast(params.get(CAP_PROP_AUDIO_SYNCHRONIZE)); syncLastFrame = (value != 0) ? true : false; } return true; } - +bool CvCapture_MSMF::checkAudioProperties() +{ + if (audioSamplesPerSecond != 0) + { + _ComPtr type; + UINT32 actualAudioSamplesPerSecond = 0; + HRESULT hr = videoFileSource->GetCurrentMediaType(dwAudioStreamIndex, &type); + if (SUCCEEDED(hr)) + { + type->GetUINT32(MF_MT_AUDIO_SAMPLES_PER_SECOND , &actualAudioSamplesPerSecond); + if (actualAudioSamplesPerSecond != audioSamplesPerSecond) + { + CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: CAP_PROP_AUDIO_SAMPLES_PER_SECOND parameter value is invalid/unsupported: " << audioSamplesPerSecond + << ". Current value of CAP_PROP_AUDIO_SAMPLES_PER_SECOND: " << actualAudioSamplesPerSecond); + close(); + return false; + } + return true; + } + return false; + } + return true; +} bool CvCapture_MSMF::grabVideoFrame() { DWORD streamIndex, flags; diff --git a/modules/videoio/test/test_audio.cpp b/modules/videoio/test/test_audio.cpp index 0b637aeabd..7c66b83e34 100644 --- a/modules/videoio/test/test_audio.cpp +++ b/modules/videoio/test/test_audio.cpp @@ -281,4 +281,15 @@ TEST(AudioOpenCheck, bad_arg_invalid_audio_stream) ASSERT_FALSE(cap.isOpened()); } +TEST(AudioOpenCheck, bad_arg_invalid_audio_sample_per_second) +{ + std::string fileName = "audio/test_audio.mp4"; + std::vector params { CAP_PROP_AUDIO_STREAM, 0, + CAP_PROP_VIDEO_STREAM, -1, + CAP_PROP_AUDIO_SAMPLES_PER_SECOND, (int)1e9 }; + VideoCapture cap; + cap.open(findDataFile(fileName), cv::CAP_MSMF, params); + ASSERT_FALSE(cap.isOpened()); +} + }} //namespace From 676a724491de423c5d964d83594f7e99aa0d31c7 Mon Sep 17 00:00:00 2001 From: Tejas M R <52408360+tezz-io@users.noreply.github.com> Date: Sun, 5 Dec 2021 18:17:44 +0530 Subject: [PATCH 133/226] Merge pull request #21180 from tezz-io:4.x Added CV_PROP_RW macro to keypoints * Added CV_PROP_RW macro to keypoints As outlined in the feature request in the issue https://github.com/opencv/opencv/issues/21171 : the keypoints field has been made parsable by the bindings. * Added test for keypoints Added test to check if the CV_PROP_RW macro added in the previous commit makes keypoints public and accessible through the python API. --- .../stitching/include/opencv2/stitching/detail/matchers.hpp | 2 +- modules/stitching/misc/python/test/test_stitching.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/stitching/include/opencv2/stitching/detail/matchers.hpp b/modules/stitching/include/opencv2/stitching/detail/matchers.hpp index ef4684fd05..cd9749ca8b 100644 --- a/modules/stitching/include/opencv2/stitching/detail/matchers.hpp +++ b/modules/stitching/include/opencv2/stitching/detail/matchers.hpp @@ -59,7 +59,7 @@ struct CV_EXPORTS_W_SIMPLE ImageFeatures { CV_PROP_RW int img_idx; CV_PROP_RW Size img_size; - std::vector keypoints; + CV_PROP_RW std::vector keypoints; CV_PROP_RW UMat descriptors; CV_WRAP std::vector getKeypoints() { return keypoints; }; }; diff --git a/modules/stitching/misc/python/test/test_stitching.py b/modules/stitching/misc/python/test/test_stitching.py index 719f0583f2..0d66182fb8 100644 --- a/modules/stitching/misc/python/test/test_stitching.py +++ b/modules/stitching/misc/python/test/test_stitching.py @@ -28,6 +28,9 @@ class stitching_detail_test(NewOpenCVTests): imgFea = cv.detail.computeImageFeatures2(finder,img) self.assertIsNotNone(imgFea) + # Added Test for PR #21180 + self.assertIsNotNone(imgFea.keypoints) + matcher = cv.detail_BestOf2NearestMatcher(False, 0.3) self.assertIsNotNone(matcher) matcher = cv.detail_AffineBestOf2NearestMatcher(False, False, 0.3) From 973e1acb67fd2aeb198c3ddc674cf32376609c62 Mon Sep 17 00:00:00 2001 From: Maksim Shabunin Date: Sun, 5 Dec 2021 15:49:36 +0300 Subject: [PATCH 134/226] Merge pull request #21182 from mshabunin:split-cv2cpp Split cv2.cpp * split cv2.cpp: util, numpy * split cv2.cpp: convert * split cv2.cpp: highgui, more utils * split cv2.cpp: fix numpy import --- modules/python/common.cmake | 12 +- modules/python/src2/cv2.cpp | 2150 +-------------------------- modules/python/src2/cv2.hpp | 55 + modules/python/src2/cv2_convert.cpp | 1052 +++++++++++++ modules/python/src2/cv2_convert.hpp | 529 +++++++ modules/python/src2/cv2_highgui.cpp | 184 +++ modules/python/src2/cv2_highgui.hpp | 14 + modules/python/src2/cv2_numpy.cpp | 73 + modules/python/src2/cv2_numpy.hpp | 217 +++ modules/python/src2/cv2_util.cpp | 178 +++ modules/python/src2/cv2_util.hpp | 134 ++ modules/python/src2/pycompat.hpp | 2 + 12 files changed, 2458 insertions(+), 2142 deletions(-) create mode 100644 modules/python/src2/cv2.hpp create mode 100644 modules/python/src2/cv2_convert.cpp create mode 100644 modules/python/src2/cv2_convert.hpp create mode 100644 modules/python/src2/cv2_highgui.cpp create mode 100644 modules/python/src2/cv2_highgui.hpp create mode 100644 modules/python/src2/cv2_numpy.cpp create mode 100644 modules/python/src2/cv2_numpy.hpp create mode 100644 modules/python/src2/cv2_util.cpp create mode 100644 modules/python/src2/cv2_util.hpp diff --git a/modules/python/common.cmake b/modules/python/common.cmake index ebbb2e2f65..264714f187 100644 --- a/modules/python/common.cmake +++ b/modules/python/common.cmake @@ -19,7 +19,17 @@ if(NOT WIN32 AND NOT APPLE AND NOT OPENCV_PYTHON_SKIP_LINKER_EXCLUDE_LIBS) set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -Wl,--exclude-libs=ALL") endif() -ocv_add_library(${the_module} MODULE ${PYTHON_SOURCE_DIR}/src2/cv2.cpp ${cv2_generated_hdrs} ${opencv_userdef_hdrs} ${cv2_custom_hdr}) +ocv_add_library(${the_module} MODULE + ${PYTHON_SOURCE_DIR}/src2/cv2.cpp + ${PYTHON_SOURCE_DIR}/src2/cv2_util.cpp + ${PYTHON_SOURCE_DIR}/src2/cv2_numpy.cpp + ${PYTHON_SOURCE_DIR}/src2/cv2_convert.cpp + ${PYTHON_SOURCE_DIR}/src2/cv2_highgui.cpp + ${cv2_generated_hdrs} + ${opencv_userdef_hdrs} + ${cv2_custom_hdr} +) + if(TARGET gen_opencv_python_source) add_dependencies(${the_module} gen_opencv_python_source) endif() diff --git a/modules/python/src2/cv2.cpp b/modules/python/src2/cv2.cpp index 58b1357eff..b39db34fcb 100644 --- a/modules/python/src2/cv2.cpp +++ b/modules/python/src2/cv2.cpp @@ -1,530 +1,24 @@ -//warning number '5033' not a valid compiler warning in vc12 -#if defined(_MSC_VER) && (_MSC_VER > 1800) -// eliminating duplicated round() declaration -#define HAVE_ROUND 1 -#pragma warning(push) -#pragma warning(disable:5033) // 'register' is no longer a supported storage class -#endif +// must be defined before importing numpy headers +// https://numpy.org/doc/1.17/reference/c-api.array.html#importing-the-api +#define PY_ARRAY_UNIQUE_SYMBOL opencv_ARRAY_API -// #define CVPY_DYNAMIC_INIT -// #define Py_DEBUG - -#if defined(CVPY_DYNAMIC_INIT) && !defined(Py_DEBUG) -# define Py_LIMITED_API 0x03030000 -#endif - -#include -#include -#include - -#if PY_MAJOR_VERSION < 3 -#undef CVPY_DYNAMIC_INIT -#else -#define CV_PYTHON_3 1 -#endif - -#if defined(_MSC_VER) && (_MSC_VER > 1800) -#pragma warning(pop) -#endif - -#define MODULESTR "cv2" -#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION - -#include +#include "cv2.hpp" #include "opencv2/opencv_modules.hpp" #include "opencv2/core.hpp" -#include "opencv2/core/utils/configuration.private.hpp" #include "opencv2/core/utils/logger.hpp" -#include "opencv2/core/utils/tls.hpp" #include "pyopencv_generated_include.h" #include "opencv2/core/types_c.h" -#include "pycompat.hpp" -#include -#include // std::enable_if -#define CV_HAS_CONVERSION_ERROR(x) (((x) == -1) && PyErr_Occurred()) - -static PyObject* opencv_error = NULL; - -static PyTypeObject* pyopencv_Mat_TypePtr = nullptr; - -class ArgInfo -{ -public: - const char* name; - bool outputarg; - // more fields may be added if necessary - - ArgInfo(const char* name_, bool outputarg_) : name(name_), outputarg(outputarg_) {} - -private: - ArgInfo(const ArgInfo&) = delete; - ArgInfo& operator=(const ArgInfo&) = delete; -}; - -template // TEnable is used for SFINAE checks -struct PyOpenCV_Converter -{ - //static inline bool to(PyObject* obj, T& p, const ArgInfo& info); - //static inline PyObject* from(const T& src); -}; - -// exception-safe pyopencv_to -template static -bool pyopencv_to_safe(PyObject* obj, _Tp& value, const ArgInfo& info) -{ - try - { - return pyopencv_to(obj, value, info); - } - catch (const std::exception &e) - { - PyErr_SetString(opencv_error, cv::format("Conversion error: %s, what: %s", info.name, e.what()).c_str()); - return false; - } - catch (...) - { - PyErr_SetString(opencv_error, cv::format("Conversion error: %s", info.name).c_str()); - return false; - } -} - -template static -bool pyopencv_to(PyObject* obj, T& p, const ArgInfo& info) { return PyOpenCV_Converter::to(obj, p, info); } - -template static -PyObject* pyopencv_from(const T& src) { return PyOpenCV_Converter::from(src); } - -static bool isPythonBindingsDebugEnabled() -{ - static bool param_debug = cv::utils::getConfigurationParameterBool("OPENCV_PYTHON_DEBUG", false); - return param_debug; -} - -static void emit_failmsg(PyObject * exc, const char *msg) -{ - static bool param_debug = isPythonBindingsDebugEnabled(); - if (param_debug) - { - CV_LOG_WARNING(NULL, "Bindings conversion failed: " << msg); - } - PyErr_SetString(exc, msg); -} - -static int failmsg(const char *fmt, ...) -{ - char str[1000]; - - va_list ap; - va_start(ap, fmt); - vsnprintf(str, sizeof(str), fmt, ap); - va_end(ap); - - emit_failmsg(PyExc_TypeError, str); - return 0; -} - -static PyObject* failmsgp(const char *fmt, ...) -{ - char str[1000]; - - va_list ap; - va_start(ap, fmt); - vsnprintf(str, sizeof(str), fmt, ap); - va_end(ap); - - emit_failmsg(PyExc_TypeError, str); - return 0; -} - -class PyAllowThreads -{ -public: - PyAllowThreads() : _state(PyEval_SaveThread()) {} - ~PyAllowThreads() - { - PyEval_RestoreThread(_state); - } -private: - PyThreadState* _state; -}; - -class PyEnsureGIL -{ -public: - PyEnsureGIL() : _state(PyGILState_Ensure()) {} - ~PyEnsureGIL() - { - PyGILState_Release(_state); - } -private: - PyGILState_STATE _state; -}; - -/** - * Light weight RAII wrapper for `PyObject*` owning references. - * In comparisson to C++11 `std::unique_ptr` with custom deleter, it provides - * implicit conversion functions that might be useful to initialize it with - * Python functions those returns owning references through the `PyObject**` - * e.g. `PyErr_Fetch` or directly pass it to functions those want to borrow - * reference to object (doesn't extend object lifetime) e.g. `PyObject_Str`. - */ -class PySafeObject -{ -public: - PySafeObject() : obj_(NULL) {} - - explicit PySafeObject(PyObject* obj) : obj_(obj) {} - - ~PySafeObject() - { - Py_CLEAR(obj_); - } - - operator PyObject*() - { - return obj_; - } - - operator PyObject**() - { - return &obj_; - } - - PyObject* release() - { - PyObject* obj = obj_; - obj_ = NULL; - return obj; - } - -private: - PyObject* obj_; - - // Explicitly disable copy operations - PySafeObject(const PySafeObject*); // = delete - PySafeObject& operator=(const PySafeObject&); // = delete -}; - -static void pyRaiseCVException(const cv::Exception &e) -{ - PyObject_SetAttrString(opencv_error, "file", PyString_FromString(e.file.c_str())); - PyObject_SetAttrString(opencv_error, "func", PyString_FromString(e.func.c_str())); - PyObject_SetAttrString(opencv_error, "line", PyInt_FromLong(e.line)); - PyObject_SetAttrString(opencv_error, "code", PyInt_FromLong(e.code)); - PyObject_SetAttrString(opencv_error, "msg", PyString_FromString(e.msg.c_str())); - PyObject_SetAttrString(opencv_error, "err", PyString_FromString(e.err.c_str())); - PyErr_SetString(opencv_error, e.what()); -} - -#define ERRWRAP2(expr) \ -try \ -{ \ - PyAllowThreads allowThreads; \ - expr; \ -} \ -catch (const cv::Exception &e) \ -{ \ - pyRaiseCVException(e); \ - return 0; \ -} \ -catch (const std::exception &e) \ -{ \ - PyErr_SetString(opencv_error, e.what()); \ - return 0; \ -} \ -catch (...) \ -{ \ - PyErr_SetString(opencv_error, "Unknown C++ exception from OpenCV code"); \ - return 0; \ -} +#include "cv2_util.hpp" +#include "cv2_numpy.hpp" +#include "cv2_convert.hpp" +#include "cv2_highgui.hpp" using namespace cv; - -namespace { -template -NPY_TYPES asNumpyType() -{ - return NPY_OBJECT; -} - -template<> -NPY_TYPES asNumpyType() -{ - return NPY_BOOL; -} - -#define CV_GENERATE_INTEGRAL_TYPE_NPY_CONVERSION(src, dst) \ - template<> \ - NPY_TYPES asNumpyType() \ - { \ - return NPY_##dst; \ - } \ - template<> \ - NPY_TYPES asNumpyType() \ - { \ - return NPY_U##dst; \ - } - -CV_GENERATE_INTEGRAL_TYPE_NPY_CONVERSION(int8_t, INT8); - -CV_GENERATE_INTEGRAL_TYPE_NPY_CONVERSION(int16_t, INT16); - -CV_GENERATE_INTEGRAL_TYPE_NPY_CONVERSION(int32_t, INT32); - -CV_GENERATE_INTEGRAL_TYPE_NPY_CONVERSION(int64_t, INT64); - -#undef CV_GENERATE_INTEGRAL_TYPE_NPY_CONVERSION - -template<> -NPY_TYPES asNumpyType() -{ - return NPY_FLOAT; -} - -template<> -NPY_TYPES asNumpyType() -{ - return NPY_DOUBLE; -} - -template -PyArray_Descr* getNumpyTypeDescriptor() -{ - return PyArray_DescrFromType(asNumpyType()); -} - -template <> -PyArray_Descr* getNumpyTypeDescriptor() -{ -#if SIZE_MAX == ULONG_MAX - return PyArray_DescrFromType(NPY_ULONG); -#elif SIZE_MAX == ULLONG_MAX - return PyArray_DescrFromType(NPY_ULONGLONG); -#else - return PyArray_DescrFromType(NPY_UINT); -#endif -} - -template -bool isRepresentable(U value) { - return (std::numeric_limits::min() <= value) && (value <= std::numeric_limits::max()); -} - -template -bool canBeSafelyCasted(PyObject* obj, PyArray_Descr* to) -{ - return PyArray_CanCastTo(PyArray_DescrFromScalar(obj), to) != 0; -} - - -template<> -bool canBeSafelyCasted(PyObject* obj, PyArray_Descr* to) -{ - PyArray_Descr* from = PyArray_DescrFromScalar(obj); - if (PyArray_CanCastTo(from, to)) - { - return true; - } - else - { - // False negative scenarios: - // - Signed input is positive so it can be safely cast to unsigned output - // - Input has wider limits but value is representable within output limits - // - All the above - if (PyDataType_ISSIGNED(from)) - { - int64_t input = 0; - PyArray_CastScalarToCtype(obj, &input, getNumpyTypeDescriptor()); - return (input >= 0) && isRepresentable(static_cast(input)); - } - else - { - uint64_t input = 0; - PyArray_CastScalarToCtype(obj, &input, getNumpyTypeDescriptor()); - return isRepresentable(input); - } - return false; - } -} - - -template -bool parseNumpyScalar(PyObject* obj, T& value) -{ - if (PyArray_CheckScalar(obj)) - { - // According to the numpy documentation: - // There are 21 statically-defined PyArray_Descr objects for the built-in data-types - // So descriptor pointer is not owning. - PyArray_Descr* to = getNumpyTypeDescriptor(); - if (canBeSafelyCasted(obj, to)) - { - PyArray_CastScalarToCtype(obj, &value, to); - return true; - } - } - return false; -} - -TLSData > conversionErrorsTLS; - -inline void pyPrepareArgumentConversionErrorsStorage(std::size_t size) -{ - std::vector& conversionErrors = conversionErrorsTLS.getRef(); - conversionErrors.clear(); - conversionErrors.reserve(size); -} - -void pyRaiseCVOverloadException(const std::string& functionName) -{ - const std::vector& conversionErrors = conversionErrorsTLS.getRef(); - const std::size_t conversionErrorsCount = conversionErrors.size(); - if (conversionErrorsCount > 0) - { - // In modern std libraries small string optimization is used = no dynamic memory allocations, - // but it can be applied only for string with length < 18 symbols (in GCC) - const std::string bullet = "\n - "; - - // Estimate required buffer size - save dynamic memory allocations = faster - std::size_t requiredBufferSize = bullet.size() * conversionErrorsCount; - for (std::size_t i = 0; i < conversionErrorsCount; ++i) - { - requiredBufferSize += conversionErrors[i].size(); - } - - // Only string concatenation is required so std::string is way faster than - // std::ostringstream - std::string errorMessage("Overload resolution failed:"); - errorMessage.reserve(errorMessage.size() + requiredBufferSize); - for (std::size_t i = 0; i < conversionErrorsCount; ++i) - { - errorMessage += bullet; - errorMessage += conversionErrors[i]; - } - cv::Exception exception(CV_StsBadArg, errorMessage, functionName, "", -1); - pyRaiseCVException(exception); - } - else - { - cv::Exception exception(CV_StsInternal, "Overload resolution failed, but no errors reported", - functionName, "", -1); - pyRaiseCVException(exception); - } -} - -void pyPopulateArgumentConversionErrors() -{ - if (PyErr_Occurred()) - { - PySafeObject exception_type; - PySafeObject exception_value; - PySafeObject exception_traceback; - PyErr_Fetch(exception_type, exception_value, exception_traceback); - PyErr_NormalizeException(exception_type, exception_value, - exception_traceback); - - PySafeObject exception_message(PyObject_Str(exception_value)); - std::string message; - getUnicodeString(exception_message, message); -#ifdef CV_CXX11 - conversionErrorsTLS.getRef().push_back(std::move(message)); -#else - conversionErrorsTLS.getRef().push_back(message); -#endif - } -} - -struct SafeSeqItem -{ - PyObject * item; - SafeSeqItem(PyObject *obj, size_t idx) { item = PySequence_GetItem(obj, idx); } - ~SafeSeqItem() { Py_XDECREF(item); } - -private: - SafeSeqItem(const SafeSeqItem&); // = delete - SafeSeqItem& operator=(const SafeSeqItem&); // = delete -}; - -template -class RefWrapper -{ -public: - RefWrapper(T& item) : item_(item) {} - - T& get() CV_NOEXCEPT { return item_; } - -private: - T& item_; -}; - -// In order to support this conversion on 3.x branch - use custom reference_wrapper -// and C-style array instead of std::array -template -bool parseSequence(PyObject* obj, RefWrapper (&value)[N], const ArgInfo& info) -{ - if (!obj || obj == Py_None) - { - return true; - } - if (!PySequence_Check(obj)) - { - failmsg("Can't parse '%s'. Input argument doesn't provide sequence " - "protocol", info.name); - return false; - } - const std::size_t sequenceSize = PySequence_Size(obj); - if (sequenceSize != N) - { - failmsg("Can't parse '%s'. Expected sequence length %lu, got %lu", - info.name, N, sequenceSize); - return false; - } - for (std::size_t i = 0; i < N; ++i) - { - SafeSeqItem seqItem(obj, i); - if (!pyopencv_to(seqItem.item, value[i].get(), info)) - { - failmsg("Can't parse '%s'. Sequence item with index %lu has a " - "wrong type", info.name, i); - return false; - } - } - return true; -} -} // namespace - -namespace traits { -template -struct BooleanConstant -{ - static const bool value = Value; - typedef BooleanConstant type; -}; - -typedef BooleanConstant TrueType; -typedef BooleanConstant FalseType; - -template -struct VoidType { - typedef void type; -}; - -template -struct IsRepresentableAsMatDataType : FalseType -{ -}; - -template -struct IsRepresentableAsMatDataType::channel_type>::type> : TrueType -{ -}; -} // namespace traits - typedef std::vector vector_uchar; typedef std::vector vector_char; typedef std::vector vector_int; @@ -559,1634 +53,8 @@ typedef std::vector > vector_vector_Point3f; typedef std::vector > vector_vector_DMatch; typedef std::vector > vector_vector_KeyPoint; -class NumpyAllocator : public MatAllocator -{ -public: - NumpyAllocator() { stdAllocator = Mat::getStdAllocator(); } - ~NumpyAllocator() {} +// enum { ARG_NONE = 0, ARG_MAT = 1, ARG_SCALAR = 2 }; - UMatData* allocate(PyObject* o, int dims, const int* sizes, int type, size_t* step) const - { - UMatData* u = new UMatData(this); - u->data = u->origdata = (uchar*)PyArray_DATA((PyArrayObject*) o); - npy_intp* _strides = PyArray_STRIDES((PyArrayObject*) o); - for( int i = 0; i < dims - 1; i++ ) - step[i] = (size_t)_strides[i]; - step[dims-1] = CV_ELEM_SIZE(type); - u->size = sizes[0]*step[0]; - u->userdata = o; - return u; - } - - UMatData* allocate(int dims0, const int* sizes, int type, void* data, size_t* step, AccessFlag flags, UMatUsageFlags usageFlags) const CV_OVERRIDE - { - if( data != 0 ) - { - // issue #6969: CV_Error(Error::StsAssert, "The data should normally be NULL!"); - // probably this is safe to do in such extreme case - return stdAllocator->allocate(dims0, sizes, type, data, step, flags, usageFlags); - } - PyEnsureGIL gil; - - int depth = CV_MAT_DEPTH(type); - int cn = CV_MAT_CN(type); - const int f = (int)(sizeof(size_t)/8); - int typenum = depth == CV_8U ? NPY_UBYTE : depth == CV_8S ? NPY_BYTE : - depth == CV_16U ? NPY_USHORT : depth == CV_16S ? NPY_SHORT : - depth == CV_32S ? NPY_INT : depth == CV_32F ? NPY_FLOAT : - depth == CV_64F ? NPY_DOUBLE : f*NPY_ULONGLONG + (f^1)*NPY_UINT; - int i, dims = dims0; - cv::AutoBuffer _sizes(dims + 1); - for( i = 0; i < dims; i++ ) - _sizes[i] = sizes[i]; - if( cn > 1 ) - _sizes[dims++] = cn; - PyObject* o = PyArray_SimpleNew(dims, _sizes.data(), typenum); - if(!o) - CV_Error_(Error::StsError, ("The numpy array of typenum=%d, ndims=%d can not be created", typenum, dims)); - return allocate(o, dims0, sizes, type, step); - } - - bool allocate(UMatData* u, AccessFlag accessFlags, UMatUsageFlags usageFlags) const CV_OVERRIDE - { - return stdAllocator->allocate(u, accessFlags, usageFlags); - } - - void deallocate(UMatData* u) const CV_OVERRIDE - { - if(!u) - return; - PyEnsureGIL gil; - CV_Assert(u->urefcount >= 0); - CV_Assert(u->refcount >= 0); - if(u->refcount == 0) - { - PyObject* o = (PyObject*)u->userdata; - Py_XDECREF(o); - delete u; - } - } - - const MatAllocator* stdAllocator; -}; - -NumpyAllocator g_numpyAllocator; - - -enum { ARG_NONE = 0, ARG_MAT = 1, ARG_SCALAR = 2 }; - -static bool isBool(PyObject* obj) CV_NOEXCEPT -{ - return PyArray_IsScalar(obj, Bool) || PyBool_Check(obj); -} - -template -static std::string pycv_dumpArray(const T* arr, int n) -{ - std::ostringstream out; - out << "["; - for (int i = 0; i < n; ++i) - out << " " << arr[i]; - out << " ]"; - return out.str(); -} - -// special case, when the converter needs full ArgInfo structure -static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) -{ - if(!o || o == Py_None) - { - if( !m.data ) - m.allocator = &g_numpyAllocator; - return true; - } - - if( PyInt_Check(o) ) - { - double v[] = {static_cast(PyInt_AsLong((PyObject*)o)), 0., 0., 0.}; - m = Mat(4, 1, CV_64F, v).clone(); - return true; - } - if( PyFloat_Check(o) ) - { - double v[] = {PyFloat_AsDouble((PyObject*)o), 0., 0., 0.}; - m = Mat(4, 1, CV_64F, v).clone(); - return true; - } - if( PyTuple_Check(o) ) - { - int i, sz = (int)PyTuple_Size((PyObject*)o); - m = Mat(sz, 1, CV_64F); - for( i = 0; i < sz; i++ ) - { - PyObject* oi = PyTuple_GetItem(o, i); - if( PyInt_Check(oi) ) - m.at(i) = (double)PyInt_AsLong(oi); - else if( PyFloat_Check(oi) ) - m.at(i) = (double)PyFloat_AsDouble(oi); - else - { - failmsg("%s is not a numerical tuple", info.name); - m.release(); - return false; - } - } - return true; - } - - if( !PyArray_Check(o) ) - { - failmsg("%s is not a numpy array, neither a scalar", info.name); - return false; - } - - PyArrayObject* oarr = (PyArrayObject*) o; - - bool needcopy = false, needcast = false; - int typenum = PyArray_TYPE(oarr), new_typenum = typenum; - int type = typenum == NPY_UBYTE ? CV_8U : - typenum == NPY_BYTE ? CV_8S : - typenum == NPY_USHORT ? CV_16U : - typenum == NPY_SHORT ? CV_16S : - typenum == NPY_INT ? CV_32S : - typenum == NPY_INT32 ? CV_32S : - typenum == NPY_FLOAT ? CV_32F : - typenum == NPY_DOUBLE ? CV_64F : -1; - - if( type < 0 ) - { - if( typenum == NPY_INT64 || typenum == NPY_UINT64 || typenum == NPY_LONG ) - { - needcopy = needcast = true; - new_typenum = NPY_INT; - type = CV_32S; - } - else - { - failmsg("%s data type = %d is not supported", info.name, typenum); - return false; - } - } - -#ifndef CV_MAX_DIM - const int CV_MAX_DIM = 32; -#endif - - int ndims = PyArray_NDIM(oarr); - if(ndims >= CV_MAX_DIM) - { - failmsg("%s dimensionality (=%d) is too high", info.name, ndims); - return false; - } - - size_t elemsize = CV_ELEM_SIZE1(type); - const npy_intp* _sizes = PyArray_DIMS(oarr); - const npy_intp* _strides = PyArray_STRIDES(oarr); - - CV_LOG_DEBUG(NULL, "Incoming ndarray '" << info.name << "': ndims=" << ndims << " _sizes=" << pycv_dumpArray(_sizes, ndims) << " _strides=" << pycv_dumpArray(_strides, ndims)); - - bool ismultichannel = ndims == 3 && _sizes[2] <= CV_CN_MAX; - if (pyopencv_Mat_TypePtr && PyObject_TypeCheck(o, pyopencv_Mat_TypePtr)) - { - bool wrapChannels = false; - PyObject* pyobj_wrap_channels = PyObject_GetAttrString(o, "wrap_channels"); - if (pyobj_wrap_channels) - { - if (!pyopencv_to_safe(pyobj_wrap_channels, wrapChannels, ArgInfo("cv.Mat.wrap_channels", 0))) - { - // TODO extra message - Py_DECREF(pyobj_wrap_channels); - return false; - } - Py_DECREF(pyobj_wrap_channels); - } - ismultichannel = wrapChannels && ndims >= 1; - } - - for( int i = ndims-1; i >= 0 && !needcopy; i-- ) - { - // these checks handle cases of - // a) multi-dimensional (ndims > 2) arrays, as well as simpler 1- and 2-dimensional cases - // b) transposed arrays, where _strides[] elements go in non-descending order - // c) flipped arrays, where some of _strides[] elements are negative - // the _sizes[i] > 1 is needed to avoid spurious copies when NPY_RELAXED_STRIDES is set - if( (i == ndims-1 && _sizes[i] > 1 && (size_t)_strides[i] != elemsize) || - (i < ndims-1 && _sizes[i] > 1 && _strides[i] < _strides[i+1]) ) - needcopy = true; - } - - if (ismultichannel) - { - int channels = ndims >= 1 ? (int)_sizes[ndims - 1] : 1; - if (channels > CV_CN_MAX) - { - failmsg("%s unable to wrap channels, too high (%d > CV_CN_MAX=%d)", info.name, (int)channels, (int)CV_CN_MAX); - return false; - } - ndims--; - type |= CV_MAKETYPE(0, channels); - - if (ndims >= 1 && _strides[ndims - 1] != (npy_intp)elemsize*_sizes[ndims]) - needcopy = true; - - elemsize = CV_ELEM_SIZE(type); - } - - if (needcopy) - { - if (info.outputarg) - { - failmsg("Layout of the output array %s is incompatible with cv::Mat", info.name); - return false; - } - - if( needcast ) { - o = PyArray_Cast(oarr, new_typenum); - oarr = (PyArrayObject*) o; - } - else { - oarr = PyArray_GETCONTIGUOUS(oarr); - o = (PyObject*) oarr; - } - - _strides = PyArray_STRIDES(oarr); - } - - int size[CV_MAX_DIM+1] = {}; - size_t step[CV_MAX_DIM+1] = {}; - - // Normalize strides in case NPY_RELAXED_STRIDES is set - size_t default_step = elemsize; - for ( int i = ndims - 1; i >= 0; --i ) - { - size[i] = (int)_sizes[i]; - if ( size[i] > 1 ) - { - step[i] = (size_t)_strides[i]; - default_step = step[i] * size[i]; - } - else - { - step[i] = default_step; - default_step *= size[i]; - } - } - - // handle degenerate case - // FIXIT: Don't force 1D for Scalars - if( ndims == 0) { - size[ndims] = 1; - step[ndims] = elemsize; - ndims++; - } - -#if 1 - CV_LOG_DEBUG(NULL, "Construct Mat: ndims=" << ndims << " size=" << pycv_dumpArray(size, ndims) << " step=" << pycv_dumpArray(step, ndims) << " type=" << cv::typeToString(type)); -#endif - - m = Mat(ndims, size, type, PyArray_DATA(oarr), step); - m.u = g_numpyAllocator.allocate(o, ndims, size, type, step); - m.addref(); - - if( !needcopy ) - { - Py_INCREF(o); - } - m.allocator = &g_numpyAllocator; - - return true; -} - -template -bool pyopencv_to(PyObject* o, Matx<_Tp, m, n>& mx, const ArgInfo& info) -{ - Mat tmp; - if (!pyopencv_to(o, tmp, info)) { - return false; - } - - tmp.copyTo(mx); - return true; -} - -template -bool pyopencv_to(PyObject* o, Vec<_Tp, cn>& vec, const ArgInfo& info) -{ - return pyopencv_to(o, (Matx<_Tp, cn, 1>&)vec, info); -} - -template<> -PyObject* pyopencv_from(const Mat& m) -{ - if( !m.data ) - Py_RETURN_NONE; - Mat temp, *p = (Mat*)&m; - if(!p->u || p->allocator != &g_numpyAllocator) - { - temp.allocator = &g_numpyAllocator; - ERRWRAP2(m.copyTo(temp)); - p = &temp; - } - PyObject* o = (PyObject*)p->u->userdata; - Py_INCREF(o); - return o; -} - -template -PyObject* pyopencv_from(const Matx<_Tp, m, n>& matx) -{ - return pyopencv_from(Mat(matx)); -} - -template -struct PyOpenCV_Converter< cv::Ptr > -{ - static PyObject* from(const cv::Ptr& p) - { - if (!p) - Py_RETURN_NONE; - return pyopencv_from(*p); - } - static bool to(PyObject *o, Ptr& p, const ArgInfo& info) - { - if (!o || o == Py_None) - return true; - p = makePtr(); - return pyopencv_to(o, *p, info); - } -}; - -template<> -bool pyopencv_to(PyObject* obj, void*& ptr, const ArgInfo& info) -{ - CV_UNUSED(info); - if (!obj || obj == Py_None) - return true; - - if (!PyLong_Check(obj)) - return false; - ptr = PyLong_AsVoidPtr(obj); - return ptr != NULL && !PyErr_Occurred(); -} - -static PyObject* pyopencv_from(void*& ptr) -{ - return PyLong_FromVoidPtr(ptr); -} - -static bool pyopencv_to(PyObject *o, Scalar& s, const ArgInfo& info) -{ - if(!o || o == Py_None) - return true; - if (PySequence_Check(o)) { - if (4 < PySequence_Size(o)) - { - failmsg("Scalar value for argument '%s' is longer than 4", info.name); - return false; - } - for (Py_ssize_t i = 0; i < PySequence_Size(o); i++) { - SafeSeqItem item_wrap(o, i); - PyObject *item = item_wrap.item; - if (PyFloat_Check(item) || PyInt_Check(item)) { - s[(int)i] = PyFloat_AsDouble(item); - } else { - failmsg("Scalar value for argument '%s' is not numeric", info.name); - return false; - } - } - } else { - if (PyFloat_Check(o) || PyInt_Check(o)) { - s[0] = PyFloat_AsDouble(o); - } else { - failmsg("Scalar value for argument '%s' is not numeric", info.name); - return false; - } - } - return true; -} - -template<> -PyObject* pyopencv_from(const Scalar& src) -{ - return Py_BuildValue("(dddd)", src[0], src[1], src[2], src[3]); -} - -template<> -PyObject* pyopencv_from(const bool& value) -{ - return PyBool_FromLong(value); -} - -template<> -bool pyopencv_to(PyObject* obj, bool& value, const ArgInfo& info) -{ - if (!obj || obj == Py_None) - { - return true; - } - if (isBool(obj) || PyArray_IsIntegerScalar(obj)) - { - npy_bool npy_value = NPY_FALSE; - const int ret_code = PyArray_BoolConverter(obj, &npy_value); - if (ret_code >= 0) - { - value = (npy_value == NPY_TRUE); - return true; - } - } - failmsg("Argument '%s' is not convertable to bool", info.name); - return false; -} - -template<> -PyObject* pyopencv_from(const size_t& value) -{ - return PyLong_FromSize_t(value); -} - -template<> -bool pyopencv_to(PyObject* obj, size_t& value, const ArgInfo& info) -{ - if (!obj || obj == Py_None) - { - return true; - } - if (isBool(obj)) - { - failmsg("Argument '%s' must be integer type, not bool", info.name); - return false; - } - if (PyArray_IsIntegerScalar(obj)) - { - if (PyLong_Check(obj)) - { -#if defined(CV_PYTHON_3) - value = PyLong_AsSize_t(obj); -#else - #if ULONG_MAX == SIZE_MAX - value = PyLong_AsUnsignedLong(obj); - #else - value = PyLong_AsUnsignedLongLong(obj); - #endif -#endif - } -#if !defined(CV_PYTHON_3) - // Python 2.x has PyIntObject which is not a subtype of PyLongObject - // Overflow check here is unnecessary because object will be converted to long on the - // interpreter side - else if (PyInt_Check(obj)) - { - const long res = PyInt_AsLong(obj); - if (res < 0) { - failmsg("Argument '%s' can not be safely parsed to 'size_t'", info.name); - return false; - } - #if ULONG_MAX == SIZE_MAX - value = PyInt_AsUnsignedLongMask(obj); - #else - value = PyInt_AsUnsignedLongLongMask(obj); - #endif - } -#endif - else - { - const bool isParsed = parseNumpyScalar(obj, value); - if (!isParsed) { - failmsg("Argument '%s' can not be safely parsed to 'size_t'", info.name); - return false; - } - } - } - else - { - failmsg("Argument '%s' is required to be an integer", info.name); - return false; - } - return !PyErr_Occurred(); -} - -template<> -PyObject* pyopencv_from(const int& value) -{ - return PyInt_FromLong(value); -} - -template<> -bool pyopencv_to(PyObject* obj, int& value, const ArgInfo& info) -{ - if (!obj || obj == Py_None) - { - return true; - } - if (isBool(obj)) - { - failmsg("Argument '%s' must be integer, not bool", info.name); - return false; - } - if (PyArray_IsIntegerScalar(obj)) - { - value = PyArray_PyIntAsInt(obj); - } - else - { - failmsg("Argument '%s' is required to be an integer", info.name); - return false; - } - return !CV_HAS_CONVERSION_ERROR(value); -} - -// There is conflict between "size_t" and "unsigned int". -// They are the same type on some 32-bit platforms. -template -struct PyOpenCV_Converter - < T, typename std::enable_if< std::is_same::value && !std::is_same::value >::type > -{ - static inline PyObject* from(const unsigned int& value) - { - return PyLong_FromUnsignedLong(value); - } - - static inline bool to(PyObject* obj, unsigned int& value, const ArgInfo& info) - { - CV_UNUSED(info); - if(!obj || obj == Py_None) - return true; - if(PyInt_Check(obj)) - value = (unsigned int)PyInt_AsLong(obj); - else if(PyLong_Check(obj)) - value = (unsigned int)PyLong_AsLong(obj); - else - return false; - return value != (unsigned int)-1 || !PyErr_Occurred(); - } -}; - -template<> -PyObject* pyopencv_from(const uchar& value) -{ - return PyInt_FromLong(value); -} - -template<> -bool pyopencv_to(PyObject* obj, uchar& value, const ArgInfo& info) -{ - CV_UNUSED(info); - if(!obj || obj == Py_None) - return true; - int ivalue = (int)PyInt_AsLong(obj); - value = cv::saturate_cast(ivalue); - return ivalue != -1 || !PyErr_Occurred(); -} - -template<> -bool pyopencv_to(PyObject* obj, char& value, const ArgInfo& info) -{ - if (!obj || obj == Py_None) - { - return true; - } - if (isBool(obj)) - { - failmsg("Argument '%s' must be an integer, not bool", info.name); - return false; - } - if (PyArray_IsIntegerScalar(obj)) - { - value = saturate_cast(PyArray_PyIntAsInt(obj)); - } - else - { - failmsg("Argument '%s' is required to be an integer", info.name); - return false; - } - return !CV_HAS_CONVERSION_ERROR(value); -} - -template<> -PyObject* pyopencv_from(const double& value) -{ - return PyFloat_FromDouble(value); -} - -template<> -bool pyopencv_to(PyObject* obj, double& value, const ArgInfo& info) -{ - if (!obj || obj == Py_None) - { - return true; - } - if (isBool(obj)) - { - failmsg("Argument '%s' must be double, not bool", info.name); - return false; - } - if (PyArray_IsPythonNumber(obj)) - { - if (PyLong_Check(obj)) - { - value = PyLong_AsDouble(obj); - } - else - { - value = PyFloat_AsDouble(obj); - } - } - else if (PyArray_CheckScalar(obj)) - { - const bool isParsed = parseNumpyScalar(obj, value); - if (!isParsed) { - failmsg("Argument '%s' can not be safely parsed to 'double'", info.name); - return false; - } - } - else - { - failmsg("Argument '%s' can not be treated as a double", info.name); - return false; - } - return !PyErr_Occurred(); -} - -template<> -PyObject* pyopencv_from(const float& value) -{ - return PyFloat_FromDouble(value); -} - -template<> -bool pyopencv_to(PyObject* obj, float& value, const ArgInfo& info) -{ - if (!obj || obj == Py_None) - { - return true; - } - if (isBool(obj)) - { - failmsg("Argument '%s' must be float, not bool", info.name); - return false; - } - if (PyArray_IsPythonNumber(obj)) - { - if (PyLong_Check(obj)) - { - double res = PyLong_AsDouble(obj); - value = static_cast(res); - } - else - { - double res = PyFloat_AsDouble(obj); - value = static_cast(res); - } - } - else if (PyArray_CheckScalar(obj)) - { - const bool isParsed = parseNumpyScalar(obj, value); - if (!isParsed) { - failmsg("Argument '%s' can not be safely parsed to 'float'", info.name); - return false; - } - } - else - { - failmsg("Argument '%s' can't be treated as a float", info.name); - return false; - } - return !PyErr_Occurred(); -} - -template<> -PyObject* pyopencv_from(const int64& value) -{ - return PyLong_FromLongLong(value); -} - -template<> -PyObject* pyopencv_from(const String& value) -{ - return PyString_FromString(value.empty() ? "" : value.c_str()); -} - -#if CV_VERSION_MAJOR == 3 -template<> -PyObject* pyopencv_from(const std::string& value) -{ - return PyString_FromString(value.empty() ? "" : value.c_str()); -} -#endif - -template<> -bool pyopencv_to(PyObject* obj, String &value, const ArgInfo& info) -{ - if(!obj || obj == Py_None) - { - return true; - } - std::string str; - if (getUnicodeString(obj, str)) - { - value = str; - return true; - } - else - { - // If error hasn't been already set by Python conversion functions - if (!PyErr_Occurred()) - { - // Direct access to underlying slots of PyObjectType is not allowed - // when limited API is enabled -#ifdef Py_LIMITED_API - failmsg("Can't convert object to 'str' for '%s'", info.name); -#else - failmsg("Can't convert object of type '%s' to 'str' for '%s'", - obj->ob_type->tp_name, info.name); -#endif - } - } - return false; -} - -template<> -bool pyopencv_to(PyObject* obj, Size& sz, const ArgInfo& info) -{ - RefWrapper values[] = {RefWrapper(sz.width), - RefWrapper(sz.height)}; - return parseSequence(obj, values, info); -} - -template<> -PyObject* pyopencv_from(const Size& sz) -{ - return Py_BuildValue("(ii)", sz.width, sz.height); -} - -template<> -bool pyopencv_to(PyObject* obj, Size_& sz, const ArgInfo& info) -{ - RefWrapper values[] = {RefWrapper(sz.width), - RefWrapper(sz.height)}; - return parseSequence(obj, values, info); -} - -template<> -PyObject* pyopencv_from(const Size_& sz) -{ - return Py_BuildValue("(ff)", sz.width, sz.height); -} - -template<> -bool pyopencv_to(PyObject* obj, Rect& r, const ArgInfo& info) -{ - RefWrapper values[] = {RefWrapper(r.x), RefWrapper(r.y), - RefWrapper(r.width), - RefWrapper(r.height)}; - return parseSequence(obj, values, info); -} - -template<> -PyObject* pyopencv_from(const Rect& r) -{ - return Py_BuildValue("(iiii)", r.x, r.y, r.width, r.height); -} - -template<> -bool pyopencv_to(PyObject* obj, Rect2d& r, const ArgInfo& info) -{ - RefWrapper values[] = { - RefWrapper(r.x), RefWrapper(r.y), - RefWrapper(r.width), RefWrapper(r.height)}; - return parseSequence(obj, values, info); -} - -template<> -PyObject* pyopencv_from(const Rect2d& r) -{ - return Py_BuildValue("(dddd)", r.x, r.y, r.width, r.height); -} - -template<> -bool pyopencv_to(PyObject* obj, Range& r, const ArgInfo& info) -{ - if (!obj || obj == Py_None) - { - return true; - } - if (PyObject_Size(obj) == 0) - { - r = Range::all(); - return true; - } - RefWrapper values[] = {RefWrapper(r.start), RefWrapper(r.end)}; - return parseSequence(obj, values, info); -} - -template<> -PyObject* pyopencv_from(const Range& r) -{ - return Py_BuildValue("(ii)", r.start, r.end); -} - -template<> -bool pyopencv_to(PyObject* obj, Point& p, const ArgInfo& info) -{ - RefWrapper values[] = {RefWrapper(p.x), RefWrapper(p.y)}; - return parseSequence(obj, values, info); -} - -template <> -bool pyopencv_to(PyObject* obj, Point2f& p, const ArgInfo& info) -{ - RefWrapper values[] = {RefWrapper(p.x), - RefWrapper(p.y)}; - return parseSequence(obj, values, info); -} - -template<> -bool pyopencv_to(PyObject* obj, Point2d& p, const ArgInfo& info) -{ - RefWrapper values[] = {RefWrapper(p.x), - RefWrapper(p.y)}; - return parseSequence(obj, values, info); -} - -template<> -bool pyopencv_to(PyObject* obj, Point3f& p, const ArgInfo& info) -{ - RefWrapper values[] = {RefWrapper(p.x), - RefWrapper(p.y), - RefWrapper(p.z)}; - return parseSequence(obj, values, info); -} - -template<> -bool pyopencv_to(PyObject* obj, Point3d& p, const ArgInfo& info) -{ - RefWrapper values[] = {RefWrapper(p.x), - RefWrapper(p.y), - RefWrapper(p.z)}; - return parseSequence(obj, values, info); -} - -template<> -PyObject* pyopencv_from(const Point& p) -{ - return Py_BuildValue("(ii)", p.x, p.y); -} - -template<> -PyObject* pyopencv_from(const Point2f& p) -{ - return Py_BuildValue("(dd)", p.x, p.y); -} - -template<> -PyObject* pyopencv_from(const Point3f& p) -{ - return Py_BuildValue("(ddd)", p.x, p.y, p.z); -} - -static bool pyopencv_to(PyObject* obj, Vec4d& v, ArgInfo& info) -{ - RefWrapper values[] = {RefWrapper(v[0]), RefWrapper(v[1]), - RefWrapper(v[2]), RefWrapper(v[3])}; - return parseSequence(obj, values, info); -} - -static bool pyopencv_to(PyObject* obj, Vec4f& v, ArgInfo& info) -{ - RefWrapper values[] = {RefWrapper(v[0]), RefWrapper(v[1]), - RefWrapper(v[2]), RefWrapper(v[3])}; - return parseSequence(obj, values, info); -} - -static bool pyopencv_to(PyObject* obj, Vec4i& v, ArgInfo& info) -{ - RefWrapper values[] = {RefWrapper(v[0]), RefWrapper(v[1]), - RefWrapper(v[2]), RefWrapper(v[3])}; - return parseSequence(obj, values, info); -} - -static bool pyopencv_to(PyObject* obj, Vec3d& v, ArgInfo& info) -{ - RefWrapper values[] = {RefWrapper(v[0]), - RefWrapper(v[1]), - RefWrapper(v[2])}; - return parseSequence(obj, values, info); -} - -static bool pyopencv_to(PyObject* obj, Vec3f& v, ArgInfo& info) -{ - RefWrapper values[] = {RefWrapper(v[0]), - RefWrapper(v[1]), - RefWrapper(v[2])}; - return parseSequence(obj, values, info); -} - -static bool pyopencv_to(PyObject* obj, Vec3i& v, ArgInfo& info) -{ - RefWrapper values[] = {RefWrapper(v[0]), RefWrapper(v[1]), - RefWrapper(v[2])}; - return parseSequence(obj, values, info); -} - -static bool pyopencv_to(PyObject* obj, Vec2d& v, ArgInfo& info) -{ - RefWrapper values[] = {RefWrapper(v[0]), - RefWrapper(v[1])}; - return parseSequence(obj, values, info); -} - -static bool pyopencv_to(PyObject* obj, Vec2f& v, ArgInfo& info) -{ - RefWrapper values[] = {RefWrapper(v[0]), - RefWrapper(v[1])}; - return parseSequence(obj, values, info); -} - -static bool pyopencv_to(PyObject* obj, Vec2i& v, ArgInfo& info) -{ - RefWrapper values[] = {RefWrapper(v[0]), RefWrapper(v[1])}; - return parseSequence(obj, values, info); -} - -template<> -PyObject* pyopencv_from(const Vec4d& v) -{ - return Py_BuildValue("(dddd)", v[0], v[1], v[2], v[3]); -} - -template<> -PyObject* pyopencv_from(const Vec4f& v) -{ - return Py_BuildValue("(ffff)", v[0], v[1], v[2], v[3]); -} - -template<> -PyObject* pyopencv_from(const Vec4i& v) -{ - return Py_BuildValue("(iiii)", v[0], v[1], v[2], v[3]); -} - -template<> -PyObject* pyopencv_from(const Vec3d& v) -{ - return Py_BuildValue("(ddd)", v[0], v[1], v[2]); -} - -template<> -PyObject* pyopencv_from(const Vec3f& v) -{ - return Py_BuildValue("(fff)", v[0], v[1], v[2]); -} - -template<> -PyObject* pyopencv_from(const Vec3i& v) -{ - return Py_BuildValue("(iii)", v[0], v[1], v[2]); -} - -template<> -PyObject* pyopencv_from(const Vec2d& v) -{ - return Py_BuildValue("(dd)", v[0], v[1]); -} - -template<> -PyObject* pyopencv_from(const Vec2f& v) -{ - return Py_BuildValue("(ff)", v[0], v[1]); -} - -template<> -PyObject* pyopencv_from(const Vec2i& v) -{ - return Py_BuildValue("(ii)", v[0], v[1]); -} - -template<> -PyObject* pyopencv_from(const Point2d& p) -{ - return Py_BuildValue("(dd)", p.x, p.y); -} - -template<> -PyObject* pyopencv_from(const Point3d& p) -{ - return Py_BuildValue("(ddd)", p.x, p.y, p.z); -} - -template<> -PyObject* pyopencv_from(const std::pair& src) -{ - return Py_BuildValue("(id)", src.first, src.second); -} - -template<> -bool pyopencv_to(PyObject* obj, TermCriteria& dst, const ArgInfo& info) -{ - if (!obj || obj == Py_None) - { - return true; - } - if (!PySequence_Check(obj)) - { - failmsg("Can't parse '%s' as TermCriteria." - "Input argument doesn't provide sequence protocol", - info.name); - return false; - } - const std::size_t sequenceSize = PySequence_Size(obj); - if (sequenceSize != 3) { - failmsg("Can't parse '%s' as TermCriteria. Expected sequence length 3, " - "got %lu", - info.name, sequenceSize); - return false; - } - { - const String typeItemName = format("'%s' criteria type", info.name); - const ArgInfo typeItemInfo(typeItemName.c_str(), false); - SafeSeqItem typeItem(obj, 0); - if (!pyopencv_to(typeItem.item, dst.type, typeItemInfo)) - { - return false; - } - } - { - const String maxCountItemName = format("'%s' max count", info.name); - const ArgInfo maxCountItemInfo(maxCountItemName.c_str(), false); - SafeSeqItem maxCountItem(obj, 1); - if (!pyopencv_to(maxCountItem.item, dst.maxCount, maxCountItemInfo)) - { - return false; - } - } - { - const String epsilonItemName = format("'%s' epsilon", info.name); - const ArgInfo epsilonItemInfo(epsilonItemName.c_str(), false); - SafeSeqItem epsilonItem(obj, 2); - if (!pyopencv_to(epsilonItem.item, dst.epsilon, epsilonItemInfo)) - { - return false; - } - } - return true; -} - -template<> -PyObject* pyopencv_from(const TermCriteria& src) -{ - return Py_BuildValue("(iid)", src.type, src.maxCount, src.epsilon); -} - -template<> -bool pyopencv_to(PyObject* obj, RotatedRect& dst, const ArgInfo& info) -{ - if (!obj || obj == Py_None) - { - return true; - } - if (!PySequence_Check(obj)) - { - failmsg("Can't parse '%s' as RotatedRect." - "Input argument doesn't provide sequence protocol", - info.name); - return false; - } - const std::size_t sequenceSize = PySequence_Size(obj); - if (sequenceSize != 3) - { - failmsg("Can't parse '%s' as RotatedRect. Expected sequence length 3, got %lu", - info.name, sequenceSize); - return false; - } - { - const String centerItemName = format("'%s' center point", info.name); - const ArgInfo centerItemInfo(centerItemName.c_str(), false); - SafeSeqItem centerItem(obj, 0); - if (!pyopencv_to(centerItem.item, dst.center, centerItemInfo)) - { - return false; - } - } - { - const String sizeItemName = format("'%s' size", info.name); - const ArgInfo sizeItemInfo(sizeItemName.c_str(), false); - SafeSeqItem sizeItem(obj, 1); - if (!pyopencv_to(sizeItem.item, dst.size, sizeItemInfo)) - { - return false; - } - } - { - const String angleItemName = format("'%s' angle", info.name); - const ArgInfo angleItemInfo(angleItemName.c_str(), false); - SafeSeqItem angleItem(obj, 2); - if (!pyopencv_to(angleItem.item, dst.angle, angleItemInfo)) - { - return false; - } - } - return true; -} - -template<> -PyObject* pyopencv_from(const RotatedRect& src) -{ - return Py_BuildValue("((ff)(ff)f)", src.center.x, src.center.y, src.size.width, src.size.height, src.angle); -} - -template<> -PyObject* pyopencv_from(const Moments& m) -{ - return Py_BuildValue("{s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d}", - "m00", m.m00, "m10", m.m10, "m01", m.m01, - "m20", m.m20, "m11", m.m11, "m02", m.m02, - "m30", m.m30, "m21", m.m21, "m12", m.m12, "m03", m.m03, - "mu20", m.mu20, "mu11", m.mu11, "mu02", m.mu02, - "mu30", m.mu30, "mu21", m.mu21, "mu12", m.mu12, "mu03", m.mu03, - "nu20", m.nu20, "nu11", m.nu11, "nu02", m.nu02, - "nu30", m.nu30, "nu21", m.nu21, "nu12", m.nu12, "nu03", m.nu03); -} - -template -struct pyopencvVecConverter; - -template -bool pyopencv_to(PyObject* obj, std::vector& value, const ArgInfo& info) -{ - if (!obj || obj == Py_None) - { - return true; - } - return pyopencvVecConverter::to(obj, value, info); -} - -template -PyObject* pyopencv_from(const std::vector& value) -{ - return pyopencvVecConverter::from(value); -} - -template -static bool pyopencv_to_generic_vec(PyObject* obj, std::vector& value, const ArgInfo& info) -{ - if (!obj || obj == Py_None) - { - return true; - } - if (!PySequence_Check(obj)) - { - failmsg("Can't parse '%s'. Input argument doesn't provide sequence protocol", info.name); - return false; - } - const size_t n = static_cast(PySequence_Size(obj)); - value.resize(n); - for (size_t i = 0; i < n; i++) - { - SafeSeqItem item_wrap(obj, i); - if (!pyopencv_to(item_wrap.item, value[i], info)) - { - failmsg("Can't parse '%s'. Sequence item with index %lu has a wrong type", info.name, i); - return false; - } - } - return true; -} - -template<> inline bool pyopencv_to_generic_vec(PyObject* obj, std::vector& value, const ArgInfo& info) -{ - if (!obj || obj == Py_None) - { - return true; - } - if (!PySequence_Check(obj)) - { - failmsg("Can't parse '%s'. Input argument doesn't provide sequence protocol", info.name); - return false; - } - const size_t n = static_cast(PySequence_Size(obj)); - value.resize(n); - for (size_t i = 0; i < n; i++) - { - SafeSeqItem item_wrap(obj, i); - bool elem{}; - if (!pyopencv_to(item_wrap.item, elem, info)) - { - failmsg("Can't parse '%s'. Sequence item with index %lu has a wrong type", info.name, i); - return false; - } - value[i] = elem; - } - return true; -} - - -template -static PyObject* pyopencv_from_generic_vec(const std::vector& value) -{ - Py_ssize_t n = static_cast(value.size()); - PySafeObject seq(PyTuple_New(n)); - for (Py_ssize_t i = 0; i < n; i++) - { - PyObject* item = pyopencv_from(value[i]); - // If item can't be assigned - PyTuple_SetItem raises exception and returns -1. - if (!item || PyTuple_SetItem(seq, i, item) == -1) - { - return NULL; - } - } - return seq.release(); -} - -template<> inline PyObject* pyopencv_from_generic_vec(const std::vector& value) -{ - Py_ssize_t n = static_cast(value.size()); - PySafeObject seq(PyTuple_New(n)); - for (Py_ssize_t i = 0; i < n; i++) - { - bool elem = value[i]; - PyObject* item = pyopencv_from(elem); - // If item can't be assigned - PyTuple_SetItem raises exception and returns -1. - if (!item || PyTuple_SetItem(seq, i, item) == -1) - { - return NULL; - } - } - return seq.release(); -} - - -template -inline typename std::enable_if::type -convert_to_python_tuple(const std::tuple&, PyObject*) { } - -template -inline typename std::enable_if::type -convert_to_python_tuple(const std::tuple& cpp_tuple, PyObject* py_tuple) -{ - PyObject* item = pyopencv_from(std::get(cpp_tuple)); - - if (!item) - return; - - PyTuple_SetItem(py_tuple, I, item); - convert_to_python_tuple(cpp_tuple, py_tuple); -} - - -template -PyObject* pyopencv_from(const std::tuple& cpp_tuple) -{ - size_t size = sizeof...(Ts); - PyObject* py_tuple = PyTuple_New(size); - convert_to_python_tuple(cpp_tuple, py_tuple); - size_t actual_size = PyTuple_Size(py_tuple); - - if (actual_size < size) - { - Py_DECREF(py_tuple); - return NULL; - } - - return py_tuple; -} - -template -struct pyopencvVecConverter -{ - typedef typename std::vector::iterator VecIt; - - static bool to(PyObject* obj, std::vector& value, const ArgInfo& info) - { - if (!PyArray_Check(obj)) - { - return pyopencv_to_generic_vec(obj, value, info); - } - // If user passed an array it is possible to make faster conversions in several cases - PyArrayObject* array_obj = reinterpret_cast(obj); - const NPY_TYPES target_type = asNumpyType(); - const NPY_TYPES source_type = static_cast(PyArray_TYPE(array_obj)); - if (target_type == NPY_OBJECT) - { - // Non-planar arrays representing objects (e.g. array of N Rect is an array of shape Nx4) have NPY_OBJECT - // as their target type. - return pyopencv_to_generic_vec(obj, value, info); - } - if (PyArray_NDIM(array_obj) > 1) - { - failmsg("Can't parse %dD array as '%s' vector argument", PyArray_NDIM(array_obj), info.name); - return false; - } - if (target_type != source_type) - { - // Source type requires conversion - // Allowed conversions for target type is handled in the corresponding pyopencv_to function - return pyopencv_to_generic_vec(obj, value, info); - } - // For all other cases, all array data can be directly copied to std::vector data - // Simple `memcpy` is not possible because NumPy array can reference a slice of the bigger array: - // ``` - // arr = np.ones((8, 4, 5), dtype=np.int32) - // convertible_to_vector_of_int = arr[:, 0, 1] - // ``` - value.resize(static_cast(PyArray_SIZE(array_obj))); - const npy_intp item_step = PyArray_STRIDE(array_obj, 0) / PyArray_ITEMSIZE(array_obj); - const Tp* data_ptr = static_cast(PyArray_DATA(array_obj)); - for (VecIt it = value.begin(); it != value.end(); ++it, data_ptr += item_step) { - *it = *data_ptr; - } - return true; - } - - static PyObject* from(const std::vector& value) - { - if (value.empty()) - { - return PyTuple_New(0); - } - return from(value, ::traits::IsRepresentableAsMatDataType()); - } - -private: - static PyObject* from(const std::vector& value, ::traits::FalseType) - { - // Underlying type is not representable as Mat Data Type - return pyopencv_from_generic_vec(value); - } - - static PyObject* from(const std::vector& value, ::traits::TrueType) - { - // Underlying type is representable as Mat Data Type, so faster return type is available - typedef DataType DType; - typedef typename DType::channel_type UnderlyingArrayType; - - // If Mat is always exposed as NumPy array this code path can be reduced to the following snipped: - // Mat src(value); - // PyObject* array = pyopencv_from(src); - // return PyArray_Squeeze(reinterpret_cast(array)); - // This puts unnecessary restrictions on Mat object those might be avoided without losing the performance. - // Moreover, this version is a bit faster, because it doesn't create temporary objects with reference counting. - - const NPY_TYPES target_type = asNumpyType(); - const int cols = DType::channels; - PyObject* array = NULL; - if (cols == 1) - { - npy_intp dims = static_cast(value.size()); - array = PyArray_SimpleNew(1, &dims, target_type); - } - else - { - npy_intp dims[2] = {static_cast(value.size()), cols}; - array = PyArray_SimpleNew(2, dims, target_type); - } - if(!array) - { - // NumPy arrays with shape (N, 1) and (N) are not equal, so correct error message should distinguish - // them too. - String shape; - if (cols > 1) - { - shape = format("(%d x %d)", static_cast(value.size()), cols); - } - else - { - shape = format("(%d)", static_cast(value.size())); - } - const String error_message = format("Can't allocate NumPy array for vector with dtype=%d and shape=%s", - static_cast(target_type), shape.c_str()); - emit_failmsg(PyExc_MemoryError, error_message.c_str()); - return array; - } - // Fill the array - PyArrayObject* array_obj = reinterpret_cast(array); - UnderlyingArrayType* array_data = static_cast(PyArray_DATA(array_obj)); - // if Tp is representable as Mat DataType, so the following cast is pretty safe... - const UnderlyingArrayType* value_data = reinterpret_cast(value.data()); - memcpy(array_data, value_data, sizeof(UnderlyingArrayType) * value.size() * static_cast(cols)); - return array; - } -}; - -static int OnError(int status, const char *func_name, const char *err_msg, const char *file_name, int line, void *userdata) -{ - PyGILState_STATE gstate; - gstate = PyGILState_Ensure(); - - PyObject *on_error = (PyObject*)userdata; - PyObject *args = Py_BuildValue("isssi", status, func_name, err_msg, file_name, line); - - PyObject *r = PyObject_Call(on_error, args, NULL); - if (r == NULL) { - PyErr_Print(); - } else { - Py_DECREF(r); - } - - Py_DECREF(args); - PyGILState_Release(gstate); - - return 0; // The return value isn't used -} - -static PyObject *pycvRedirectError(PyObject*, PyObject *args, PyObject *kw) -{ - const char *keywords[] = { "on_error", NULL }; - PyObject *on_error; - - if (!PyArg_ParseTupleAndKeywords(args, kw, "O", (char**)keywords, &on_error)) - return NULL; - - if ((on_error != Py_None) && !PyCallable_Check(on_error)) { - PyErr_SetString(PyExc_TypeError, "on_error must be callable"); - return NULL; - } - - // Keep track of the previous handler parameter, so we can decref it when no longer used - static PyObject* last_on_error = NULL; - if (last_on_error) { - Py_DECREF(last_on_error); - last_on_error = NULL; - } - - if (on_error == Py_None) { - ERRWRAP2(redirectError(NULL)); - } else { - last_on_error = on_error; - Py_INCREF(last_on_error); - ERRWRAP2(redirectError(OnError, last_on_error)); - } - Py_RETURN_NONE; -} - -static void OnMouse(int event, int x, int y, int flags, void* param) -{ - PyGILState_STATE gstate; - gstate = PyGILState_Ensure(); - - PyObject *o = (PyObject*)param; - PyObject *args = Py_BuildValue("iiiiO", event, x, y, flags, PyTuple_GetItem(o, 1)); - - PyObject *r = PyObject_Call(PyTuple_GetItem(o, 0), args, NULL); - if (r == NULL) - PyErr_Print(); - else - Py_DECREF(r); - Py_DECREF(args); - PyGILState_Release(gstate); -} - -#ifdef HAVE_OPENCV_HIGHGUI -static PyObject *pycvSetMouseCallback(PyObject*, PyObject *args, PyObject *kw) -{ - const char *keywords[] = { "window_name", "on_mouse", "param", NULL }; - char* name; - PyObject *on_mouse; - PyObject *param = NULL; - - if (!PyArg_ParseTupleAndKeywords(args, kw, "sO|O", (char**)keywords, &name, &on_mouse, ¶m)) - return NULL; - if (!PyCallable_Check(on_mouse)) { - PyErr_SetString(PyExc_TypeError, "on_mouse must be callable"); - return NULL; - } - if (param == NULL) { - param = Py_None; - } - PyObject* py_callback_info = Py_BuildValue("OO", on_mouse, param); - static std::map registered_callbacks; - std::map::iterator i = registered_callbacks.find(name); - if (i != registered_callbacks.end()) - { - Py_DECREF(i->second); - i->second = py_callback_info; - } - else - { - registered_callbacks.insert(std::pair(std::string(name), py_callback_info)); - } - ERRWRAP2(setMouseCallback(name, OnMouse, py_callback_info)); - Py_RETURN_NONE; -} -#endif - -static void OnChange(int pos, void *param) -{ - PyGILState_STATE gstate; - gstate = PyGILState_Ensure(); - - PyObject *o = (PyObject*)param; - PyObject *args = Py_BuildValue("(i)", pos); - PyObject *r = PyObject_Call(PyTuple_GetItem(o, 0), args, NULL); - if (r == NULL) - PyErr_Print(); - else - Py_DECREF(r); - Py_DECREF(args); - PyGILState_Release(gstate); -} - -#ifdef HAVE_OPENCV_HIGHGUI -// workaround for #20408, use nullptr, set value later -static int _createTrackbar(const String &trackbar_name, const String &window_name, int value, int count, - TrackbarCallback onChange, PyObject* py_callback_info) -{ - int n = createTrackbar(trackbar_name, window_name, NULL, count, onChange, py_callback_info); - setTrackbarPos(trackbar_name, window_name, value); - return n; -} -static PyObject *pycvCreateTrackbar(PyObject*, PyObject *args) -{ - PyObject *on_change; - char* trackbar_name; - char* window_name; - int value; - int count; - - if (!PyArg_ParseTuple(args, "ssiiO", &trackbar_name, &window_name, &value, &count, &on_change)) - return NULL; - if (!PyCallable_Check(on_change)) { - PyErr_SetString(PyExc_TypeError, "on_change must be callable"); - return NULL; - } - PyObject* py_callback_info = Py_BuildValue("OO", on_change, Py_None); - std::string name = std::string(window_name) + ":" + std::string(trackbar_name); - static std::map registered_callbacks; - std::map::iterator i = registered_callbacks.find(name); - if (i != registered_callbacks.end()) - { - Py_DECREF(i->second); - i->second = py_callback_info; - } - else - { - registered_callbacks.insert(std::pair(name, py_callback_info)); - } - ERRWRAP2(_createTrackbar(trackbar_name, window_name, value, count, OnChange, py_callback_info)); - Py_RETURN_NONE; -} - -static void OnButtonChange(int state, void *param) -{ - PyGILState_STATE gstate; - gstate = PyGILState_Ensure(); - - PyObject *o = (PyObject*)param; - PyObject *args; - if(PyTuple_GetItem(o, 1) != NULL) - { - args = Py_BuildValue("(iO)", state, PyTuple_GetItem(o,1)); - } - else - { - args = Py_BuildValue("(i)", state); - } - - PyObject *r = PyObject_Call(PyTuple_GetItem(o, 0), args, NULL); - if (r == NULL) - PyErr_Print(); - else - Py_DECREF(r); - Py_DECREF(args); - PyGILState_Release(gstate); -} - -static PyObject *pycvCreateButton(PyObject*, PyObject *args, PyObject *kw) -{ - const char* keywords[] = {"buttonName", "onChange", "userData", "buttonType", "initialButtonState", NULL}; - PyObject *on_change; - PyObject *userdata = NULL; - char* button_name; - int button_type = 0; - int initial_button_state = 0; - - if (!PyArg_ParseTupleAndKeywords(args, kw, "sO|Oii", (char**)keywords, &button_name, &on_change, &userdata, &button_type, &initial_button_state)) - return NULL; - if (!PyCallable_Check(on_change)) { - PyErr_SetString(PyExc_TypeError, "onChange must be callable"); - return NULL; - } - if (userdata == NULL) { - userdata = Py_None; - } - - PyObject* py_callback_info = Py_BuildValue("OO", on_change, userdata); - std::string name(button_name); - - static std::map registered_callbacks; - std::map::iterator i = registered_callbacks.find(name); - if (i != registered_callbacks.end()) - { - Py_DECREF(i->second); - i->second = py_callback_info; - } - else - { - registered_callbacks.insert(std::pair(name, py_callback_info)); - } - ERRWRAP2(createButton(button_name, OnButtonChange, py_callback_info, button_type, initial_button_state != 0)); - Py_RETURN_NONE; -} -#endif /////////////////////////////////////////////////////////////////////////////////////// diff --git a/modules/python/src2/cv2.hpp b/modules/python/src2/cv2.hpp new file mode 100644 index 0000000000..9293a593f2 --- /dev/null +++ b/modules/python/src2/cv2.hpp @@ -0,0 +1,55 @@ +#ifndef CV2_HPP +#define CV2_HPP + +//warning number '5033' not a valid compiler warning in vc12 +#if defined(_MSC_VER) && (_MSC_VER > 1800) +// eliminating duplicated round() declaration +#define HAVE_ROUND 1 +#pragma warning(push) +#pragma warning(disable:5033) // 'register' is no longer a supported storage class +#endif + +// #define CVPY_DYNAMIC_INIT +// #define Py_DEBUG + +#if defined(CVPY_DYNAMIC_INIT) && !defined(Py_DEBUG) +# define Py_LIMITED_API 0x03030000 +#endif + +#include +#include +#include + +#if PY_MAJOR_VERSION < 3 +#undef CVPY_DYNAMIC_INIT +#else +#define CV_PYTHON_3 1 +#endif + +#if defined(_MSC_VER) && (_MSC_VER > 1800) +#pragma warning(pop) +#endif + +#define MODULESTR "cv2" +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION + +#include + +#include "pycompat.hpp" + +class ArgInfo +{ +public: + const char* name; + bool outputarg; + // more fields may be added if necessary + + ArgInfo(const char* name_, bool outputarg_) : name(name_), outputarg(outputarg_) {} + +private: + ArgInfo(const ArgInfo&) = delete; + ArgInfo& operator=(const ArgInfo&) = delete; +}; + + +#endif // CV2_HPP diff --git a/modules/python/src2/cv2_convert.cpp b/modules/python/src2/cv2_convert.cpp new file mode 100644 index 0000000000..eb800b6ad5 --- /dev/null +++ b/modules/python/src2/cv2_convert.cpp @@ -0,0 +1,1052 @@ +// must be defined before importing numpy headers +// https://numpy.org/doc/1.17/reference/c-api.array.html#importing-the-api +#define NO_IMPORT_ARRAY +#define PY_ARRAY_UNIQUE_SYMBOL opencv_ARRAY_API + +#include "cv2_convert.hpp" +#include "cv2_numpy.hpp" +#include "opencv2/core/utils/logger.hpp" + +PyTypeObject* pyopencv_Mat_TypePtr = nullptr; + +//====================================================================================================================== + +using namespace cv; + +template +static std::string pycv_dumpArray(const T* arr, int n) +{ + std::ostringstream out; + out << "["; + for (int i = 0; i < n; ++i) + out << " " << arr[i]; + out << " ]"; + return out.str(); +} + +//====================================================================================================================== + +// --- Mat + +// special case, when the converter needs full ArgInfo structure +template<> +bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) +{ + if(!o || o == Py_None) + { + if( !m.data ) + m.allocator = &g_numpyAllocator; + return true; + } + + if( PyInt_Check(o) ) + { + double v[] = {static_cast(PyInt_AsLong((PyObject*)o)), 0., 0., 0.}; + m = Mat(4, 1, CV_64F, v).clone(); + return true; + } + if( PyFloat_Check(o) ) + { + double v[] = {PyFloat_AsDouble((PyObject*)o), 0., 0., 0.}; + m = Mat(4, 1, CV_64F, v).clone(); + return true; + } + if( PyTuple_Check(o) ) + { + int i, sz = (int)PyTuple_Size((PyObject*)o); + m = Mat(sz, 1, CV_64F); + for( i = 0; i < sz; i++ ) + { + PyObject* oi = PyTuple_GetItem(o, i); + if( PyInt_Check(oi) ) + m.at(i) = (double)PyInt_AsLong(oi); + else if( PyFloat_Check(oi) ) + m.at(i) = (double)PyFloat_AsDouble(oi); + else + { + failmsg("%s is not a numerical tuple", info.name); + m.release(); + return false; + } + } + return true; + } + + if( !PyArray_Check(o) ) + { + failmsg("%s is not a numpy array, neither a scalar", info.name); + return false; + } + + PyArrayObject* oarr = (PyArrayObject*) o; + + bool needcopy = false, needcast = false; + int typenum = PyArray_TYPE(oarr), new_typenum = typenum; + int type = typenum == NPY_UBYTE ? CV_8U : + typenum == NPY_BYTE ? CV_8S : + typenum == NPY_USHORT ? CV_16U : + typenum == NPY_SHORT ? CV_16S : + typenum == NPY_INT ? CV_32S : + typenum == NPY_INT32 ? CV_32S : + typenum == NPY_FLOAT ? CV_32F : + typenum == NPY_DOUBLE ? CV_64F : -1; + + if( type < 0 ) + { + if( typenum == NPY_INT64 || typenum == NPY_UINT64 || typenum == NPY_LONG ) + { + needcopy = needcast = true; + new_typenum = NPY_INT; + type = CV_32S; + } + else + { + failmsg("%s data type = %d is not supported", info.name, typenum); + return false; + } + } + +#ifndef CV_MAX_DIM + const int CV_MAX_DIM = 32; +#endif + + int ndims = PyArray_NDIM(oarr); + if(ndims >= CV_MAX_DIM) + { + failmsg("%s dimensionality (=%d) is too high", info.name, ndims); + return false; + } + + size_t elemsize = CV_ELEM_SIZE1(type); + const npy_intp* _sizes = PyArray_DIMS(oarr); + const npy_intp* _strides = PyArray_STRIDES(oarr); + + CV_LOG_DEBUG(NULL, "Incoming ndarray '" << info.name << "': ndims=" << ndims << " _sizes=" << pycv_dumpArray(_sizes, ndims) << " _strides=" << pycv_dumpArray(_strides, ndims)); + + bool ismultichannel = ndims == 3 && _sizes[2] <= CV_CN_MAX; + if (pyopencv_Mat_TypePtr && PyObject_TypeCheck(o, pyopencv_Mat_TypePtr)) + { + bool wrapChannels = false; + PyObject* pyobj_wrap_channels = PyObject_GetAttrString(o, "wrap_channels"); + if (pyobj_wrap_channels) + { + if (!pyopencv_to_safe(pyobj_wrap_channels, wrapChannels, ArgInfo("cv.Mat.wrap_channels", 0))) + { + // TODO extra message + Py_DECREF(pyobj_wrap_channels); + return false; + } + Py_DECREF(pyobj_wrap_channels); + } + ismultichannel = wrapChannels && ndims >= 1; + } + + for( int i = ndims-1; i >= 0 && !needcopy; i-- ) + { + // these checks handle cases of + // a) multi-dimensional (ndims > 2) arrays, as well as simpler 1- and 2-dimensional cases + // b) transposed arrays, where _strides[] elements go in non-descending order + // c) flipped arrays, where some of _strides[] elements are negative + // the _sizes[i] > 1 is needed to avoid spurious copies when NPY_RELAXED_STRIDES is set + if( (i == ndims-1 && _sizes[i] > 1 && (size_t)_strides[i] != elemsize) || + (i < ndims-1 && _sizes[i] > 1 && _strides[i] < _strides[i+1]) ) + needcopy = true; + } + + if (ismultichannel) + { + int channels = ndims >= 1 ? (int)_sizes[ndims - 1] : 1; + if (channels > CV_CN_MAX) + { + failmsg("%s unable to wrap channels, too high (%d > CV_CN_MAX=%d)", info.name, (int)channels, (int)CV_CN_MAX); + return false; + } + ndims--; + type |= CV_MAKETYPE(0, channels); + + if (ndims >= 1 && _strides[ndims - 1] != (npy_intp)elemsize*_sizes[ndims]) + needcopy = true; + + elemsize = CV_ELEM_SIZE(type); + } + + if (needcopy) + { + if (info.outputarg) + { + failmsg("Layout of the output array %s is incompatible with cv::Mat", info.name); + return false; + } + + if( needcast ) { + o = PyArray_Cast(oarr, new_typenum); + oarr = (PyArrayObject*) o; + } + else { + oarr = PyArray_GETCONTIGUOUS(oarr); + o = (PyObject*) oarr; + } + + _strides = PyArray_STRIDES(oarr); + } + + int size[CV_MAX_DIM+1] = {}; + size_t step[CV_MAX_DIM+1] = {}; + + // Normalize strides in case NPY_RELAXED_STRIDES is set + size_t default_step = elemsize; + for ( int i = ndims - 1; i >= 0; --i ) + { + size[i] = (int)_sizes[i]; + if ( size[i] > 1 ) + { + step[i] = (size_t)_strides[i]; + default_step = step[i] * size[i]; + } + else + { + step[i] = default_step; + default_step *= size[i]; + } + } + + // handle degenerate case + // FIXIT: Don't force 1D for Scalars + if( ndims == 0) { + size[ndims] = 1; + step[ndims] = elemsize; + ndims++; + } + +#if 1 + CV_LOG_DEBUG(NULL, "Construct Mat: ndims=" << ndims << " size=" << pycv_dumpArray(size, ndims) << " step=" << pycv_dumpArray(step, ndims) << " type=" << cv::typeToString(type)); +#endif + + m = Mat(ndims, size, type, PyArray_DATA(oarr), step); + m.u = g_numpyAllocator.allocate(o, ndims, size, type, step); + m.addref(); + + if( !needcopy ) + { + Py_INCREF(o); + } + m.allocator = &g_numpyAllocator; + + return true; +} + +template<> +PyObject* pyopencv_from(const cv::Mat& m) +{ + if( !m.data ) + Py_RETURN_NONE; + cv::Mat temp, *p = (cv::Mat*)&m; + if(!p->u || p->allocator != &g_numpyAllocator) + { + temp.allocator = &g_numpyAllocator; + ERRWRAP2(m.copyTo(temp)); + p = &temp; + } + PyObject* o = (PyObject*)p->u->userdata; + Py_INCREF(o); + return o; +} + +// --- bool + +template<> +bool pyopencv_to(PyObject* obj, bool& value, const ArgInfo& info) +{ + if (!obj || obj == Py_None) + { + return true; + } + if (isBool(obj) || PyArray_IsIntegerScalar(obj)) + { + npy_bool npy_value = NPY_FALSE; + const int ret_code = PyArray_BoolConverter(obj, &npy_value); + if (ret_code >= 0) + { + value = (npy_value == NPY_TRUE); + return true; + } + } + failmsg("Argument '%s' is not convertable to bool", info.name); + return false; +} + +template<> +PyObject* pyopencv_from(const bool& value) +{ + return PyBool_FromLong(value); +} + +// --- ptr + +template<> +bool pyopencv_to(PyObject* obj, void*& ptr, const ArgInfo& info) +{ + CV_UNUSED(info); + if (!obj || obj == Py_None) + return true; + + if (!PyLong_Check(obj)) + return false; + ptr = PyLong_AsVoidPtr(obj); + return ptr != NULL && !PyErr_Occurred(); +} + +PyObject* pyopencv_from(void*& ptr) +{ + return PyLong_FromVoidPtr(ptr); +} + +// -- Scalar + +template<> +bool pyopencv_to(PyObject *o, Scalar& s, const ArgInfo& info) +{ + if(!o || o == Py_None) + return true; + if (PySequence_Check(o)) { + if (4 < PySequence_Size(o)) + { + failmsg("Scalar value for argument '%s' is longer than 4", info.name); + return false; + } + for (Py_ssize_t i = 0; i < PySequence_Size(o); i++) { + SafeSeqItem item_wrap(o, i); + PyObject *item = item_wrap.item; + if (PyFloat_Check(item) || PyInt_Check(item)) { + s[(int)i] = PyFloat_AsDouble(item); + } else { + failmsg("Scalar value for argument '%s' is not numeric", info.name); + return false; + } + } + } else { + if (PyFloat_Check(o) || PyInt_Check(o)) { + s[0] = PyFloat_AsDouble(o); + } else { + failmsg("Scalar value for argument '%s' is not numeric", info.name); + return false; + } + } + return true; +} + +template<> +PyObject* pyopencv_from(const Scalar& src) +{ + return Py_BuildValue("(dddd)", src[0], src[1], src[2], src[3]); +} + +// --- size_t + +template<> +bool pyopencv_to(PyObject* obj, size_t& value, const ArgInfo& info) +{ + if (!obj || obj == Py_None) + { + return true; + } + if (isBool(obj)) + { + failmsg("Argument '%s' must be integer type, not bool", info.name); + return false; + } + if (PyArray_IsIntegerScalar(obj)) + { + if (PyLong_Check(obj)) + { +#if defined(CV_PYTHON_3) + value = PyLong_AsSize_t(obj); +#else + #if ULONG_MAX == SIZE_MAX + value = PyLong_AsUnsignedLong(obj); + #else + value = PyLong_AsUnsignedLongLong(obj); + #endif +#endif + } +#if !defined(CV_PYTHON_3) + // Python 2.x has PyIntObject which is not a subtype of PyLongObject + // Overflow check here is unnecessary because object will be converted to long on the + // interpreter side + else if (PyInt_Check(obj)) + { + const long res = PyInt_AsLong(obj); + if (res < 0) { + failmsg("Argument '%s' can not be safely parsed to 'size_t'", info.name); + return false; + } + #if ULONG_MAX == SIZE_MAX + value = PyInt_AsUnsignedLongMask(obj); + #else + value = PyInt_AsUnsignedLongLongMask(obj); + #endif + } +#endif + else + { + const bool isParsed = parseNumpyScalar(obj, value); + if (!isParsed) { + failmsg("Argument '%s' can not be safely parsed to 'size_t'", info.name); + return false; + } + } + } + else + { + failmsg("Argument '%s' is required to be an integer", info.name); + return false; + } + return !PyErr_Occurred(); +} + +template<> +PyObject* pyopencv_from(const size_t& value) +{ + return PyLong_FromSize_t(value); +} + +// --- int + +template<> +bool pyopencv_to(PyObject* obj, int& value, const ArgInfo& info) +{ + if (!obj || obj == Py_None) + { + return true; + } + if (isBool(obj)) + { + failmsg("Argument '%s' must be integer, not bool", info.name); + return false; + } + if (PyArray_IsIntegerScalar(obj)) + { + value = PyArray_PyIntAsInt(obj); + } + else + { + failmsg("Argument '%s' is required to be an integer", info.name); + return false; + } + return !CV_HAS_CONVERSION_ERROR(value); +} + +template<> +PyObject* pyopencv_from(const int& value) +{ + return PyInt_FromLong(value); +} + +// --- int64 + +template<> +PyObject* pyopencv_from(const int64& value) +{ + return PyLong_FromLongLong(value); +} + + +// --- uchar + +template<> +bool pyopencv_to(PyObject* obj, uchar& value, const ArgInfo& info) +{ + CV_UNUSED(info); + if(!obj || obj == Py_None) + return true; + int ivalue = (int)PyInt_AsLong(obj); + value = cv::saturate_cast(ivalue); + return ivalue != -1 || !PyErr_Occurred(); +} + +template<> +PyObject* pyopencv_from(const uchar& value) +{ + return PyInt_FromLong(value); +} + +// --- char + +template<> +bool pyopencv_to(PyObject* obj, char& value, const ArgInfo& info) +{ + if (!obj || obj == Py_None) + { + return true; + } + if (isBool(obj)) + { + failmsg("Argument '%s' must be an integer, not bool", info.name); + return false; + } + if (PyArray_IsIntegerScalar(obj)) + { + value = saturate_cast(PyArray_PyIntAsInt(obj)); + } + else + { + failmsg("Argument '%s' is required to be an integer", info.name); + return false; + } + return !CV_HAS_CONVERSION_ERROR(value); +} + +// --- double + +template<> +bool pyopencv_to(PyObject* obj, double& value, const ArgInfo& info) +{ + if (!obj || obj == Py_None) + { + return true; + } + if (isBool(obj)) + { + failmsg("Argument '%s' must be double, not bool", info.name); + return false; + } + if (PyArray_IsPythonNumber(obj)) + { + if (PyLong_Check(obj)) + { + value = PyLong_AsDouble(obj); + } + else + { + value = PyFloat_AsDouble(obj); + } + } + else if (PyArray_CheckScalar(obj)) + { + const bool isParsed = parseNumpyScalar(obj, value); + if (!isParsed) { + failmsg("Argument '%s' can not be safely parsed to 'double'", info.name); + return false; + } + } + else + { + failmsg("Argument '%s' can not be treated as a double", info.name); + return false; + } + return !PyErr_Occurred(); +} + +template<> +PyObject* pyopencv_from(const double& value) +{ + return PyFloat_FromDouble(value); +} + +// --- float + +template<> +bool pyopencv_to(PyObject* obj, float& value, const ArgInfo& info) +{ + if (!obj || obj == Py_None) + { + return true; + } + if (isBool(obj)) + { + failmsg("Argument '%s' must be float, not bool", info.name); + return false; + } + if (PyArray_IsPythonNumber(obj)) + { + if (PyLong_Check(obj)) + { + double res = PyLong_AsDouble(obj); + value = static_cast(res); + } + else + { + double res = PyFloat_AsDouble(obj); + value = static_cast(res); + } + } + else if (PyArray_CheckScalar(obj)) + { + const bool isParsed = parseNumpyScalar(obj, value); + if (!isParsed) { + failmsg("Argument '%s' can not be safely parsed to 'float'", info.name); + return false; + } + } + else + { + failmsg("Argument '%s' can't be treated as a float", info.name); + return false; + } + return !PyErr_Occurred(); +} + +template<> +PyObject* pyopencv_from(const float& value) +{ + return PyFloat_FromDouble(value); +} + +// --- string + +template<> +bool pyopencv_to(PyObject* obj, String &value, const ArgInfo& info) +{ + if(!obj || obj == Py_None) + { + return true; + } + std::string str; + if (getUnicodeString(obj, str)) + { + value = str; + return true; + } + else + { + // If error hasn't been already set by Python conversion functions + if (!PyErr_Occurred()) + { + // Direct access to underlying slots of PyObjectType is not allowed + // when limited API is enabled +#ifdef Py_LIMITED_API + failmsg("Can't convert object to 'str' for '%s'", info.name); +#else + failmsg("Can't convert object of type '%s' to 'str' for '%s'", + obj->ob_type->tp_name, info.name); +#endif + } + } + return false; +} + +template<> +PyObject* pyopencv_from(const String& value) +{ + return PyString_FromString(value.empty() ? "" : value.c_str()); +} + +#if CV_VERSION_MAJOR == 3 +template<> +PyObject* pyopencv_from(const std::string& value) +{ + return PyString_FromString(value.empty() ? "" : value.c_str()); +} +#endif + +// --- Size + +template<> +bool pyopencv_to(PyObject* obj, Size& sz, const ArgInfo& info) +{ + RefWrapper values[] = {RefWrapper(sz.width), + RefWrapper(sz.height)}; + return parseSequence(obj, values, info); +} + +template<> +PyObject* pyopencv_from(const Size& sz) +{ + return Py_BuildValue("(ii)", sz.width, sz.height); +} + +template<> +bool pyopencv_to(PyObject* obj, Size_& sz, const ArgInfo& info) +{ + RefWrapper values[] = {RefWrapper(sz.width), + RefWrapper(sz.height)}; + return parseSequence(obj, values, info); +} + +template<> +PyObject* pyopencv_from(const Size_& sz) +{ + return Py_BuildValue("(ff)", sz.width, sz.height); +} + +// --- Rect + +template<> +bool pyopencv_to(PyObject* obj, Rect& r, const ArgInfo& info) +{ + RefWrapper values[] = {RefWrapper(r.x), RefWrapper(r.y), + RefWrapper(r.width), + RefWrapper(r.height)}; + return parseSequence(obj, values, info); +} + +template<> +PyObject* pyopencv_from(const Rect& r) +{ + return Py_BuildValue("(iiii)", r.x, r.y, r.width, r.height); +} + +template<> +bool pyopencv_to(PyObject* obj, Rect2d& r, const ArgInfo& info) +{ + RefWrapper values[] = { + RefWrapper(r.x), RefWrapper(r.y), + RefWrapper(r.width), RefWrapper(r.height)}; + return parseSequence(obj, values, info); +} + +template<> +PyObject* pyopencv_from(const Rect2d& r) +{ + return Py_BuildValue("(dddd)", r.x, r.y, r.width, r.height); +} + +// --- RotatedRect + +template<> +bool pyopencv_to(PyObject* obj, RotatedRect& dst, const ArgInfo& info) +{ + if (!obj || obj == Py_None) + { + return true; + } + if (!PySequence_Check(obj)) + { + failmsg("Can't parse '%s' as RotatedRect." + "Input argument doesn't provide sequence protocol", + info.name); + return false; + } + const std::size_t sequenceSize = PySequence_Size(obj); + if (sequenceSize != 3) + { + failmsg("Can't parse '%s' as RotatedRect. Expected sequence length 3, got %lu", + info.name, sequenceSize); + return false; + } + { + const String centerItemName = format("'%s' center point", info.name); + const ArgInfo centerItemInfo(centerItemName.c_str(), false); + SafeSeqItem centerItem(obj, 0); + if (!pyopencv_to(centerItem.item, dst.center, centerItemInfo)) + { + return false; + } + } + { + const String sizeItemName = format("'%s' size", info.name); + const ArgInfo sizeItemInfo(sizeItemName.c_str(), false); + SafeSeqItem sizeItem(obj, 1); + if (!pyopencv_to(sizeItem.item, dst.size, sizeItemInfo)) + { + return false; + } + } + { + const String angleItemName = format("'%s' angle", info.name); + const ArgInfo angleItemInfo(angleItemName.c_str(), false); + SafeSeqItem angleItem(obj, 2); + if (!pyopencv_to(angleItem.item, dst.angle, angleItemInfo)) + { + return false; + } + } + return true; +} + +template<> +PyObject* pyopencv_from(const RotatedRect& src) +{ + return Py_BuildValue("((ff)(ff)f)", src.center.x, src.center.y, src.size.width, src.size.height, src.angle); +} + +// --- Range + +template<> +bool pyopencv_to(PyObject* obj, Range& r, const ArgInfo& info) +{ + if (!obj || obj == Py_None) + { + return true; + } + if (PyObject_Size(obj) == 0) + { + r = Range::all(); + return true; + } + RefWrapper values[] = {RefWrapper(r.start), RefWrapper(r.end)}; + return parseSequence(obj, values, info); +} + +template<> +PyObject* pyopencv_from(const Range& r) +{ + return Py_BuildValue("(ii)", r.start, r.end); +} + +// --- Point + +template<> +bool pyopencv_to(PyObject* obj, Point& p, const ArgInfo& info) +{ + RefWrapper values[] = {RefWrapper(p.x), RefWrapper(p.y)}; + return parseSequence(obj, values, info); +} + +template<> +PyObject* pyopencv_from(const Point& p) +{ + return Py_BuildValue("(ii)", p.x, p.y); +} + +template <> +bool pyopencv_to(PyObject* obj, Point2f& p, const ArgInfo& info) +{ + RefWrapper values[] = {RefWrapper(p.x), + RefWrapper(p.y)}; + return parseSequence(obj, values, info); +} + +template<> +PyObject* pyopencv_from(const Point2f& p) +{ + return Py_BuildValue("(dd)", p.x, p.y); +} + +template<> +bool pyopencv_to(PyObject* obj, Point2d& p, const ArgInfo& info) +{ + RefWrapper values[] = {RefWrapper(p.x), + RefWrapper(p.y)}; + return parseSequence(obj, values, info); +} + +template<> +PyObject* pyopencv_from(const Point2d& p) +{ + return Py_BuildValue("(dd)", p.x, p.y); +} + +template<> +bool pyopencv_to(PyObject* obj, Point3f& p, const ArgInfo& info) +{ + RefWrapper values[] = {RefWrapper(p.x), + RefWrapper(p.y), + RefWrapper(p.z)}; + return parseSequence(obj, values, info); +} + +template<> +PyObject* pyopencv_from(const Point3f& p) +{ + return Py_BuildValue("(ddd)", p.x, p.y, p.z); +} + +template<> +bool pyopencv_to(PyObject* obj, Point3d& p, const ArgInfo& info) +{ + RefWrapper values[] = {RefWrapper(p.x), + RefWrapper(p.y), + RefWrapper(p.z)}; + return parseSequence(obj, values, info); +} + +template<> +PyObject* pyopencv_from(const Point3d& p) +{ + return Py_BuildValue("(ddd)", p.x, p.y, p.z); +} + +// --- Vec + +bool pyopencv_to(PyObject* obj, Vec4d& v, ArgInfo& info) +{ + RefWrapper values[] = {RefWrapper(v[0]), RefWrapper(v[1]), + RefWrapper(v[2]), RefWrapper(v[3])}; + return parseSequence(obj, values, info); +} + +PyObject* pyopencv_from(const Vec4d& v) +{ + return Py_BuildValue("(dddd)", v[0], v[1], v[2], v[3]); +} + +bool pyopencv_to(PyObject* obj, Vec4f& v, ArgInfo& info) +{ + RefWrapper values[] = {RefWrapper(v[0]), RefWrapper(v[1]), + RefWrapper(v[2]), RefWrapper(v[3])}; + return parseSequence(obj, values, info); +} + +PyObject* pyopencv_from(const Vec4f& v) +{ + return Py_BuildValue("(ffff)", v[0], v[1], v[2], v[3]); +} + +bool pyopencv_to(PyObject* obj, Vec4i& v, ArgInfo& info) +{ + RefWrapper values[] = {RefWrapper(v[0]), RefWrapper(v[1]), + RefWrapper(v[2]), RefWrapper(v[3])}; + return parseSequence(obj, values, info); +} + +PyObject* pyopencv_from(const Vec4i& v) +{ + return Py_BuildValue("(iiii)", v[0], v[1], v[2], v[3]); +} + +bool pyopencv_to(PyObject* obj, Vec3d& v, ArgInfo& info) +{ + RefWrapper values[] = {RefWrapper(v[0]), + RefWrapper(v[1]), + RefWrapper(v[2])}; + return parseSequence(obj, values, info); +} + +PyObject* pyopencv_from(const Vec3d& v) +{ + return Py_BuildValue("(ddd)", v[0], v[1], v[2]); +} + +bool pyopencv_to(PyObject* obj, Vec3f& v, ArgInfo& info) +{ + RefWrapper values[] = {RefWrapper(v[0]), + RefWrapper(v[1]), + RefWrapper(v[2])}; + return parseSequence(obj, values, info); +} + +PyObject* pyopencv_from(const Vec3f& v) +{ + return Py_BuildValue("(fff)", v[0], v[1], v[2]); +} + +bool pyopencv_to(PyObject* obj, Vec3i& v, ArgInfo& info) +{ + RefWrapper values[] = {RefWrapper(v[0]), RefWrapper(v[1]), + RefWrapper(v[2])}; + return parseSequence(obj, values, info); +} + +PyObject* pyopencv_from(const Vec3i& v) +{ + return Py_BuildValue("(iii)", v[0], v[1], v[2]); +} + +bool pyopencv_to(PyObject* obj, Vec2d& v, ArgInfo& info) +{ + RefWrapper values[] = {RefWrapper(v[0]), + RefWrapper(v[1])}; + return parseSequence(obj, values, info); +} + +PyObject* pyopencv_from(const Vec2d& v) +{ + return Py_BuildValue("(dd)", v[0], v[1]); +} + +bool pyopencv_to(PyObject* obj, Vec2f& v, ArgInfo& info) +{ + RefWrapper values[] = {RefWrapper(v[0]), + RefWrapper(v[1])}; + return parseSequence(obj, values, info); +} + +PyObject* pyopencv_from(const Vec2f& v) +{ + return Py_BuildValue("(ff)", v[0], v[1]); +} + +bool pyopencv_to(PyObject* obj, Vec2i& v, ArgInfo& info) +{ + RefWrapper values[] = {RefWrapper(v[0]), RefWrapper(v[1])}; + return parseSequence(obj, values, info); +} + +PyObject* pyopencv_from(const Vec2i& v) +{ + return Py_BuildValue("(ii)", v[0], v[1]); +} + + +// --- TermCriteria + +template<> +bool pyopencv_to(PyObject* obj, TermCriteria& dst, const ArgInfo& info) +{ + if (!obj || obj == Py_None) + { + return true; + } + if (!PySequence_Check(obj)) + { + failmsg("Can't parse '%s' as TermCriteria." + "Input argument doesn't provide sequence protocol", + info.name); + return false; + } + const std::size_t sequenceSize = PySequence_Size(obj); + if (sequenceSize != 3) { + failmsg("Can't parse '%s' as TermCriteria. Expected sequence length 3, " + "got %lu", + info.name, sequenceSize); + return false; + } + { + const String typeItemName = format("'%s' criteria type", info.name); + const ArgInfo typeItemInfo(typeItemName.c_str(), false); + SafeSeqItem typeItem(obj, 0); + if (!pyopencv_to(typeItem.item, dst.type, typeItemInfo)) + { + return false; + } + } + { + const String maxCountItemName = format("'%s' max count", info.name); + const ArgInfo maxCountItemInfo(maxCountItemName.c_str(), false); + SafeSeqItem maxCountItem(obj, 1); + if (!pyopencv_to(maxCountItem.item, dst.maxCount, maxCountItemInfo)) + { + return false; + } + } + { + const String epsilonItemName = format("'%s' epsilon", info.name); + const ArgInfo epsilonItemInfo(epsilonItemName.c_str(), false); + SafeSeqItem epsilonItem(obj, 2); + if (!pyopencv_to(epsilonItem.item, dst.epsilon, epsilonItemInfo)) + { + return false; + } + } + return true; +} + +template<> +PyObject* pyopencv_from(const TermCriteria& src) +{ + return Py_BuildValue("(iid)", src.type, src.maxCount, src.epsilon); +} + +// --- Moments + +template<> +PyObject* pyopencv_from(const Moments& m) +{ + return Py_BuildValue("{s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d,s:d}", + "m00", m.m00, "m10", m.m10, "m01", m.m01, + "m20", m.m20, "m11", m.m11, "m02", m.m02, + "m30", m.m30, "m21", m.m21, "m12", m.m12, "m03", m.m03, + "mu20", m.mu20, "mu11", m.mu11, "mu02", m.mu02, + "mu30", m.mu30, "mu21", m.mu21, "mu12", m.mu12, "mu03", m.mu03, + "nu20", m.nu20, "nu11", m.nu11, "nu02", m.nu02, + "nu30", m.nu30, "nu21", m.nu21, "nu12", m.nu12, "nu03", m.nu03); +} + +// --- pair + +template<> +PyObject* pyopencv_from(const std::pair& src) +{ + return Py_BuildValue("(id)", src.first, src.second); +} diff --git a/modules/python/src2/cv2_convert.hpp b/modules/python/src2/cv2_convert.hpp new file mode 100644 index 0000000000..e9ed71258b --- /dev/null +++ b/modules/python/src2/cv2_convert.hpp @@ -0,0 +1,529 @@ +#ifndef CV2_CONVERT_HPP +#define CV2_CONVERT_HPP + +#include "cv2.hpp" +#include "cv2_util.hpp" +#include "cv2_numpy.hpp" +#include +#include +#include // std::enable_if + +extern PyTypeObject* pyopencv_Mat_TypePtr; + +#define CV_HAS_CONVERSION_ERROR(x) (((x) == -1) && PyErr_Occurred()) + +inline bool isBool(PyObject* obj) CV_NOEXCEPT +{ + return PyArray_IsScalar(obj, Bool) || PyBool_Check(obj); +} + +//====================================================================================================================== + + +// exception-safe pyopencv_to +template static +bool pyopencv_to_safe(PyObject* obj, _Tp& value, const ArgInfo& info) +{ + try + { + return pyopencv_to(obj, value, info); + } + catch (const std::exception &e) + { + PyErr_SetString(opencv_error, cv::format("Conversion error: %s, what: %s", info.name, e.what()).c_str()); + return false; + } + catch (...) + { + PyErr_SetString(opencv_error, cv::format("Conversion error: %s", info.name).c_str()); + return false; + } +} + +//====================================================================================================================== + +template // TEnable is used for SFINAE checks +struct PyOpenCV_Converter +{ + //static inline bool to(PyObject* obj, T& p, const ArgInfo& info); + //static inline PyObject* from(const T& src); +}; + +// --- Generic + +template +bool pyopencv_to(PyObject* obj, T& p, const ArgInfo& info) { return PyOpenCV_Converter::to(obj, p, info); } + +template +PyObject* pyopencv_from(const T& src) { return PyOpenCV_Converter::from(src); } + +// --- Matx + +template +bool pyopencv_to(PyObject* o, cv::Matx<_Tp, m, n>& mx, const ArgInfo& info) +{ + cv::Mat tmp; + if (!pyopencv_to(o, tmp, info)) { + return false; + } + + tmp.copyTo(mx); + return true; +} + +template +PyObject* pyopencv_from(const cv::Matx<_Tp, m, n>& matx) +{ + return pyopencv_from(cv::Mat(matx)); +} + +// --- bool +template<> bool pyopencv_to(PyObject* obj, bool& value, const ArgInfo& info); +template<> PyObject* pyopencv_from(const bool& value); + +// --- Mat +template<> bool pyopencv_to(PyObject* o, cv::Mat& m, const ArgInfo& info); +template<> PyObject* pyopencv_from(const cv::Mat& m); + +// --- Ptr +template +struct PyOpenCV_Converter< cv::Ptr > +{ + static PyObject* from(const cv::Ptr& p) + { + if (!p) + Py_RETURN_NONE; + return pyopencv_from(*p); + } + static bool to(PyObject *o, cv::Ptr& p, const ArgInfo& info) + { + if (!o || o == Py_None) + return true; + p = cv::makePtr(); + return pyopencv_to(o, *p, info); + } +}; + +// --- ptr +template<> bool pyopencv_to(PyObject* obj, void*& ptr, const ArgInfo& info); +PyObject* pyopencv_from(void*& ptr); + +// --- Scalar +template<> bool pyopencv_to(PyObject *o, cv::Scalar& s, const ArgInfo& info); +template<> PyObject* pyopencv_from(const cv::Scalar& src); + +// --- size_t +template<> bool pyopencv_to(PyObject* obj, size_t& value, const ArgInfo& info); +template<> PyObject* pyopencv_from(const size_t& value); + +// --- int +template<> bool pyopencv_to(PyObject* obj, int& value, const ArgInfo& info); +template<> PyObject* pyopencv_from(const int& value); + +// --- int64 +template<> PyObject* pyopencv_from(const int64& value); + +// There is conflict between "size_t" and "unsigned int". +// They are the same type on some 32-bit platforms. +template +struct PyOpenCV_Converter + < T, typename std::enable_if< std::is_same::value && !std::is_same::value >::type > +{ + static inline PyObject* from(const unsigned int& value) + { + return PyLong_FromUnsignedLong(value); + } + + static inline bool to(PyObject* obj, unsigned int& value, const ArgInfo& info) + { + CV_UNUSED(info); + if(!obj || obj == Py_None) + return true; + if(PyInt_Check(obj)) + value = (unsigned int)PyInt_AsLong(obj); + else if(PyLong_Check(obj)) + value = (unsigned int)PyLong_AsLong(obj); + else + return false; + return value != (unsigned int)-1 || !PyErr_Occurred(); + } +}; + +// --- uchar +template<> bool pyopencv_to(PyObject* obj, uchar& value, const ArgInfo& info); +template<> PyObject* pyopencv_from(const uchar& value); + +// --- char +template<> bool pyopencv_to(PyObject* obj, char& value, const ArgInfo& info); + +// --- double +template<> bool pyopencv_to(PyObject* obj, double& value, const ArgInfo& info); +template<> PyObject* pyopencv_from(const double& value); + +// --- float +template<> bool pyopencv_to(PyObject* obj, float& value, const ArgInfo& info); +template<> PyObject* pyopencv_from(const float& value); + +// --- string +template<> bool pyopencv_to(PyObject* obj, cv::String &value, const ArgInfo& info); +template<> PyObject* pyopencv_from(const cv::String& value); +#if CV_VERSION_MAJOR == 3 +template<> PyObject* pyopencv_from(const std::string& value); +#endif + +// --- Size +template<> bool pyopencv_to(PyObject* obj, cv::Size& sz, const ArgInfo& info); +template<> PyObject* pyopencv_from(const cv::Size& sz); +template<> bool pyopencv_to(PyObject* obj, cv::Size_& sz, const ArgInfo& info); +template<> PyObject* pyopencv_from(const cv::Size_& sz); + +// --- Rect +template<> bool pyopencv_to(PyObject* obj, cv::Rect& r, const ArgInfo& info); +template<> PyObject* pyopencv_from(const cv::Rect& r); +template<> bool pyopencv_to(PyObject* obj, cv::Rect2d& r, const ArgInfo& info); +template<> PyObject* pyopencv_from(const cv::Rect2d& r); + +// --- RotatedRect +template<> bool pyopencv_to(PyObject* obj, cv::RotatedRect& dst, const ArgInfo& info); +template<> PyObject* pyopencv_from(const cv::RotatedRect& src); + +// --- Range +template<> bool pyopencv_to(PyObject* obj, cv::Range& r, const ArgInfo& info); +template<> PyObject* pyopencv_from(const cv::Range& r); + +// --- Point +template<> bool pyopencv_to(PyObject* obj, cv::Point& p, const ArgInfo& info); +template<> PyObject* pyopencv_from(const cv::Point& p); +template<> bool pyopencv_to(PyObject* obj, cv::Point2f& p, const ArgInfo& info); +template<> PyObject* pyopencv_from(const cv::Point2f& p); +template<> bool pyopencv_to(PyObject* obj, cv::Point2d& p, const ArgInfo& info); +template<> PyObject* pyopencv_from(const cv::Point2d& p); +template<> bool pyopencv_to(PyObject* obj, cv::Point3f& p, const ArgInfo& info); +template<> PyObject* pyopencv_from(const cv::Point3f& p); +template<> bool pyopencv_to(PyObject* obj, cv::Point3d& p, const ArgInfo& info); +template<> PyObject* pyopencv_from(const cv::Point3d& p); + +// --- Vec +template +bool pyopencv_to(PyObject* o, cv::Vec<_Tp, cn>& vec, const ArgInfo& info) +{ + return pyopencv_to(o, (cv::Matx<_Tp, cn, 1>&)vec, info); +} +bool pyopencv_to(PyObject* obj, cv::Vec4d& v, ArgInfo& info); +PyObject* pyopencv_from(const cv::Vec4d& v); +bool pyopencv_to(PyObject* obj, cv::Vec4f& v, ArgInfo& info); +PyObject* pyopencv_from(const cv::Vec4f& v); +bool pyopencv_to(PyObject* obj, cv::Vec4i& v, ArgInfo& info); +PyObject* pyopencv_from(const cv::Vec4i& v); +bool pyopencv_to(PyObject* obj, cv::Vec3d& v, ArgInfo& info); +PyObject* pyopencv_from(const cv::Vec3d& v); +bool pyopencv_to(PyObject* obj, cv::Vec3f& v, ArgInfo& info); +PyObject* pyopencv_from(const cv::Vec3f& v); +bool pyopencv_to(PyObject* obj, cv::Vec3i& v, ArgInfo& info); +PyObject* pyopencv_from(const cv::Vec3i& v); +bool pyopencv_to(PyObject* obj, cv::Vec2d& v, ArgInfo& info); +PyObject* pyopencv_from(const cv::Vec2d& v); +bool pyopencv_to(PyObject* obj, cv::Vec2f& v, ArgInfo& info); +PyObject* pyopencv_from(const cv::Vec2f& v); +bool pyopencv_to(PyObject* obj, cv::Vec2i& v, ArgInfo& info); +PyObject* pyopencv_from(const cv::Vec2i& v); + +// --- TermCriteria +template<> bool pyopencv_to(PyObject* obj, cv::TermCriteria& dst, const ArgInfo& info); +template<> PyObject* pyopencv_from(const cv::TermCriteria& src); + +// --- Moments +template<> PyObject* pyopencv_from(const cv::Moments& m); + +// --- pair +template<> PyObject* pyopencv_from(const std::pair& src); + +// --- vector +template +struct pyopencvVecConverter; + +template +bool pyopencv_to(PyObject* obj, std::vector& value, const ArgInfo& info) +{ + if (!obj || obj == Py_None) + { + return true; + } + return pyopencvVecConverter::to(obj, value, info); +} + +template +PyObject* pyopencv_from(const std::vector& value) +{ + return pyopencvVecConverter::from(value); +} + +template +static bool pyopencv_to_generic_vec(PyObject* obj, std::vector& value, const ArgInfo& info) +{ + if (!obj || obj == Py_None) + { + return true; + } + if (!PySequence_Check(obj)) + { + failmsg("Can't parse '%s'. Input argument doesn't provide sequence protocol", info.name); + return false; + } + const size_t n = static_cast(PySequence_Size(obj)); + value.resize(n); + for (size_t i = 0; i < n; i++) + { + SafeSeqItem item_wrap(obj, i); + if (!pyopencv_to(item_wrap.item, value[i], info)) + { + failmsg("Can't parse '%s'. Sequence item with index %lu has a wrong type", info.name, i); + return false; + } + } + return true; +} + +template<> inline bool pyopencv_to_generic_vec(PyObject* obj, std::vector& value, const ArgInfo& info) +{ + if (!obj || obj == Py_None) + { + return true; + } + if (!PySequence_Check(obj)) + { + failmsg("Can't parse '%s'. Input argument doesn't provide sequence protocol", info.name); + return false; + } + const size_t n = static_cast(PySequence_Size(obj)); + value.resize(n); + for (size_t i = 0; i < n; i++) + { + SafeSeqItem item_wrap(obj, i); + bool elem{}; + if (!pyopencv_to(item_wrap.item, elem, info)) + { + failmsg("Can't parse '%s'. Sequence item with index %lu has a wrong type", info.name, i); + return false; + } + value[i] = elem; + } + return true; +} + +template +static PyObject* pyopencv_from_generic_vec(const std::vector& value) +{ + Py_ssize_t n = static_cast(value.size()); + PySafeObject seq(PyTuple_New(n)); + for (Py_ssize_t i = 0; i < n; i++) + { + PyObject* item = pyopencv_from(value[i]); + // If item can't be assigned - PyTuple_SetItem raises exception and returns -1. + if (!item || PyTuple_SetItem(seq, i, item) == -1) + { + return NULL; + } + } + return seq.release(); +} + +template<> inline PyObject* pyopencv_from_generic_vec(const std::vector& value) +{ + Py_ssize_t n = static_cast(value.size()); + PySafeObject seq(PyTuple_New(n)); + for (Py_ssize_t i = 0; i < n; i++) + { + bool elem = value[i]; + PyObject* item = pyopencv_from(elem); + // If item can't be assigned - PyTuple_SetItem raises exception and returns -1. + if (!item || PyTuple_SetItem(seq, i, item) == -1) + { + return NULL; + } + } + return seq.release(); +} + +namespace traits { + +template +struct BooleanConstant +{ + static const bool value = Value; + typedef BooleanConstant type; +}; + +typedef BooleanConstant TrueType; +typedef BooleanConstant FalseType; + +template +struct VoidType { + typedef void type; +}; + +template +struct IsRepresentableAsMatDataType : FalseType +{ +}; + +template +struct IsRepresentableAsMatDataType::channel_type>::type> : TrueType +{ +}; + +} // namespace traits + +template +struct pyopencvVecConverter +{ + typedef typename std::vector::iterator VecIt; + + static bool to(PyObject* obj, std::vector& value, const ArgInfo& info) + { + if (!PyArray_Check(obj)) + { + return pyopencv_to_generic_vec(obj, value, info); + } + // If user passed an array it is possible to make faster conversions in several cases + PyArrayObject* array_obj = reinterpret_cast(obj); + const NPY_TYPES target_type = asNumpyType(); + const NPY_TYPES source_type = static_cast(PyArray_TYPE(array_obj)); + if (target_type == NPY_OBJECT) + { + // Non-planar arrays representing objects (e.g. array of N Rect is an array of shape Nx4) have NPY_OBJECT + // as their target type. + return pyopencv_to_generic_vec(obj, value, info); + } + if (PyArray_NDIM(array_obj) > 1) + { + failmsg("Can't parse %dD array as '%s' vector argument", PyArray_NDIM(array_obj), info.name); + return false; + } + if (target_type != source_type) + { + // Source type requires conversion + // Allowed conversions for target type is handled in the corresponding pyopencv_to function + return pyopencv_to_generic_vec(obj, value, info); + } + // For all other cases, all array data can be directly copied to std::vector data + // Simple `memcpy` is not possible because NumPy array can reference a slice of the bigger array: + // ``` + // arr = np.ones((8, 4, 5), dtype=np.int32) + // convertible_to_vector_of_int = arr[:, 0, 1] + // ``` + value.resize(static_cast(PyArray_SIZE(array_obj))); + const npy_intp item_step = PyArray_STRIDE(array_obj, 0) / PyArray_ITEMSIZE(array_obj); + const Tp* data_ptr = static_cast(PyArray_DATA(array_obj)); + for (VecIt it = value.begin(); it != value.end(); ++it, data_ptr += item_step) { + *it = *data_ptr; + } + return true; + } + + static PyObject* from(const std::vector& value) + { + if (value.empty()) + { + return PyTuple_New(0); + } + return from(value, ::traits::IsRepresentableAsMatDataType()); + } + +private: + static PyObject* from(const std::vector& value, ::traits::FalseType) + { + // Underlying type is not representable as Mat Data Type + return pyopencv_from_generic_vec(value); + } + + static PyObject* from(const std::vector& value, ::traits::TrueType) + { + // Underlying type is representable as Mat Data Type, so faster return type is available + typedef cv::DataType DType; + typedef typename DType::channel_type UnderlyingArrayType; + + // If Mat is always exposed as NumPy array this code path can be reduced to the following snipped: + // Mat src(value); + // PyObject* array = pyopencv_from(src); + // return PyArray_Squeeze(reinterpret_cast(array)); + // This puts unnecessary restrictions on Mat object those might be avoided without losing the performance. + // Moreover, this version is a bit faster, because it doesn't create temporary objects with reference counting. + + const NPY_TYPES target_type = asNumpyType(); + const int cols = DType::channels; + PyObject* array = NULL; + if (cols == 1) + { + npy_intp dims = static_cast(value.size()); + array = PyArray_SimpleNew(1, &dims, target_type); + } + else + { + npy_intp dims[2] = {static_cast(value.size()), cols}; + array = PyArray_SimpleNew(2, dims, target_type); + } + if(!array) + { + // NumPy arrays with shape (N, 1) and (N) are not equal, so correct error message should distinguish + // them too. + cv::String shape; + if (cols > 1) + { + shape = cv::format("(%d x %d)", static_cast(value.size()), cols); + } + else + { + shape = cv::format("(%d)", static_cast(value.size())); + } + const cv::String error_message = cv::format("Can't allocate NumPy array for vector with dtype=%d and shape=%s", + static_cast(target_type), shape.c_str()); + emit_failmsg(PyExc_MemoryError, error_message.c_str()); + return array; + } + // Fill the array + PyArrayObject* array_obj = reinterpret_cast(array); + UnderlyingArrayType* array_data = static_cast(PyArray_DATA(array_obj)); + // if Tp is representable as Mat DataType, so the following cast is pretty safe... + const UnderlyingArrayType* value_data = reinterpret_cast(value.data()); + memcpy(array_data, value_data, sizeof(UnderlyingArrayType) * value.size() * static_cast(cols)); + return array; + } +}; + +// --- tuple +template +inline typename std::enable_if::type +convert_to_python_tuple(const std::tuple&, PyObject*) { } + +template +inline typename std::enable_if::type +convert_to_python_tuple(const std::tuple& cpp_tuple, PyObject* py_tuple) +{ + PyObject* item = pyopencv_from(std::get(cpp_tuple)); + + if (!item) + return; + + PyTuple_SetItem(py_tuple, I, item); + convert_to_python_tuple(cpp_tuple, py_tuple); +} + +template +PyObject* pyopencv_from(const std::tuple& cpp_tuple) +{ + size_t size = sizeof...(Ts); + PyObject* py_tuple = PyTuple_New(size); + convert_to_python_tuple(cpp_tuple, py_tuple); + size_t actual_size = PyTuple_Size(py_tuple); + + if (actual_size < size) + { + Py_DECREF(py_tuple); + return NULL; + } + + return py_tuple; +} + +#endif // CV2_CONVERT_HPP diff --git a/modules/python/src2/cv2_highgui.cpp b/modules/python/src2/cv2_highgui.cpp new file mode 100644 index 0000000000..4262043ce7 --- /dev/null +++ b/modules/python/src2/cv2_highgui.cpp @@ -0,0 +1,184 @@ +#include "cv2_highgui.hpp" + +#ifdef HAVE_OPENCV_HIGHGUI + +#include "cv2_util.hpp" +#include "opencv2/highgui.hpp" +#include + +using namespace cv; + +//====================================================================================================================== + +static void OnMouse(int event, int x, int y, int flags, void* param) +{ + PyGILState_STATE gstate; + gstate = PyGILState_Ensure(); + + PyObject *o = (PyObject*)param; + PyObject *args = Py_BuildValue("iiiiO", event, x, y, flags, PyTuple_GetItem(o, 1)); + + PyObject *r = PyObject_Call(PyTuple_GetItem(o, 0), args, NULL); + if (r == NULL) + PyErr_Print(); + else + Py_DECREF(r); + Py_DECREF(args); + PyGILState_Release(gstate); +} + +PyObject *pycvSetMouseCallback(PyObject*, PyObject *args, PyObject *kw) +{ + const char *keywords[] = { "window_name", "on_mouse", "param", NULL }; + char* name; + PyObject *on_mouse; + PyObject *param = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kw, "sO|O", (char**)keywords, &name, &on_mouse, ¶m)) + return NULL; + if (!PyCallable_Check(on_mouse)) { + PyErr_SetString(PyExc_TypeError, "on_mouse must be callable"); + return NULL; + } + if (param == NULL) { + param = Py_None; + } + PyObject* py_callback_info = Py_BuildValue("OO", on_mouse, param); + static std::map registered_callbacks; + std::map::iterator i = registered_callbacks.find(name); + if (i != registered_callbacks.end()) + { + Py_DECREF(i->second); + i->second = py_callback_info; + } + else + { + registered_callbacks.insert(std::pair(std::string(name), py_callback_info)); + } + ERRWRAP2(setMouseCallback(name, OnMouse, py_callback_info)); + Py_RETURN_NONE; +} + +//====================================================================================================================== + +static void OnChange(int pos, void *param) +{ + PyGILState_STATE gstate; + gstate = PyGILState_Ensure(); + + PyObject *o = (PyObject*)param; + PyObject *args = Py_BuildValue("(i)", pos); + PyObject *r = PyObject_Call(PyTuple_GetItem(o, 0), args, NULL); + if (r == NULL) + PyErr_Print(); + else + Py_DECREF(r); + Py_DECREF(args); + PyGILState_Release(gstate); +} + +// workaround for #20408, use nullptr, set value later +static int _createTrackbar(const String &trackbar_name, const String &window_name, int value, int count, + TrackbarCallback onChange, PyObject* py_callback_info) +{ + int n = createTrackbar(trackbar_name, window_name, NULL, count, onChange, py_callback_info); + setTrackbarPos(trackbar_name, window_name, value); + return n; +} + +PyObject *pycvCreateTrackbar(PyObject*, PyObject *args) +{ + PyObject *on_change; + char* trackbar_name; + char* window_name; + int value; + int count; + + if (!PyArg_ParseTuple(args, "ssiiO", &trackbar_name, &window_name, &value, &count, &on_change)) + return NULL; + if (!PyCallable_Check(on_change)) { + PyErr_SetString(PyExc_TypeError, "on_change must be callable"); + return NULL; + } + PyObject* py_callback_info = Py_BuildValue("OO", on_change, Py_None); + std::string name = std::string(window_name) + ":" + std::string(trackbar_name); + static std::map registered_callbacks; + std::map::iterator i = registered_callbacks.find(name); + if (i != registered_callbacks.end()) + { + Py_DECREF(i->second); + i->second = py_callback_info; + } + else + { + registered_callbacks.insert(std::pair(name, py_callback_info)); + } + ERRWRAP2(_createTrackbar(trackbar_name, window_name, value, count, OnChange, py_callback_info)); + Py_RETURN_NONE; +} + +//====================================================================================================================== + +static void OnButtonChange(int state, void *param) +{ + PyGILState_STATE gstate; + gstate = PyGILState_Ensure(); + + PyObject *o = (PyObject*)param; + PyObject *args; + if(PyTuple_GetItem(o, 1) != NULL) + { + args = Py_BuildValue("(iO)", state, PyTuple_GetItem(o,1)); + } + else + { + args = Py_BuildValue("(i)", state); + } + + PyObject *r = PyObject_Call(PyTuple_GetItem(o, 0), args, NULL); + if (r == NULL) + PyErr_Print(); + else + Py_DECREF(r); + Py_DECREF(args); + PyGILState_Release(gstate); +} + +PyObject *pycvCreateButton(PyObject*, PyObject *args, PyObject *kw) +{ + const char* keywords[] = {"buttonName", "onChange", "userData", "buttonType", "initialButtonState", NULL}; + PyObject *on_change; + PyObject *userdata = NULL; + char* button_name; + int button_type = 0; + int initial_button_state = 0; + + if (!PyArg_ParseTupleAndKeywords(args, kw, "sO|Oii", (char**)keywords, &button_name, &on_change, &userdata, &button_type, &initial_button_state)) + return NULL; + if (!PyCallable_Check(on_change)) { + PyErr_SetString(PyExc_TypeError, "onChange must be callable"); + return NULL; + } + if (userdata == NULL) { + userdata = Py_None; + } + + PyObject* py_callback_info = Py_BuildValue("OO", on_change, userdata); + std::string name(button_name); + + static std::map registered_callbacks; + std::map::iterator i = registered_callbacks.find(name); + if (i != registered_callbacks.end()) + { + Py_DECREF(i->second); + i->second = py_callback_info; + } + else + { + registered_callbacks.insert(std::pair(name, py_callback_info)); + } + ERRWRAP2(createButton(button_name, OnButtonChange, py_callback_info, button_type, initial_button_state != 0)); + Py_RETURN_NONE; +} + +#endif // HAVE_OPENCV_HIGHGUI diff --git a/modules/python/src2/cv2_highgui.hpp b/modules/python/src2/cv2_highgui.hpp new file mode 100644 index 0000000000..97e08375d2 --- /dev/null +++ b/modules/python/src2/cv2_highgui.hpp @@ -0,0 +1,14 @@ +#ifndef CV2_HIGHGUI_HPP +#define CV2_HIGHGUI_HPP + +#include "cv2.hpp" +#include "opencv2/opencv_modules.hpp" + +#ifdef HAVE_OPENCV_HIGHGUI +PyObject *pycvSetMouseCallback(PyObject*, PyObject *args, PyObject *kw); +// workaround for #20408, use nullptr, set value later +PyObject *pycvCreateTrackbar(PyObject*, PyObject *args); +PyObject *pycvCreateButton(PyObject*, PyObject *args, PyObject *kw); +#endif + +#endif // CV2_HIGHGUI_HPP diff --git a/modules/python/src2/cv2_numpy.cpp b/modules/python/src2/cv2_numpy.cpp new file mode 100644 index 0000000000..98938db34c --- /dev/null +++ b/modules/python/src2/cv2_numpy.cpp @@ -0,0 +1,73 @@ +// must be defined before importing numpy headers +// https://numpy.org/doc/1.17/reference/c-api.array.html#importing-the-api +#define NO_IMPORT_ARRAY +#define PY_ARRAY_UNIQUE_SYMBOL opencv_ARRAY_API + +#include "cv2_numpy.hpp" +#include "cv2_util.hpp" + +NumpyAllocator g_numpyAllocator; + +using namespace cv; + +UMatData* NumpyAllocator::allocate(PyObject* o, int dims, const int* sizes, int type, size_t* step) const +{ + UMatData* u = new UMatData(this); + u->data = u->origdata = (uchar*)PyArray_DATA((PyArrayObject*) o); + npy_intp* _strides = PyArray_STRIDES((PyArrayObject*) o); + for( int i = 0; i < dims - 1; i++ ) + step[i] = (size_t)_strides[i]; + step[dims-1] = CV_ELEM_SIZE(type); + u->size = sizes[0]*step[0]; + u->userdata = o; + return u; +} + +UMatData* NumpyAllocator::allocate(int dims0, const int* sizes, int type, void* data, size_t* step, AccessFlag flags, UMatUsageFlags usageFlags) const +{ + if( data != 0 ) + { + // issue #6969: CV_Error(Error::StsAssert, "The data should normally be NULL!"); + // probably this is safe to do in such extreme case + return stdAllocator->allocate(dims0, sizes, type, data, step, flags, usageFlags); + } + PyEnsureGIL gil; + + int depth = CV_MAT_DEPTH(type); + int cn = CV_MAT_CN(type); + const int f = (int)(sizeof(size_t)/8); + int typenum = depth == CV_8U ? NPY_UBYTE : depth == CV_8S ? NPY_BYTE : + depth == CV_16U ? NPY_USHORT : depth == CV_16S ? NPY_SHORT : + depth == CV_32S ? NPY_INT : depth == CV_32F ? NPY_FLOAT : + depth == CV_64F ? NPY_DOUBLE : f*NPY_ULONGLONG + (f^1)*NPY_UINT; + int i, dims = dims0; + cv::AutoBuffer _sizes(dims + 1); + for( i = 0; i < dims; i++ ) + _sizes[i] = sizes[i]; + if( cn > 1 ) + _sizes[dims++] = cn; + PyObject* o = PyArray_SimpleNew(dims, _sizes.data(), typenum); + if(!o) + CV_Error_(Error::StsError, ("The numpy array of typenum=%d, ndims=%d can not be created", typenum, dims)); + return allocate(o, dims0, sizes, type, step); +} + +bool NumpyAllocator::allocate(UMatData* u, AccessFlag accessFlags, UMatUsageFlags usageFlags) const +{ + return stdAllocator->allocate(u, accessFlags, usageFlags); +} + +void NumpyAllocator::deallocate(UMatData* u) const +{ + if(!u) + return; + PyEnsureGIL gil; + CV_Assert(u->urefcount >= 0); + CV_Assert(u->refcount >= 0); + if(u->refcount == 0) + { + PyObject* o = (PyObject*)u->userdata; + Py_XDECREF(o); + delete u; + } +} diff --git a/modules/python/src2/cv2_numpy.hpp b/modules/python/src2/cv2_numpy.hpp new file mode 100644 index 0000000000..7c386c7dbc --- /dev/null +++ b/modules/python/src2/cv2_numpy.hpp @@ -0,0 +1,217 @@ +#ifndef CV2_NUMPY_HPP +#define CV2_NUMPY_HPP + +#include "cv2.hpp" +#include "opencv2/core.hpp" + +class NumpyAllocator : public cv::MatAllocator +{ +public: + NumpyAllocator() { stdAllocator = cv::Mat::getStdAllocator(); } + ~NumpyAllocator() {} + + cv::UMatData* allocate(PyObject* o, int dims, const int* sizes, int type, size_t* step) const; + cv::UMatData* allocate(int dims0, const int* sizes, int type, void* data, size_t* step, cv::AccessFlag flags, cv::UMatUsageFlags usageFlags) const CV_OVERRIDE; + bool allocate(cv::UMatData* u, cv::AccessFlag accessFlags, cv::UMatUsageFlags usageFlags) const CV_OVERRIDE; + void deallocate(cv::UMatData* u) const CV_OVERRIDE; + + const cv::MatAllocator* stdAllocator; +}; + +extern NumpyAllocator g_numpyAllocator; + +//====================================================================================================================== + +// HACK(?): function from cv2_util.hpp +extern int failmsg(const char *fmt, ...); + +namespace { + +template +NPY_TYPES asNumpyType() +{ + return NPY_OBJECT; +} + +template<> +NPY_TYPES asNumpyType() +{ + return NPY_BOOL; +} + +#define CV_GENERATE_INTEGRAL_TYPE_NPY_CONVERSION(src, dst) \ + template<> \ + NPY_TYPES asNumpyType() \ + { \ + return NPY_##dst; \ + } \ + template<> \ + NPY_TYPES asNumpyType() \ + { \ + return NPY_U##dst; \ + } + +CV_GENERATE_INTEGRAL_TYPE_NPY_CONVERSION(int8_t, INT8); + +CV_GENERATE_INTEGRAL_TYPE_NPY_CONVERSION(int16_t, INT16); + +CV_GENERATE_INTEGRAL_TYPE_NPY_CONVERSION(int32_t, INT32); + +CV_GENERATE_INTEGRAL_TYPE_NPY_CONVERSION(int64_t, INT64); + +#undef CV_GENERATE_INTEGRAL_TYPE_NPY_CONVERSION + +template<> +NPY_TYPES asNumpyType() +{ + return NPY_FLOAT; +} + +template<> +NPY_TYPES asNumpyType() +{ + return NPY_DOUBLE; +} + +template +PyArray_Descr* getNumpyTypeDescriptor() +{ + return PyArray_DescrFromType(asNumpyType()); +} + +template <> +PyArray_Descr* getNumpyTypeDescriptor() +{ +#if SIZE_MAX == ULONG_MAX + return PyArray_DescrFromType(NPY_ULONG); +#elif SIZE_MAX == ULLONG_MAX + return PyArray_DescrFromType(NPY_ULONGLONG); +#else + return PyArray_DescrFromType(NPY_UINT); +#endif +} + +template +bool isRepresentable(U value) { + return (std::numeric_limits::min() <= value) && (value <= std::numeric_limits::max()); +} + +template +bool canBeSafelyCasted(PyObject* obj, PyArray_Descr* to) +{ + return PyArray_CanCastTo(PyArray_DescrFromScalar(obj), to) != 0; +} + + +template<> +bool canBeSafelyCasted(PyObject* obj, PyArray_Descr* to) +{ + PyArray_Descr* from = PyArray_DescrFromScalar(obj); + if (PyArray_CanCastTo(from, to)) + { + return true; + } + else + { + // False negative scenarios: + // - Signed input is positive so it can be safely cast to unsigned output + // - Input has wider limits but value is representable within output limits + // - All the above + if (PyDataType_ISSIGNED(from)) + { + int64_t input = 0; + PyArray_CastScalarToCtype(obj, &input, getNumpyTypeDescriptor()); + return (input >= 0) && isRepresentable(static_cast(input)); + } + else + { + uint64_t input = 0; + PyArray_CastScalarToCtype(obj, &input, getNumpyTypeDescriptor()); + return isRepresentable(input); + } + return false; + } +} + + +template +bool parseNumpyScalar(PyObject* obj, T& value) +{ + if (PyArray_CheckScalar(obj)) + { + // According to the numpy documentation: + // There are 21 statically-defined PyArray_Descr objects for the built-in data-types + // So descriptor pointer is not owning. + PyArray_Descr* to = getNumpyTypeDescriptor(); + if (canBeSafelyCasted(obj, to)) + { + PyArray_CastScalarToCtype(obj, &value, to); + return true; + } + } + return false; +} + + +struct SafeSeqItem +{ + PyObject * item; + SafeSeqItem(PyObject *obj, size_t idx) { item = PySequence_GetItem(obj, idx); } + ~SafeSeqItem() { Py_XDECREF(item); } + +private: + SafeSeqItem(const SafeSeqItem&); // = delete + SafeSeqItem& operator=(const SafeSeqItem&); // = delete +}; + +template +class RefWrapper +{ +public: + RefWrapper(T& item) : item_(item) {} + + T& get() CV_NOEXCEPT { return item_; } + +private: + T& item_; +}; + +// In order to support this conversion on 3.x branch - use custom reference_wrapper +// and C-style array instead of std::array +template +bool parseSequence(PyObject* obj, RefWrapper (&value)[N], const ArgInfo& info) +{ + if (!obj || obj == Py_None) + { + return true; + } + if (!PySequence_Check(obj)) + { + failmsg("Can't parse '%s'. Input argument doesn't provide sequence " + "protocol", info.name); + return false; + } + const std::size_t sequenceSize = PySequence_Size(obj); + if (sequenceSize != N) + { + failmsg("Can't parse '%s'. Expected sequence length %lu, got %lu", + info.name, N, sequenceSize); + return false; + } + for (std::size_t i = 0; i < N; ++i) + { + SafeSeqItem seqItem(obj, i); + if (!pyopencv_to(seqItem.item, value[i].get(), info)) + { + failmsg("Can't parse '%s'. Sequence item with index %lu has a " + "wrong type", info.name, i); + return false; + } + } + return true; +} + +} // namespace + + +#endif // CV2_NUMPY_HPP diff --git a/modules/python/src2/cv2_util.cpp b/modules/python/src2/cv2_util.cpp new file mode 100644 index 0000000000..aca65eae54 --- /dev/null +++ b/modules/python/src2/cv2_util.cpp @@ -0,0 +1,178 @@ +#include "cv2_util.hpp" +#include "opencv2/core.hpp" +#include "opencv2/core/utils/configuration.private.hpp" +#include "opencv2/core/utils/logger.hpp" + +PyObject* opencv_error = NULL; +cv::TLSData > conversionErrorsTLS; + +using namespace cv; + +//====================================================================================================================== + +bool isPythonBindingsDebugEnabled() +{ + static bool param_debug = cv::utils::getConfigurationParameterBool("OPENCV_PYTHON_DEBUG", false); + return param_debug; +} + +void emit_failmsg(PyObject * exc, const char *msg) +{ + static bool param_debug = isPythonBindingsDebugEnabled(); + if (param_debug) + { + CV_LOG_WARNING(NULL, "Bindings conversion failed: " << msg); + } + PyErr_SetString(exc, msg); +} + +int failmsg(const char *fmt, ...) +{ + char str[1000]; + + va_list ap; + va_start(ap, fmt); + vsnprintf(str, sizeof(str), fmt, ap); + va_end(ap); + + emit_failmsg(PyExc_TypeError, str); + return 0; +} + +PyObject* failmsgp(const char *fmt, ...) +{ + char str[1000]; + + va_list ap; + va_start(ap, fmt); + vsnprintf(str, sizeof(str), fmt, ap); + va_end(ap); + + emit_failmsg(PyExc_TypeError, str); + return 0; +} + +void pyRaiseCVException(const cv::Exception &e) +{ + PyObject_SetAttrString(opencv_error, "file", PyString_FromString(e.file.c_str())); + PyObject_SetAttrString(opencv_error, "func", PyString_FromString(e.func.c_str())); + PyObject_SetAttrString(opencv_error, "line", PyInt_FromLong(e.line)); + PyObject_SetAttrString(opencv_error, "code", PyInt_FromLong(e.code)); + PyObject_SetAttrString(opencv_error, "msg", PyString_FromString(e.msg.c_str())); + PyObject_SetAttrString(opencv_error, "err", PyString_FromString(e.err.c_str())); + PyErr_SetString(opencv_error, e.what()); +} + +//====================================================================================================================== + +void pyRaiseCVOverloadException(const std::string& functionName) +{ + const std::vector& conversionErrors = conversionErrorsTLS.getRef(); + const std::size_t conversionErrorsCount = conversionErrors.size(); + if (conversionErrorsCount > 0) + { + // In modern std libraries small string optimization is used = no dynamic memory allocations, + // but it can be applied only for string with length < 18 symbols (in GCC) + const std::string bullet = "\n - "; + + // Estimate required buffer size - save dynamic memory allocations = faster + std::size_t requiredBufferSize = bullet.size() * conversionErrorsCount; + for (std::size_t i = 0; i < conversionErrorsCount; ++i) + { + requiredBufferSize += conversionErrors[i].size(); + } + + // Only string concatenation is required so std::string is way faster than + // std::ostringstream + std::string errorMessage("Overload resolution failed:"); + errorMessage.reserve(errorMessage.size() + requiredBufferSize); + for (std::size_t i = 0; i < conversionErrorsCount; ++i) + { + errorMessage += bullet; + errorMessage += conversionErrors[i]; + } + cv::Exception exception(Error::StsBadArg, errorMessage, functionName, "", -1); + pyRaiseCVException(exception); + } + else + { + cv::Exception exception(Error::StsInternal, "Overload resolution failed, but no errors reported", + functionName, "", -1); + pyRaiseCVException(exception); + } +} + +void pyPopulateArgumentConversionErrors() +{ + if (PyErr_Occurred()) + { + PySafeObject exception_type; + PySafeObject exception_value; + PySafeObject exception_traceback; + PyErr_Fetch(exception_type, exception_value, exception_traceback); + PyErr_NormalizeException(exception_type, exception_value, + exception_traceback); + + PySafeObject exception_message(PyObject_Str(exception_value)); + std::string message; + getUnicodeString(exception_message, message); +#ifdef CV_CXX11 + conversionErrorsTLS.getRef().push_back(std::move(message)); +#else + conversionErrorsTLS.getRef().push_back(message); +#endif + } +} + +//====================================================================================================================== + +static int OnError(int status, const char *func_name, const char *err_msg, const char *file_name, int line, void *userdata) +{ + PyGILState_STATE gstate; + gstate = PyGILState_Ensure(); + + PyObject *on_error = (PyObject*)userdata; + PyObject *args = Py_BuildValue("isssi", status, func_name, err_msg, file_name, line); + + PyObject *r = PyObject_Call(on_error, args, NULL); + if (r == NULL) { + PyErr_Print(); + } else { + Py_DECREF(r); + } + + Py_DECREF(args); + PyGILState_Release(gstate); + + return 0; // The return value isn't used +} + +PyObject *pycvRedirectError(PyObject*, PyObject *args, PyObject *kw) +{ + const char *keywords[] = { "on_error", NULL }; + PyObject *on_error; + + if (!PyArg_ParseTupleAndKeywords(args, kw, "O", (char**)keywords, &on_error)) + return NULL; + + if ((on_error != Py_None) && !PyCallable_Check(on_error)) { + PyErr_SetString(PyExc_TypeError, "on_error must be callable"); + return NULL; + } + + // Keep track of the previous handler parameter, so we can decref it when no longer used + static PyObject* last_on_error = NULL; + if (last_on_error) { + Py_DECREF(last_on_error); + last_on_error = NULL; + } + + if (on_error == Py_None) { + ERRWRAP2(redirectError(NULL)); + } else { + last_on_error = on_error; + Py_INCREF(last_on_error); + ERRWRAP2(redirectError(OnError, last_on_error)); + } + Py_RETURN_NONE; +} diff --git a/modules/python/src2/cv2_util.hpp b/modules/python/src2/cv2_util.hpp new file mode 100644 index 0000000000..3662ffcd4d --- /dev/null +++ b/modules/python/src2/cv2_util.hpp @@ -0,0 +1,134 @@ +#ifndef CV2_UTIL_HPP +#define CV2_UTIL_HPP + +#include "cv2.hpp" +#include "opencv2/core.hpp" +#include "opencv2/core/utils/tls.hpp" +#include +#include + +//====================================================================================================================== + +bool isPythonBindingsDebugEnabled(); +void emit_failmsg(PyObject * exc, const char *msg); +int failmsg(const char *fmt, ...); +PyObject* failmsgp(const char *fmt, ...);; + +//====================================================================================================================== + +class PyAllowThreads +{ +public: + PyAllowThreads() : _state(PyEval_SaveThread()) {} + ~PyAllowThreads() + { + PyEval_RestoreThread(_state); + } +private: + PyThreadState* _state; +}; + +class PyEnsureGIL +{ +public: + PyEnsureGIL() : _state(PyGILState_Ensure()) {} + ~PyEnsureGIL() + { + PyGILState_Release(_state); + } +private: + PyGILState_STATE _state; +}; + +/** + * Light weight RAII wrapper for `PyObject*` owning references. + * In comparisson to C++11 `std::unique_ptr` with custom deleter, it provides + * implicit conversion functions that might be useful to initialize it with + * Python functions those returns owning references through the `PyObject**` + * e.g. `PyErr_Fetch` or directly pass it to functions those want to borrow + * reference to object (doesn't extend object lifetime) e.g. `PyObject_Str`. + */ +class PySafeObject +{ +public: + PySafeObject() : obj_(NULL) {} + + explicit PySafeObject(PyObject* obj) : obj_(obj) {} + + ~PySafeObject() + { + Py_CLEAR(obj_); + } + + operator PyObject*() + { + return obj_; + } + + operator PyObject**() + { + return &obj_; + } + + PyObject* release() + { + PyObject* obj = obj_; + obj_ = NULL; + return obj; + } + +private: + PyObject* obj_; + + // Explicitly disable copy operations + PySafeObject(const PySafeObject*); // = delete + PySafeObject& operator=(const PySafeObject&); // = delete +}; + +//====================================================================================================================== + +extern PyObject* opencv_error; + +void pyRaiseCVException(const cv::Exception &e); + +#define ERRWRAP2(expr) \ +try \ +{ \ + PyAllowThreads allowThreads; \ + expr; \ +} \ +catch (const cv::Exception &e) \ +{ \ + pyRaiseCVException(e); \ + return 0; \ +} \ +catch (const std::exception &e) \ +{ \ + PyErr_SetString(opencv_error, e.what()); \ + return 0; \ +} \ +catch (...) \ +{ \ + PyErr_SetString(opencv_error, "Unknown C++ exception from OpenCV code"); \ + return 0; \ +} + +//====================================================================================================================== + +extern cv::TLSData > conversionErrorsTLS; + +inline void pyPrepareArgumentConversionErrorsStorage(std::size_t size) +{ + std::vector& conversionErrors = conversionErrorsTLS.getRef(); + conversionErrors.clear(); + conversionErrors.reserve(size); +} + +void pyRaiseCVOverloadException(const std::string& functionName); +void pyPopulateArgumentConversionErrors(); + +//====================================================================================================================== + +PyObject *pycvRedirectError(PyObject*, PyObject *args, PyObject *kw); + +#endif // CV2_UTIL_HPP diff --git a/modules/python/src2/pycompat.hpp b/modules/python/src2/pycompat.hpp index 2650554b3f..03379ec956 100644 --- a/modules/python/src2/pycompat.hpp +++ b/modules/python/src2/pycompat.hpp @@ -44,6 +44,8 @@ #ifndef __PYCOMPAT_HPP__ #define __PYCOMPAT_HPP__ +#include + #if PY_MAJOR_VERSION >= 3 // Python3 treats all ints as longs, PyInt_X functions have been removed. From f09a577ab5795451835526383be75f20f88f4c8f Mon Sep 17 00:00:00 2001 From: Sinitsina Maria Date: Sun, 5 Dec 2021 17:58:44 +0300 Subject: [PATCH 135/226] add OpenCV audio reading --- samples/dnn/speech_recognition.py | 113 +++++++++++++++++++++++------- 1 file changed, 87 insertions(+), 26 deletions(-) diff --git a/samples/dnn/speech_recognition.py b/samples/dnn/speech_recognition.py index 025607edab..7bc424b37c 100644 --- a/samples/dnn/speech_recognition.py +++ b/samples/dnn/speech_recognition.py @@ -2,7 +2,6 @@ import numpy as np import cv2 as cv import argparse import os -import soundfile as sf # Temporary import to load audio files ''' You can download the converted onnx model from https://drive.google.com/drive/folders/1wLtxyao4ItAg8tt4Sb63zt6qXzhcQoR6?usp=sharing @@ -399,11 +398,6 @@ def predict(features, net, decoder): decoder : Decoder object return : Predicted text ''' - # This is a workaround https://github.com/opencv/opencv/issues/19091 - # expanding 1 dimentions allows us to pass it to the network - # from python. This should be resolved in the future. - features = np.expand_dims(features,axis=3) - # make prediction net.setInput(features) output = net.forward() @@ -412,6 +406,63 @@ def predict(features, net, decoder): prediction = decoder.decode(output.squeeze(0)) return prediction[0] +def readAudioFile(file, audioStream): + cap = cv.VideoCapture(file) + samplingRate = 16000 + params = np.asarray([cv.CAP_PROP_AUDIO_STREAM, audioStream, + cv.CAP_PROP_VIDEO_STREAM, -1, + cv.CAP_PROP_AUDIO_DATA_DEPTH, cv.CV_32F, + cv.CAP_PROP_AUDIO_SAMPLES_PER_SECOND, samplingRate + ]) + cap.open(file, cv.CAP_ANY, params) + if cap.isOpened() is False: + print("Error : Can't read audio file:", file, "with audioStream = ", audioStream) + return + audioBaseIndex = int (cap.get(cv.CAP_PROP_AUDIO_BASE_INDEX)) + inputAudio = [] + while(1): + if (cap.grab()): + frame = np.asarray([]) + frame = cap.retrieve(frame, audioBaseIndex) + for i in range(len(frame[1][0])): + inputAudio.append(frame[1][0][i]) + else: + break + inputAudio = np.asarray(inputAudio, dtype=np.float64) + return inputAudio, samplingRate + +def readAudioMicrophone(microTime): + cap = cv.VideoCapture() + samplingRate = 16000 + params = np.asarray([cv.CAP_PROP_AUDIO_STREAM, 0, + cv.CAP_PROP_VIDEO_STREAM, -1, + cv.CAP_PROP_AUDIO_DATA_DEPTH, cv.CV_32F, + cv.CAP_PROP_AUDIO_SAMPLES_PER_SECOND, samplingRate + ]) + cap.open(0, cv.CAP_ANY, params) + if cap.isOpened() is False: + print("Error: Can't open microphone") + print("Error: problems with audio reading, check input arguments") + return + audioBaseIndex = int(cap.get(cv.CAP_PROP_AUDIO_BASE_INDEX)) + cvTickFreq = cv.getTickFrequency() + sysTimeCurr = cv.getTickCount() + sysTimePrev = sysTimeCurr + inputAudio = [] + while ((sysTimeCurr - sysTimePrev) / cvTickFreq < microTime): + if (cap.grab()): + frame = np.asarray([]) + frame = cap.retrieve(frame, audioBaseIndex) + for i in range(len(frame[1][0])): + inputAudio.append(frame[1][0][i]) + sysTimeCurr = cv.getTickCount() + else: + print("Error: Grab error") + break + inputAudio = np.asarray(inputAudio, dtype=np.float64) + print("Number of samples: ", len(inputAudio)) + return inputAudio, samplingRate + if __name__ == '__main__': # Computation backends supported by layers @@ -421,7 +472,10 @@ if __name__ == '__main__': parser = argparse.ArgumentParser(description='This script runs Jasper Speech recognition model', formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('--input_audio', type=str, required=True, help='Path to input audio file. OR Path to a txt file with relative path to multiple audio files in different lines') + parser.add_argument('--input_type', type=str, required=True, help='file or microphone') + parser.add_argument('--micro_time', type=int, default=15, help='Duration of microphone work in seconds. Must be more than 6 sec') + parser.add_argument('--input_audio', type=str, help='Path to input audio file. OR Path to a txt file with relative path to multiple audio files in different lines') + parser.add_argument('--audio_stream', type=int, default=0, help='CAP_PROP_AUDIO_STREAM value') parser.add_argument('--show_spectrogram', action='store_true', help='Whether to show a spectrogram of the input audio.') parser.add_argument('--model', type=str, default='jasper.onnx', help='Path to the onnx file of Jasper. default="jasper.onnx"') parser.add_argument('--output', type=str, help='Path to file where recognized audio transcript must be saved. Leave this to print on console.') @@ -442,28 +496,35 @@ if __name__ == '__main__': raise OSError("Input audio file does not exist") if not os.path.isfile(args.model): raise OSError("Jasper model file does not exist") - if args.input_audio.endswith('.txt'): - with open(args.input_audio) as f: - content = f.readlines() - content = [x.strip() for x in content] - audio_file_paths = content - for audio_file_path in audio_file_paths: - if not os.path.isfile(audio_file_path): - raise OSError("Audio file({audio_file_path}) does not exist") - else: - audio_file_paths = [args.input_audio] - audio_file_paths = [os.path.abspath(x) for x in audio_file_paths] - # Read audio Files features = [] - try: + if args.input_type == "file": + if args.input_audio.endswith('.txt'): + with open(args.input_audio) as f: + content = f.readlines() + content = [x.strip() for x in content] + audio_file_paths = content + for audio_file_path in audio_file_paths: + if not os.path.isfile(audio_file_path): + raise OSError("Audio file({audio_file_path}) does not exist") + else: + audio_file_paths = [args.input_audio] + audio_file_paths = [os.path.abspath(x) for x in audio_file_paths] + + # Read audio Files for audio_file_path in audio_file_paths: - audio = sf.read(audio_file_path) - # If audio is stereo, just take one channel. - X = audio[0] if audio[0].ndim==1 else audio[0][:,0] - features.append(X) - except: - raise Exception(f"Soundfile cannot read {args.input_audio}. Try a different format") + audio = readAudioFile(audio_file_path, args.audio_stream) + if audio is None: + raise Exception(f"Can't read {args.input_audio}. Try a different format") + features.append(audio[0]) + elif args.input_type == "microphone": + # Read audio from microphone + audio = readAudioMicrophone(args.micro_time) + if audio is None: + raise Exception(f"Can't open microphone. Try a different format") + features.append(audio[0]) + else: + raise Exception(f"input_type {args.input_type} doesn't exist. Please enter 'file' or 'microphone'") # Get Filterbank Features feature_extractor = FilterbankFeatures() From a4d6bcba0998bdb30eac2e6d333d389906617bed Mon Sep 17 00:00:00 2001 From: Anna Khakimova Date: Mon, 6 Dec 2021 13:59:26 +0300 Subject: [PATCH 136/226] GAPI Fluid: Enable dynamic dispatching for AbsDiffC kernel. --- .../gapi/perf/common/gapi_core_perf_tests.hpp | 2 +- .../perf/common/gapi_core_perf_tests_inl.hpp | 12 +- .../perf/cpu/gapi_core_perf_tests_cpu.cpp | 3 +- .../perf/cpu/gapi_core_perf_tests_fluid.cpp | 7 +- .../perf/gpu/gapi_core_perf_tests_gpu.cpp | 3 +- .../gapi/src/backends/fluid/gfluidcore.cpp | 322 +++--------------- .../fluid/gfluidcore_func.dispatch.cpp | 15 + .../src/backends/fluid/gfluidcore_func.hpp | 23 +- .../backends/fluid/gfluidcore_func.simd.hpp | 50 +++ 9 files changed, 136 insertions(+), 301 deletions(-) diff --git a/modules/gapi/perf/common/gapi_core_perf_tests.hpp b/modules/gapi/perf/common/gapi_core_perf_tests.hpp index 4084ed3e88..f3f251167b 100644 --- a/modules/gapi/perf/common/gapi_core_perf_tests.hpp +++ b/modules/gapi/perf/common/gapi_core_perf_tests.hpp @@ -50,7 +50,7 @@ namespace opencv_test class MinPerfTest : public TestPerfParams> {}; class MaxPerfTest : public TestPerfParams> {}; class AbsDiffPerfTest : public TestPerfParams> {}; - class AbsDiffCPerfTest : public TestPerfParams> {}; + class AbsDiffCPerfTest : public TestPerfParams> {}; class SumPerfTest : public TestPerfParams> {}; class CountNonZeroPerfTest : public TestPerfParams> {}; class AddWeightedPerfTest : public TestPerfParams> {}; diff --git a/modules/gapi/perf/common/gapi_core_perf_tests_inl.hpp b/modules/gapi/perf/common/gapi_core_perf_tests_inl.hpp index d4144cd71a..96ce369081 100644 --- a/modules/gapi/perf/common/gapi_core_perf_tests_inl.hpp +++ b/modules/gapi/perf/common/gapi_core_perf_tests_inl.hpp @@ -970,9 +970,10 @@ PERF_TEST_P_(AbsDiffPerfTest, TestPerformance) PERF_TEST_P_(AbsDiffCPerfTest, TestPerformance) { - cv::Size sz_in = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - cv::GCompileArgs compile_args = get<2>(GetParam()); + compare_f cmpF = get<0>(GetParam()); + cv::Size sz_in = get<1>(GetParam()); + MatType type = get<2>(GetParam()); + cv::GCompileArgs compile_args = get<3>(GetParam()); initMatsRandU(type, sz_in, type, false); @@ -997,8 +998,9 @@ PERF_TEST_P_(AbsDiffCPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - // FIXIT unrealiable check: EXPECT_EQ(0, cv::countNonZero(out_mat_gapi != out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz_in); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } diff --git a/modules/gapi/perf/cpu/gapi_core_perf_tests_cpu.cpp b/modules/gapi/perf/cpu/gapi_core_perf_tests_cpu.cpp index 1255f5ca52..c110de4fdd 100644 --- a/modules/gapi/perf/cpu/gapi_core_perf_tests_cpu.cpp +++ b/modules/gapi/perf/cpu/gapi_core_perf_tests_cpu.cpp @@ -156,7 +156,8 @@ INSTANTIATE_TEST_CASE_P(AbsDiffPerfTestCPU, AbsDiffPerfTest, Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(AbsDiffCPerfTestCPU, AbsDiffCPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), Values(cv::compile_args(CORE_CPU)))); diff --git a/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp b/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp index 058cff69ac..442d9efa7a 100644 --- a/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp +++ b/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp @@ -153,10 +153,9 @@ INSTANTIATE_TEST_CASE_P(AbsDiffPerfTestFluid, AbsDiffPerfTest, Values(cv::compile_args(CORE_FLUID)))); INSTANTIATE_TEST_CASE_P(AbsDiffCPerfTestFluid, AbsDiffCPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_16UC1, CV_16SC1, CV_8UC2, - CV_16UC2, CV_16SC2, CV_8UC3, CV_16UC3, - CV_16SC3, CV_8UC4, CV_16UC4, CV_16SC4), + Combine(Values(Tolerance_FloatRel_IntAbs(1e-6, 1).to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), Values(cv::compile_args(CORE_FLUID)))); // INSTANTIATE_TEST_CASE_P(SumPerfTestFluid, SumPerfTest, diff --git a/modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp b/modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp index 025ea5331d..b567f8cd8a 100644 --- a/modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp +++ b/modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp @@ -154,7 +154,8 @@ INSTANTIATE_TEST_CASE_P(AbsDiffPerfTestGPU, AbsDiffPerfTest, Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(AbsDiffCPerfTestGPU, AbsDiffCPerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), Values(cv::compile_args(CORE_GPU)))); diff --git a/modules/gapi/src/backends/fluid/gfluidcore.cpp b/modules/gapi/src/backends/fluid/gfluidcore.cpp index a0513a09cd..8342a26d0d 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore.cpp +++ b/modules/gapi/src/backends/fluid/gfluidcore.cpp @@ -994,244 +994,6 @@ static void run_arithm_s(DST out[], const SRC in[], int width, int chan, CV_Error(cv::Error::StsBadArg, "unsupported number of channels"); } -#if CV_SIMD -CV_ALWAYS_INLINE void absdiffc_short_store_c1c2c4(short* out_ptr, const v_int32& c1, const v_int32& c2) -{ - vx_store(out_ptr, v_pack(c1, c2)); -} - -CV_ALWAYS_INLINE void absdiffc_short_store_c1c2c4(ushort* out_ptr, const v_int32& c1, const v_int32& c2) -{ - vx_store(out_ptr, v_pack_u(c1, c2)); -} - -template -CV_ALWAYS_INLINE int absdiffc_simd_c1c2c4(const T in[], T out[], - const v_float32& s, const int length) -{ - static_assert((std::is_same::value) || (std::is_same::value), - "This templated overload is only for short or ushort type combinations."); - - constexpr int nlanes = (std::is_same::value) ? static_cast(v_uint16::nlanes) : - static_cast(v_int16::nlanes); - if (length < nlanes) - return 0; - - int x = 0; - for (;;) - { - for (; x <= length - nlanes; x += nlanes) - { - v_float32 a1 = v_load_f32(in + x); - v_float32 a2 = v_load_f32(in + x + nlanes / 2); - - absdiffc_short_store_c1c2c4(&out[x], v_round(v_absdiff(a1, s)), - v_round(v_absdiff(a2, s))); - } - - if (x < length && (in != out)) - { - x = length - nlanes; - continue; // process unaligned tail - } - break; - } - return x; -} - -template<> -CV_ALWAYS_INLINE int absdiffc_simd_c1c2c4(const uchar in[], uchar out[], - const v_float32& s, const int length) -{ - constexpr int nlanes = static_cast(v_uint8::nlanes); - - if (length < nlanes) - return 0; - - int x = 0; - for (;;) - { - for (; x <= length - nlanes; x += nlanes) - { - v_float32 a1 = v_load_f32(in + x); - v_float32 a2 = v_load_f32(in + x + nlanes / 4); - v_float32 a3 = v_load_f32(in + x + nlanes / 2); - v_float32 a4 = v_load_f32(in + x + 3 * nlanes / 4); - - vx_store(&out[x], v_pack_u(v_pack(v_round(v_absdiff(a1, s)), - v_round(v_absdiff(a2, s))), - v_pack(v_round(v_absdiff(a3, s)), - v_round(v_absdiff(a4, s))))); - } - - if (x < length && (in != out)) - { - x = length - nlanes; - continue; // process unaligned tail - } - break; - } - return x; -} - -CV_ALWAYS_INLINE void absdiffc_short_store_c3(short* out_ptr, const v_int32& c1, - const v_int32& c2, const v_int32& c3, - const v_int32& c4, const v_int32& c5, - const v_int32& c6) -{ - constexpr int nlanes = static_cast(v_int16::nlanes); - vx_store(out_ptr, v_pack(c1, c2)); - vx_store(out_ptr + nlanes, v_pack(c3, c4)); - vx_store(out_ptr + 2*nlanes, v_pack(c5, c6)); -} - -CV_ALWAYS_INLINE void absdiffc_short_store_c3(ushort* out_ptr, const v_int32& c1, - const v_int32& c2, const v_int32& c3, - const v_int32& c4, const v_int32& c5, - const v_int32& c6) -{ - constexpr int nlanes = static_cast(v_uint16::nlanes); - vx_store(out_ptr, v_pack_u(c1, c2)); - vx_store(out_ptr + nlanes, v_pack_u(c3, c4)); - vx_store(out_ptr + 2*nlanes, v_pack_u(c5, c6)); -} - -template -CV_ALWAYS_INLINE int absdiffc_simd_c3_impl(const T in[], T out[], - const v_float32& s1, const v_float32& s2, - const v_float32& s3, const int length) -{ - static_assert((std::is_same::value) || (std::is_same::value), - "This templated overload is only for short or ushort type combinations."); - - constexpr int nlanes = (std::is_same::value) ? static_cast(v_uint16::nlanes): - static_cast(v_int16::nlanes); - - if (length < 3 * nlanes) - return 0; - - int x = 0; - for (;;) - { - for (; x <= length - 3 * nlanes; x += 3 * nlanes) - { - v_float32 a1 = v_load_f32(in + x); - v_float32 a2 = v_load_f32(in + x + nlanes / 2); - v_float32 a3 = v_load_f32(in + x + nlanes); - v_float32 a4 = v_load_f32(in + x + 3 * nlanes / 2); - v_float32 a5 = v_load_f32(in + x + 2 * nlanes); - v_float32 a6 = v_load_f32(in + x + 5 * nlanes / 2); - - absdiffc_short_store_c3(&out[x], v_round(v_absdiff(a1, s1)), - v_round(v_absdiff(a2, s2)), - v_round(v_absdiff(a3, s3)), - v_round(v_absdiff(a4, s1)), - v_round(v_absdiff(a5, s2)), - v_round(v_absdiff(a6, s3))); - } - - if (x < length && (in != out)) - { - x = length - 3 * nlanes; - continue; // process unaligned tail - } - break; - } - return x; -} - -template<> -CV_ALWAYS_INLINE int absdiffc_simd_c3_impl(const uchar in[], uchar out[], - const v_float32& s1, const v_float32& s2, - const v_float32& s3, const int length) -{ - constexpr int nlanes = static_cast(v_uint8::nlanes); - - if (length < 3 * nlanes) - return 0; - - int x = 0; - - for (;;) - { - for (; x <= length - 3 * nlanes; x += 3 * nlanes) - { - vx_store(&out[x], - v_pack_u(v_pack(v_round(v_absdiff(v_load_f32(in + x), s1)), - v_round(v_absdiff(v_load_f32(in + x + nlanes/4), s2))), - v_pack(v_round(v_absdiff(v_load_f32(in + x + nlanes/2), s3)), - v_round(v_absdiff(v_load_f32(in + x + 3*nlanes/4), s1))))); - - vx_store(&out[x + nlanes], - v_pack_u(v_pack(v_round(v_absdiff(v_load_f32(in + x + nlanes), s2)), - v_round(v_absdiff(v_load_f32(in + x + 5*nlanes/4), s3))), - v_pack(v_round(v_absdiff(v_load_f32(in + x + 3*nlanes/2), s1)), - v_round(v_absdiff(v_load_f32(in + x + 7*nlanes/4), s2))))); - - vx_store(&out[x + 2 * nlanes], - v_pack_u(v_pack(v_round(v_absdiff(v_load_f32(in + x + 2*nlanes), s3)), - v_round(v_absdiff(v_load_f32(in + x + 9*nlanes/4), s1))), - v_pack(v_round(v_absdiff(v_load_f32(in + x + 5*nlanes/2), s2)), - v_round(v_absdiff(v_load_f32(in + x + 11*nlanes/4), s3))))); - } - - if (x < length && (in != out)) - { - x = length - 3 * nlanes; - continue; // process unaligned tail - } - break; - } - return x; -} - -template -CV_ALWAYS_INLINE int absdiffc_simd_channels(const T in[], const float scalar[], T out[], - const int width, int chan) -{ - int length = width * chan; - v_float32 s = vx_load(scalar); - - return absdiffc_simd_c1c2c4(in, out, s, length); -} - -template -CV_ALWAYS_INLINE int absdiffc_simd_c3(const T in[], const float scalar[], T out[], int width) -{ - constexpr int chan = 3; - int length = width * chan; - - v_float32 s1 = vx_load(scalar); -#if CV_SIMD_WIDTH == 32 - v_float32 s2 = vx_load(scalar + 2); - v_float32 s3 = vx_load(scalar + 1); -#else - v_float32 s2 = vx_load(scalar + 1); - v_float32 s3 = vx_load(scalar + 2); -#endif - - return absdiffc_simd_c3_impl(in, out, s1, s2, s3, length); -} - -template -CV_ALWAYS_INLINE int absdiffc_simd(const T in[], const float scalar[], T out[], int width, int chan) -{ - switch (chan) - { - case 1: - case 2: - case 4: - return absdiffc_simd_channels(in, scalar, out, width, chan); - case 3: - return absdiffc_simd_c3(in, scalar, out, width); - default: - break; - } - - return 0; -} -#endif // CV_SIMD - template static void run_absdiffc(Buffer &dst, const View &src, const float scalar[]) { @@ -1240,13 +1002,14 @@ static void run_absdiffc(Buffer &dst, const View &src, const float scalar[]) int width = dst.length(); int chan = dst.meta().chan; + const int length = width * chan; int w = 0; #if CV_SIMD - w = absdiffc_simd(in, scalar, out, width, chan); + w = absdiffc_simd(in, scalar, out, length, chan); #endif - for (; w < width*chan; ++w) + for (; w < length; ++w) out[w] = absdiff(in[w], scalar[w%chan]); } @@ -1349,49 +1112,6 @@ static void run_arithm_rs(Buffer &dst, const View &src, const float scalar[4], A } } -GAPI_FLUID_KERNEL(GFluidAbsDiffC, cv::gapi::core::GAbsDiffC, true) -{ - static const int Window = 1; - - static void run(const View &src, const cv::Scalar& _scalar, Buffer &dst, Buffer& scratch) - { - if (dst.y() == 0) - { - const int chan = src.meta().chan; - float* sc = scratch.OutLine(); - - for (int i = 0; i < scratch.length(); ++i) - sc[i] = static_cast(_scalar[i % chan]); - } - - const float* scalar = scratch.OutLine(); - - // DST SRC OP __VA_ARGS__ - UNARY_(uchar, uchar, run_absdiffc, dst, src, scalar); - UNARY_(ushort, ushort, run_absdiffc, dst, src, scalar); - UNARY_(short, short, run_absdiffc, dst, src, scalar); - - CV_Error(cv::Error::StsBadArg, "unsupported combination of types"); - } - - static void initScratch(const GMatDesc&, const GScalarDesc&, Buffer& scratch) - { -#if CV_SIMD - constexpr int buflen = static_cast(v_float32::nlanes) + 2; // buffer size -#else - constexpr int buflen = 4; -#endif - cv::Size bufsize(buflen, 1); - GMatDesc bufdesc = { CV_32F, 1, bufsize }; - Buffer buffer(bufdesc); - scratch = std::move(buffer); - } - - static void resetScratch(Buffer& /* scratch */) - { - } -}; - CV_ALWAYS_INLINE void initScratchBuffer(Buffer& scratch) { #if CV_SIMD @@ -1418,6 +1138,42 @@ CV_ALWAYS_INLINE void initScratchBuffer(Buffer& scratch) scratch = std::move(buffer); } +GAPI_FLUID_KERNEL(GFluidAbsDiffC, cv::gapi::core::GAbsDiffC, true) +{ + static const int Window = 1; + + static void run(const View &src, const cv::Scalar& _scalar, Buffer &dst, Buffer& scratch) + { + if (dst.y() == 0) + { + const int chan = src.meta().chan; + float* sc = scratch.OutLine(); + + for (int i = 0; i < scratch.length(); ++i) + sc[i] = static_cast(_scalar[i % chan]); + } + + const float* scalar = scratch.OutLine(); + + // DST SRC OP __VA_ARGS__ + UNARY_(uchar, uchar, run_absdiffc, dst, src, scalar); + UNARY_(ushort, ushort, run_absdiffc, dst, src, scalar); + UNARY_(short, short, run_absdiffc, dst, src, scalar); + UNARY_(float, float, run_absdiffc, dst, src, scalar); + + CV_Error(cv::Error::StsBadArg, "unsupported combination of types"); + } + + static void initScratch(const GMatDesc&, const GScalarDesc&, Buffer& scratch) + { + initScratchBuffer(scratch); + } + + static void resetScratch(Buffer& /* scratch */) + { + } +}; + GAPI_FLUID_KERNEL(GFluidAddC, cv::gapi::core::GAddC, true) { static const int Window = 1; diff --git a/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp b/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp index f596779286..ab6b013694 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp +++ b/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp @@ -165,6 +165,21 @@ MULC_SIMD(float, float) #undef MULC_SIMD +#define ABSDIFFC_SIMD(SRC) \ +int absdiffc_simd(const SRC in[], const float scalar[], SRC out[], \ + const int length, const int chan) \ +{ \ + CV_CPU_DISPATCH(absdiffc_simd, (in, scalar, out, length, chan), \ + CV_CPU_DISPATCH_MODES_ALL); \ +} + +ABSDIFFC_SIMD(uchar) +ABSDIFFC_SIMD(short) +ABSDIFFC_SIMD(ushort) +ABSDIFFC_SIMD(float) + +#undef ABSDIFFC_SIMD + } // namespace fluid } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/backends/fluid/gfluidcore_func.hpp b/modules/gapi/src/backends/fluid/gfluidcore_func.hpp index 541870e548..522d7b8b44 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore_func.hpp +++ b/modules/gapi/src/backends/fluid/gfluidcore_func.hpp @@ -60,8 +60,8 @@ MUL_SIMD(float, float) #undef MUL_SIMD -#define ADDC_SIMD(SRC, DST) \ -int addc_simd(const SRC in[], const float scalar[], DST out[], \ +#define ADDC_SIMD(SRC, DST) \ +int addc_simd(const SRC in[], const float scalar[], DST out[], \ const int length, const int chan); ADDC_SIMD(uchar, uchar) @@ -83,8 +83,8 @@ ADDC_SIMD(float, float) #undef ADDC_SIMD -#define SUBC_SIMD(SRC, DST) \ -int subc_simd(const SRC in[], const float scalar[], DST out[], \ +#define SUBC_SIMD(SRC, DST) \ +int subc_simd(const SRC in[], const float scalar[], DST out[], \ const int length, const int chan); SUBC_SIMD(uchar, uchar) @@ -106,8 +106,8 @@ SUBC_SIMD(float, float) #undef SUBC_SIMD -#define MULC_SIMD(SRC, DST) \ -int mulc_simd(const SRC in[], const float scalar[], DST out[], \ +#define MULC_SIMD(SRC, DST) \ +int mulc_simd(const SRC in[], const float scalar[], DST out[], \ const int length, const int chan, const float scale); MULC_SIMD(uchar, uchar) @@ -129,6 +129,17 @@ MULC_SIMD(float, float) #undef MULC_SIMD +#define ABSDIFFC_SIMD(T) \ +int absdiffc_simd(const T in[], const float scalar[], T out[], \ + const int length, const int chan); + +ABSDIFFC_SIMD(uchar) +ABSDIFFC_SIMD(short) +ABSDIFFC_SIMD(ushort) +ABSDIFFC_SIMD(float) + +#undef ABSDIFFC_SIMD + } // namespace fluid } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp b/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp index 45974131c3..12b74f8f67 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp +++ b/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp @@ -151,6 +151,17 @@ MULC_SIMD(float, float) #undef MULC_SIMD +#define ABSDIFFC_SIMD(T) \ +int absdiffc_simd(const T in[], const float scalar[], T out[], \ + const int length, const int chan); + +ABSDIFFC_SIMD(uchar) +ABSDIFFC_SIMD(short) +ABSDIFFC_SIMD(ushort) +ABSDIFFC_SIMD(float) + +#undef ABSDIFFC_SIMD + #ifndef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY struct scale_tag {}; @@ -901,6 +912,7 @@ MUL_SIMD(float, float) struct add_tag {}; struct sub_tag {}; struct mul_tag {}; +struct absdiff_tag {}; CV_ALWAYS_INLINE void arithmOpScalar_pack_store_c3(short* outx, const v_int32& c1, const v_int32& c2, const v_int32& c3, @@ -938,6 +950,12 @@ CV_ALWAYS_INLINE v_float32 oper(mul_tag, const v_float32& a, const v_float32& sc { return a * sc; } + +CV_ALWAYS_INLINE v_float32 oper(absdiff_tag, const v_float32& a, const v_float32& sc) +{ + return v_absdiff(a, sc); +} + //------------------------------------------------------------------------------------------------- template @@ -1450,6 +1468,38 @@ MULC_SIMD(float, float) #undef MULC_SIMD +//------------------------- +// +// Fluid kernels: AbsDiffC +// +//------------------------- + +#define ABSDIFFC_SIMD(SRC) \ +int absdiffc_simd(const SRC in[], const float scalar[], SRC out[], \ + const int length, const int chan) \ +{ \ + switch (chan) \ + { \ + case 1: \ + case 2: \ + case 4: \ + return arithmOpScalar_simd_common(absdiff_tag{}, in, scalar, out, length); \ + case 3: \ + return arithmOpScalar_simd_c3(absdiff_tag{}, in, scalar, out, length); \ + default: \ + GAPI_Assert(chan <= 4); \ + break; \ + } \ + return 0; \ +} + +ABSDIFFC_SIMD(uchar) +ABSDIFFC_SIMD(short) +ABSDIFFC_SIMD(ushort) +ABSDIFFC_SIMD(float) + +#undef ABSDIFFC_SIMD + #endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY CV_CPU_OPTIMIZATION_NAMESPACE_END From 5aef565fb6afdee919c297acb2dd479ba5cf9616 Mon Sep 17 00:00:00 2001 From: UncleLLD Date: Tue, 7 Dec 2021 00:14:17 +0800 Subject: [PATCH 137/226] Merge pull request #21188 from UncleLLD:fix-markdown-error fix issue 21187: markdown file: gray image does not have three dimensions --- .../py_feature_homography/py_feature_homography.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/py_tutorials/py_feature2d/py_feature_homography/py_feature_homography.markdown b/doc/py_tutorials/py_feature2d/py_feature_homography/py_feature_homography.markdown index 8602cc9398..f8836b095b 100644 --- a/doc/py_tutorials/py_feature2d/py_feature_homography/py_feature_homography.markdown +++ b/doc/py_tutorials/py_feature2d/py_feature_homography/py_feature_homography.markdown @@ -78,7 +78,7 @@ if len(good)>MIN_MATCH_COUNT: M, mask = cv.findHomography(src_pts, dst_pts, cv.RANSAC,5.0) matchesMask = mask.ravel().tolist() - h,w,d = img1.shape + h,w = img1.shape pts = np.float32([ [0,0],[0,h-1],[w-1,h-1],[w-1,0] ]).reshape(-1,1,2) dst = cv.perspectiveTransform(pts,M) From 8dd68822221cf5aaf556be5969f16d02afe0039a Mon Sep 17 00:00:00 2001 From: "Anastasiya(Asya) Pronina" Date: Mon, 6 Dec 2021 19:54:21 +0300 Subject: [PATCH 138/226] Merge pull request #20709 from AsyaPronina:asyadev/integrate_gstreamer_source Ported GStreamerSource to OpenCV * Ported GStreamerSource to OpenCV * Fixed CI failures * Whitespaces * Whitespaces + removed exception from destructors C4722 * Removed assert for Priv's getSS and descr_of * Removed assert for pull * Fixed last review comment Co-authored-by: Pashchenkov Maxim --- modules/gapi/CMakeLists.txt | 25 +- .../streaming/gstreamer/gstreamerpipeline.hpp | 47 ++ .../streaming/gstreamer/gstreamersource.hpp | 89 ++++ .../onevpl/device_selector_interface.hpp | 2 +- .../gstreamer/gstreamer_buffer_utils.cpp | 27 ++ .../gstreamer/gstreamer_buffer_utils.hpp | 27 ++ .../gstreamer/gstreamer_media_adapter.cpp | 122 ++++++ .../gstreamer/gstreamer_media_adapter.hpp | 63 +++ .../gstreamer/gstreamer_pipeline_facade.cpp | 314 ++++++++++++++ .../gstreamer/gstreamer_pipeline_facade.hpp | 89 ++++ .../src/streaming/gstreamer/gstreamerenv.cpp | 90 ++++ .../src/streaming/gstreamer/gstreamerenv.hpp | 37 ++ .../streaming/gstreamer/gstreamerpipeline.cpp | 112 +++++ .../gstreamer/gstreamerpipeline_priv.hpp | 58 +++ .../src/streaming/gstreamer/gstreamerptr.hpp | 177 ++++++++ .../streaming/gstreamer/gstreamersource.cpp | 383 +++++++++++++++++ .../gstreamer/gstreamersource_priv.hpp | 94 ++++ ...pi_gstreamer_pipeline_facade_int_tests.cpp | 188 ++++++++ .../streaming/gapi_gstreamersource_tests.cpp | 401 ++++++++++++++++++ 19 files changed, 2341 insertions(+), 4 deletions(-) create mode 100644 modules/gapi/include/opencv2/gapi/streaming/gstreamer/gstreamerpipeline.hpp create mode 100644 modules/gapi/include/opencv2/gapi/streaming/gstreamer/gstreamersource.hpp create mode 100644 modules/gapi/src/streaming/gstreamer/gstreamer_buffer_utils.cpp create mode 100644 modules/gapi/src/streaming/gstreamer/gstreamer_buffer_utils.hpp create mode 100644 modules/gapi/src/streaming/gstreamer/gstreamer_media_adapter.cpp create mode 100644 modules/gapi/src/streaming/gstreamer/gstreamer_media_adapter.hpp create mode 100644 modules/gapi/src/streaming/gstreamer/gstreamer_pipeline_facade.cpp create mode 100644 modules/gapi/src/streaming/gstreamer/gstreamer_pipeline_facade.hpp create mode 100644 modules/gapi/src/streaming/gstreamer/gstreamerenv.cpp create mode 100644 modules/gapi/src/streaming/gstreamer/gstreamerenv.hpp create mode 100644 modules/gapi/src/streaming/gstreamer/gstreamerpipeline.cpp create mode 100644 modules/gapi/src/streaming/gstreamer/gstreamerpipeline_priv.hpp create mode 100644 modules/gapi/src/streaming/gstreamer/gstreamerptr.hpp create mode 100644 modules/gapi/src/streaming/gstreamer/gstreamersource.cpp create mode 100644 modules/gapi/src/streaming/gstreamer/gstreamersource_priv.hpp create mode 100644 modules/gapi/test/streaming/gapi_gstreamer_pipeline_facade_int_tests.cpp create mode 100644 modules/gapi/test/streaming/gapi_gstreamersource_tests.cpp diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 79ae30ae57..855ce27088 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -47,12 +47,14 @@ file(GLOB gapi_ext_hdrs "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/infer/*.hpp" "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/ocl/*.hpp" "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/own/*.hpp" + "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/plaidml/*.hpp" + "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/python/*.hpp" "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/render/*.hpp" "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/s11n/*.hpp" "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/streaming/*.hpp" - "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/plaidml/*.hpp" + "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/streaming/gstreamer/*.hpp" + "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/streaming/onevpl/*.hpp" "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/util/*.hpp" - "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/python/*.hpp" ) set(gapi_srcs @@ -163,7 +165,7 @@ set(gapi_srcs src/backends/ie/bindings_ie.cpp src/backends/python/gpythonbackend.cpp - # Streaming source + # OpenVPL Streaming source src/streaming/onevpl/source.cpp src/streaming/onevpl/source_priv.cpp src/streaming/onevpl/file_data_provider.cpp @@ -186,6 +188,14 @@ set(gapi_srcs src/streaming/onevpl/cfg_param_device_selector.cpp src/streaming/onevpl/device_selector_interface.cpp + # GStreamer Streaming source + src/streaming/gstreamer/gstreamer_pipeline_facade.cpp + src/streaming/gstreamer/gstreamerpipeline.cpp + src/streaming/gstreamer/gstreamersource.cpp + src/streaming/gstreamer/gstreamer_buffer_utils.cpp + src/streaming/gstreamer/gstreamer_media_adapter.cpp + src/streaming/gstreamer/gstreamerenv.cpp + # Utils (ITT tracing) src/utils/itt.cpp ) @@ -283,6 +293,15 @@ if(HAVE_GAPI_ONEVPL) endif() endif() +if(HAVE_GSTREAMER) + if(TARGET opencv_test_gapi) + ocv_target_compile_definitions(opencv_test_gapi PRIVATE -DHAVE_GSTREAMER) + ocv_target_link_libraries(opencv_test_gapi PRIVATE ocv.3rdparty.gstreamer) + endif() + ocv_target_compile_definitions(${the_module} PRIVATE -DHAVE_GSTREAMER) + ocv_target_link_libraries(${the_module} PRIVATE ocv.3rdparty.gstreamer) +endif() + if(WIN32) # Required for htonl/ntohl on Windows ocv_target_link_libraries(${the_module} PRIVATE wsock32 ws2_32) diff --git a/modules/gapi/include/opencv2/gapi/streaming/gstreamer/gstreamerpipeline.hpp b/modules/gapi/include/opencv2/gapi/streaming/gstreamer/gstreamerpipeline.hpp new file mode 100644 index 0000000000..83afc99393 --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/streaming/gstreamer/gstreamerpipeline.hpp @@ -0,0 +1,47 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMERPIPELINE_HPP +#define OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMERPIPELINE_HPP + +#include +#include + +#include +#include +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace gst { + +class GAPI_EXPORTS GStreamerPipeline +{ +public: + class Priv; + + explicit GStreamerPipeline(const std::string& pipeline); + IStreamSource::Ptr getStreamingSource(const std::string& appsinkName, + const GStreamerSource::OutputType outputType = + GStreamerSource::OutputType::MAT); + virtual ~GStreamerPipeline(); + +protected: + explicit GStreamerPipeline(std::unique_ptr priv); + + std::unique_ptr m_priv; +}; + +} // namespace gst + +using GStreamerPipeline = gst::GStreamerPipeline; + +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMERPIPELINE_HPP diff --git a/modules/gapi/include/opencv2/gapi/streaming/gstreamer/gstreamersource.hpp b/modules/gapi/include/opencv2/gapi/streaming/gstreamer/gstreamersource.hpp new file mode 100644 index 0000000000..b81bad31b8 --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/streaming/gstreamer/gstreamersource.hpp @@ -0,0 +1,89 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMERSOURCE_HPP +#define OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMERSOURCE_HPP + +#include +#include + +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace gst { + +/** + * @brief OpenCV's GStreamer streaming source. + * Streams cv::Mat-s/cv::MediaFrame from passed GStreamer pipeline. + * + * This class implements IStreamSource interface. + * + * To create GStreamerSource instance you need to pass 'pipeline' and, optionally, 'outputType' + * arguments into constructor. + * 'pipeline' should represent GStreamer pipeline in form of textual description. + * Almost any custom pipeline is supported which can be successfully ran via gst-launch. + * The only two limitations are: + * - there should be __one__ appsink element in the pipeline to pass data to OpenCV app. + * Pipeline can actually contain many sink elements, but it must have one and only one + * appsink among them. + * + * - data passed to appsink should be video-frame in NV12 format. + * + * 'outputType' is used to select type of output data to produce: 'cv::MediaFrame' or 'cv::Mat'. + * To produce 'cv::MediaFrame'-s you need to pass 'GStreamerSource::OutputType::FRAME' and, + * correspondingly, 'GStreamerSource::OutputType::MAT' to produce 'cv::Mat'-s. + * Please note, that in the last case, output 'cv::Mat' will be of BGR format, internal conversion + * from NV12 GStreamer data will happen. + * Default value for 'outputType' is 'GStreamerSource::OutputType::MAT'. + * + * @note Stream sources are passed to G-API via shared pointers, so please use gapi::make_src<> + * to create objects and ptr() to pass a GStreamerSource to cv::gin(). + * + * @note You need to build OpenCV with GStreamer support to use this class. + */ + +class GStreamerPipelineFacade; + +class GAPI_EXPORTS GStreamerSource : public IStreamSource +{ +public: + class Priv; + + // Indicates what type of data should be produced by GStreamerSource: cv::MediaFrame or cv::Mat + enum class OutputType { + FRAME, + MAT + }; + + GStreamerSource(const std::string& pipeline, + const GStreamerSource::OutputType outputType = + GStreamerSource::OutputType::MAT); + GStreamerSource(std::shared_ptr pipeline, + const std::string& appsinkName, + const GStreamerSource::OutputType outputType = + GStreamerSource::OutputType::MAT); + + bool pull(cv::gapi::wip::Data& data) override; + GMetaArg descr_of() const override; + ~GStreamerSource() override; + +protected: + explicit GStreamerSource(std::unique_ptr priv); + + std::unique_ptr m_priv; +}; + +} // namespace gst + +using GStreamerSource = gst::GStreamerSource; + +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMERSOURCE_HPP diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/device_selector_interface.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/device_selector_interface.hpp index ca19849d72..04f8cae02a 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/onevpl/device_selector_interface.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/device_selector_interface.hpp @@ -19,7 +19,7 @@ namespace gapi { namespace wip { namespace onevpl { -enum class AccelType : uint8_t { +enum class AccelType: uint8_t { HOST, DX11, diff --git a/modules/gapi/src/streaming/gstreamer/gstreamer_buffer_utils.cpp b/modules/gapi/src/streaming/gstreamer/gstreamer_buffer_utils.cpp new file mode 100644 index 0000000000..227013105a --- /dev/null +++ b/modules/gapi/src/streaming/gstreamer/gstreamer_buffer_utils.cpp @@ -0,0 +1,27 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include "gstreamer_buffer_utils.hpp" +#include "gstreamerptr.hpp" +#include + +#ifdef HAVE_GSTREAMER +namespace cv { +namespace gapi { +namespace wip { +namespace gstreamer_utils { + +void mapBufferToFrame(GstBuffer& buffer, GstVideoInfo& info, GstVideoFrame& frame, + GstMapFlags mapFlags) { + bool mapped = gst_video_frame_map(&frame, &info, &buffer, mapFlags); + GAPI_Assert(mapped && "Failed to map GStreamer buffer to system memory as video-frame!"); +} + +} // namespace gstreamer_utils +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_GSTREAMER diff --git a/modules/gapi/src/streaming/gstreamer/gstreamer_buffer_utils.hpp b/modules/gapi/src/streaming/gstreamer/gstreamer_buffer_utils.hpp new file mode 100644 index 0000000000..3e6f908e0f --- /dev/null +++ b/modules/gapi/src/streaming/gstreamer/gstreamer_buffer_utils.hpp @@ -0,0 +1,27 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMER_BUFFER_UTILS_HPP +#define OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMER_BUFFER_UTILS_HPP + +#ifdef HAVE_GSTREAMER +#include +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace gstreamer_utils { + +void mapBufferToFrame(GstBuffer& buffer, GstVideoInfo& info, GstVideoFrame& frame, + GstMapFlags map_flags); + +} // namespace gstreamer_utils +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_GSTREAMER +#endif // OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMER_BUFFER_UTILS_HPP diff --git a/modules/gapi/src/streaming/gstreamer/gstreamer_media_adapter.cpp b/modules/gapi/src/streaming/gstreamer/gstreamer_media_adapter.cpp new file mode 100644 index 0000000000..9019289ae4 --- /dev/null +++ b/modules/gapi/src/streaming/gstreamer/gstreamer_media_adapter.cpp @@ -0,0 +1,122 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include "gstreamer_media_adapter.hpp" +#include "gstreamer_buffer_utils.hpp" + +#ifdef HAVE_GSTREAMER +namespace cv { +namespace gapi { +namespace wip { +namespace gst { + +GStreamerMediaAdapter::GStreamerMediaAdapter(const cv::GFrameDesc& frameDesc, + GstVideoInfo* videoInfo, + GstBuffer* buffer) : + m_frameDesc(frameDesc), + m_videoInfo(gst_video_info_copy(videoInfo)), + m_buffer(gst_buffer_ref(buffer)), + m_isMapped(false) +{ +#if GST_VERSION_MINOR >= 10 + // Check that GstBuffer has mono-view, so we can retrieve only one video-meta + GAPI_Assert((gst_buffer_get_flags(m_buffer) & GST_VIDEO_BUFFER_FLAG_MULTIPLE_VIEW) == 0); +#endif // GST_VERSION_MINOR >= 10 + + GstVideoMeta* videoMeta = gst_buffer_get_video_meta(m_buffer); + if (videoMeta != nullptr) { + m_strides = { videoMeta->stride[0], videoMeta->stride[1] }; + m_offsets = { videoMeta->offset[0], videoMeta->offset[1] }; + } else { + m_strides = { GST_VIDEO_INFO_PLANE_STRIDE(m_videoInfo.get(), 0), + GST_VIDEO_INFO_PLANE_STRIDE(m_videoInfo.get(), 1) }; + m_offsets = { GST_VIDEO_INFO_PLANE_OFFSET(m_videoInfo.get(), 0), + GST_VIDEO_INFO_PLANE_OFFSET(m_videoInfo.get(), 1) }; + } +} + +GStreamerMediaAdapter::~GStreamerMediaAdapter() { + if (m_isMapped.load(std::memory_order_acquire)) { + gst_video_frame_unmap(&m_videoFrame); + m_isMapped.store(false, std::memory_order_release); + m_mappedForWrite.store(false); + } +} + +cv::GFrameDesc GStreamerMediaAdapter::meta() const { + return m_frameDesc; +} + +cv::MediaFrame::View GStreamerMediaAdapter::access(cv::MediaFrame::Access access) { + GAPI_Assert(access == cv::MediaFrame::Access::R || + access == cv::MediaFrame::Access::W); + static std::atomic thread_counters { }; + ++thread_counters; + + // NOTE: Framework guarantees that there should be no parallel accesses to the frame + // memory if is accessing for write. + if (access == cv::MediaFrame::Access::W && !m_mappedForWrite.load(std::memory_order_acquire)) { + GAPI_Assert(thread_counters > 1 && + "Multiple access to view during mapping for write detected!"); + gst_video_frame_unmap(&m_videoFrame); + m_isMapped.store(false); + } + + if (!m_isMapped.load(std::memory_order_acquire)) { + + std::lock_guard lock(m_mutex); + + if(!m_isMapped.load(std::memory_order_relaxed)) { + + GAPI_Assert(GST_VIDEO_INFO_N_PLANES(m_videoInfo.get()) == 2); + GAPI_Assert(GST_VIDEO_INFO_FORMAT(m_videoInfo.get()) == GST_VIDEO_FORMAT_NV12); + + // TODO: Use RAII for map/unmap + if (access == cv::MediaFrame::Access::W) { + gstreamer_utils::mapBufferToFrame(*m_buffer, *m_videoInfo, m_videoFrame, + GST_MAP_WRITE); + m_mappedForWrite.store(true, std::memory_order_release); + } else { + gstreamer_utils::mapBufferToFrame(*m_buffer, *m_videoInfo, m_videoFrame, + GST_MAP_READ); + } + + GAPI_Assert(GST_VIDEO_FRAME_PLANE_STRIDE(&m_videoFrame, 0) == m_strides[0]); + GAPI_Assert(GST_VIDEO_FRAME_PLANE_STRIDE(&m_videoFrame, 1) == m_strides[1]); + GAPI_Assert(GST_VIDEO_FRAME_PLANE_OFFSET(&m_videoFrame, 0) == m_offsets[0]); + GAPI_Assert(GST_VIDEO_FRAME_PLANE_OFFSET(&m_videoFrame, 1) == m_offsets[1]); + + m_isMapped.store(true, std::memory_order_release); + } + } + + cv::MediaFrame::View::Ptrs ps { + static_cast(GST_VIDEO_FRAME_PLANE_DATA(&m_videoFrame, 0)) + m_offsets[0], // Y-plane + static_cast(GST_VIDEO_FRAME_PLANE_DATA(&m_videoFrame, 0)) + m_offsets[1], // UV-plane + nullptr, + nullptr + }; + + cv::MediaFrame::View::Strides ss = { + static_cast(m_strides[0]), // Y-plane stride + static_cast(m_strides[1]), // UV-plane stride + 0u, + 0u + }; + + --thread_counters; + return cv::MediaFrame::View(std::move(ps), std::move(ss)); +} + +cv::util::any GStreamerMediaAdapter::blobParams() const { + GAPI_Assert(false && "No implementation for GStreamerMediaAdapter::blobParams()"); +} + +} // namespace gst +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_GSTREAMER diff --git a/modules/gapi/src/streaming/gstreamer/gstreamer_media_adapter.hpp b/modules/gapi/src/streaming/gstreamer/gstreamer_media_adapter.hpp new file mode 100644 index 0000000000..4c5c137b0d --- /dev/null +++ b/modules/gapi/src/streaming/gstreamer/gstreamer_media_adapter.hpp @@ -0,0 +1,63 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMER_MEDIA_ADAPTER_HPP +#define OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMER_MEDIA_ADAPTER_HPP + +// #include +// #include + +#include "gstreamerptr.hpp" +#include + +#include +#include + +#ifdef HAVE_GSTREAMER +#include +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace gst { + +class GStreamerMediaAdapter : public cv::MediaFrame::IAdapter { +public: + explicit GStreamerMediaAdapter(const cv::GFrameDesc& frameDesc, + GstVideoInfo* videoInfo, + GstBuffer* buffer); + + ~GStreamerMediaAdapter() override; + + virtual cv::GFrameDesc meta() const override; + + cv::MediaFrame::View access(cv::MediaFrame::Access access) override; + + cv::util::any blobParams() const override; + +protected: + cv::GFrameDesc m_frameDesc; + + GStreamerPtr m_videoInfo; + GStreamerPtr m_buffer; + + std::vector m_strides; + std::vector m_offsets; + + GstVideoFrame m_videoFrame; + + std::atomic m_isMapped; + std::atomic m_mappedForWrite; + std::mutex m_mutex; +}; + +} // namespace gst +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_GSTREAMER +#endif // OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMER_MEDIA_ADAPTER_HPP diff --git a/modules/gapi/src/streaming/gstreamer/gstreamer_pipeline_facade.cpp b/modules/gapi/src/streaming/gstreamer/gstreamer_pipeline_facade.cpp new file mode 100644 index 0000000000..cd782537ca --- /dev/null +++ b/modules/gapi/src/streaming/gstreamer/gstreamer_pipeline_facade.cpp @@ -0,0 +1,314 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include "gstreamerenv.hpp" + +#include "gstreamer_pipeline_facade.hpp" + +#include + +#include + +#include + +#ifdef HAVE_GSTREAMER +#include +#include +#include +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace gst { + +GStreamerPipelineFacade::GStreamerPipelineFacade(): + m_isPrerolled(false), + m_isPlaying(false) + { } + +GStreamerPipelineFacade::GStreamerPipelineFacade(const std::string& pipelineDesc): + GStreamerPipelineFacade() +{ + m_pipelineDesc = pipelineDesc; + + // Initialize GStreamer library: + GStreamerEnv::init(); + + // Create GStreamer pipeline: + GError* error = NULL; + // gst_parse_launch [transfer floating] + m_pipeline = GST_ELEMENT(g_object_ref_sink( + gst_parse_launch(m_pipelineDesc.c_str(), &error))); + + GStreamerPtr err(error); + + if (err) + { + cv::util::throw_error( + std::runtime_error("Error in parsing pipeline: " + std::string(err->message))); + } +} + +// The destructors are noexcept by default (since C++11). +GStreamerPipelineFacade::~GStreamerPipelineFacade() +{ + // There is no mutex acquisition here, because we assume that no one will call this method + // directly. + + // Destructor may be called on empty GStreamerSource object in case if + // exception is thrown during construction. + if (m_pipeline && GST_IS_ELEMENT(m_pipeline.get())) + { + try + { + setState(GST_STATE_NULL); + } + catch(...) + { + GAPI_LOG_WARNING(NULL, "Unable to stop pipeline in destructor.\n"); + } + + m_pipeline.release(); + } +} + +std::vector GStreamerPipelineFacade::getElementsByFactoryName( + const std::string& factoryName) +{ + std::vector outElements = getElements( + [&factoryName](GstElement* element) { + GStreamerPtr name( + gst_object_get_name(GST_OBJECT(gst_element_get_factory(element)))); + return name && (0 == strcmp(name, factoryName.c_str())); + }); + + return outElements; +} + +GstElement* GStreamerPipelineFacade::getElementByName(const std::string& elementName) +{ + std::vector outElements = getElements( + [&elementName](GstElement* element) { + GStreamerPtr name(gst_element_get_name(element)); + return name && (0 == strcmp(name, elementName.c_str())); + }); + + if (outElements.empty()) + { + return nullptr; + } + else + { + GAPI_Assert(1ul == outElements.size()); + return outElements[0]; + } +} + +void GStreamerPipelineFacade::completePreroll() { + // FIXME: If there are multiple sources in pipeline and one of them is live, then pipeline + // will return GST_STATE_CHANGE_NO_PREROLL while pipeline pausing. + // But appsink may not be connected to this live source and only to anothers, + // not-live ones. So, it is not required to start the playback for appsink to complete + // the preroll. + // Starting of playback for the not-live sources before the first frame pull will lead + // to loosing of some amount of frames and pulling of the first frame can return frame + // which is far from the first. + // + // Need to handle this case or forbid to mix multiples sources of different + // categories(live, not-live) in the pipeline explicitly(assert). + + if (!m_isPrerolled.load(std::memory_order_acquire)) + { + std::lock_guard lock(m_stateChangeMutex); + + if(!m_isPrerolled.load(std::memory_order_relaxed)) + { + PipelineState state = queryState(); + + // Only move forward in the pipeline's state machine + GAPI_Assert(state.current != GST_STATE_PLAYING); + + GAPI_Assert(state.pending == GST_STATE_VOID_PENDING); + GstStateChangeReturn status = gst_element_set_state(m_pipeline, GST_STATE_PAUSED); + checkBusMessages(); + if (status == GST_STATE_CHANGE_NO_PREROLL) + { + status = gst_element_set_state(m_pipeline, GST_STATE_PLAYING); + m_isPlaying.store(true); + } + verifyStateChange(status); + + m_isPrerolled.store(true, std::memory_order_release); + } + } +} + +void GStreamerPipelineFacade::play() +{ + if (!m_isPlaying.load(std::memory_order_acquire)) + { + std::lock_guard lock(m_stateChangeMutex); + + if (!m_isPlaying.load(std::memory_order_relaxed)) + { + setState(GST_STATE_PLAYING); + m_isPlaying.store(true, std::memory_order_release); + m_isPrerolled.store(true); + } + } +} + +bool GStreamerPipelineFacade::isPlaying() { + return m_isPlaying.load(); +} + +std::vector GStreamerPipelineFacade::getElements( + std::function comparator) +{ + std::vector outElements; + GStreamerPtr it(gst_bin_iterate_elements(GST_BIN(m_pipeline.get()))); + GValue value = G_VALUE_INIT; + + GstIteratorResult status = gst_iterator_next(it, &value); + while (status != GST_ITERATOR_DONE && status != GST_ITERATOR_ERROR) + { + if (status == GST_ITERATOR_OK) + { + GstElement* element = GST_ELEMENT(g_value_get_object(&value)); + if (comparator(element)) + { + outElements.push_back(GST_ELEMENT(element)); + } + + g_value_unset(&value); + } + else if (status == GST_ITERATOR_RESYNC) + { + gst_iterator_resync(it); + } + + status = gst_iterator_next(it, &value); + } + + return outElements; +} + +PipelineState GStreamerPipelineFacade::queryState() +{ + GAPI_Assert(m_pipeline && GST_IS_ELEMENT(m_pipeline.get()) && + "GStreamer pipeline has not been created!"); + + PipelineState state; + GstClockTime timeout = 5 * GST_SECOND; + gst_element_get_state(m_pipeline, &state.current, &state.pending, timeout); + + return state; +} + +void GStreamerPipelineFacade::setState(GstState newState) +{ + PipelineState state = queryState(); + GAPI_Assert(state.pending == GST_STATE_VOID_PENDING); + + if (state.current != newState) + { + GstStateChangeReturn status = gst_element_set_state(m_pipeline, newState); + verifyStateChange(status); + } +} + +void GStreamerPipelineFacade::verifyStateChange(GstStateChangeReturn status) +{ + if (status == GST_STATE_CHANGE_ASYNC) + { + // Wait for status update. + status = gst_element_get_state(m_pipeline, NULL, NULL, GST_CLOCK_TIME_NONE); + } + + if (status == GST_STATE_CHANGE_FAILURE) + { + checkBusMessages(); + PipelineState state = queryState(); + const gchar* currentState = gst_element_state_get_name(state.current); + const gchar* pendingState = gst_element_state_get_name(state.pending); + cv::util::throw_error( + std::runtime_error(std::string("Unable to change pipeline state from ") + + std::string(currentState) + std::string(" to ") + + std::string(pendingState))); + } + + checkBusMessages(); +} + +// Handles GStreamer bus messages. +// For debugging purposes. +void GStreamerPipelineFacade::checkBusMessages() const +{ + GStreamerPtr bus(gst_element_get_bus(m_pipeline)); + + while (gst_bus_have_pending(bus)) + { + GStreamerPtr msg(gst_bus_pop(bus)); + if (!msg || !GST_IS_MESSAGE(msg.get())) + { + continue; + } + + if (gst_is_missing_plugin_message(msg)) + { + GStreamerPtr descr(gst_missing_plugin_message_get_description(msg)); + cv::util::throw_error( + std::runtime_error("Your GStreamer installation is missing a required plugin!" + "Details: " + std::string(descr))); + } + else + { + switch (GST_MESSAGE_TYPE(msg)) + { + case GST_MESSAGE_STATE_CHANGED: + { + if (GST_MESSAGE_SRC(msg.get()) == GST_OBJECT(m_pipeline.get())) + { + GstState oldState = GST_STATE_NULL, + newState = GST_STATE_NULL; + gst_message_parse_state_changed(msg, &oldState, &newState, NULL); + const gchar* oldStateName = gst_element_state_get_name(oldState); + const gchar* newStateName = gst_element_state_get_name(newState); + GAPI_LOG_INFO(NULL, "Pipeline state changed from " << oldStateName << " to " + << newStateName); + } + break; + } + case GST_MESSAGE_ERROR: + { + GError* error = NULL; + gchar* debug = NULL; + + gst_message_parse_error(msg, &error, &debug); // transfer full for out args + + GStreamerPtr err(error); + GStreamerPtr deb(debug); + + GStreamerPtr name(gst_element_get_name(GST_MESSAGE_SRC(msg.get()))); + GAPI_LOG_WARNING(NULL, "Embedded video playback halted; module " << name.get() + << " reported: " << err->message); + GAPI_LOG_WARNING(NULL, "GStreamer debug: " << deb); + + break; + } + default: + break; + } + } + } +} + +} // namespace gst +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_GSTREAMER diff --git a/modules/gapi/src/streaming/gstreamer/gstreamer_pipeline_facade.hpp b/modules/gapi/src/streaming/gstreamer/gstreamer_pipeline_facade.hpp new file mode 100644 index 0000000000..4ebd8f0197 --- /dev/null +++ b/modules/gapi/src/streaming/gstreamer/gstreamer_pipeline_facade.hpp @@ -0,0 +1,89 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMER_PIPELINE_FACADE_HPP +#define OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMER_PIPELINE_FACADE_HPP + +#include "gstreamerptr.hpp" + +#include +#include +#include + +#ifdef HAVE_GSTREAMER +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace gst { + +// GAPI_EXPORTS here is only for testing purposes. +struct GAPI_EXPORTS PipelineState +{ + GstState current = GST_STATE_NULL; + GstState pending = GST_STATE_NULL; +}; + +// This class represents facade for pipeline GstElement and related functions. +// Class restricts pipeline to only move forward in its state machine: +// NULL -> READY -> PAUSED -> PLAYING. +// There is no possibility to pause and resume pipeline, it can be only once played. +// GAPI_EXPORTS here is only for testing purposes. +class GAPI_EXPORTS GStreamerPipelineFacade +{ +public: + // Strong exception guarantee. + explicit GStreamerPipelineFacade(const std::string& pipeline); + + // The destructors are noexcept by default. (since C++11) + ~GStreamerPipelineFacade(); + + // Elements getters are not guarded with mutexes because elements order is not supposed + // to change in the pipeline. + std::vector getElementsByFactoryName(const std::string& factoryName); + GstElement* getElementByName(const std::string& elementName); + + // Pipeline state modifiers: can be called only once, MT-safe, mutually exclusive. + void completePreroll(); + void play(); + + // Pipeline state checker: MT-safe. + bool isPlaying(); + +private: + std::string m_pipelineDesc; + + GStreamerPtr m_pipeline; + + std::atomic m_isPrerolled; + std::atomic m_isPlaying; + // Mutex to guard state(paused, playing) from changes from multiple threads + std::mutex m_stateChangeMutex; + +private: + // This constructor is needed only to make public constructor as delegating constructor + // and allow it to throw exceptions. + GStreamerPipelineFacade(); + + // Elements getter is not guarded with mutex because elements order is not supposed + // to change in the pipeline. + std::vector getElements(std::function comparator); + + // Getters, modifiers, verifiers are not MT-safe, because they are called from + // MT-safe mutually exclusive public functions. + PipelineState queryState(); + void setState(GstState state); + void verifyStateChange(GstStateChangeReturn status); + void checkBusMessages() const; +}; + +} // namespace gst +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_GSTREAMER +#endif // OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMER_PIPELINE_FACADE_HPP diff --git a/modules/gapi/src/streaming/gstreamer/gstreamerenv.cpp b/modules/gapi/src/streaming/gstreamer/gstreamerenv.cpp new file mode 100644 index 0000000000..138589b9a6 --- /dev/null +++ b/modules/gapi/src/streaming/gstreamer/gstreamerenv.cpp @@ -0,0 +1,90 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include "gstreamerenv.hpp" +#include "gstreamerptr.hpp" + +#ifdef HAVE_GSTREAMER +#include +#endif // HAVE_GSTREAMER + +namespace cv { +namespace gapi { +namespace wip { +namespace gst { + +#ifdef HAVE_GSTREAMER + +const GStreamerEnv& GStreamerEnv::init() +{ + static GStreamerEnv gInit; + return gInit; +} + +GStreamerEnv::GStreamerEnv() +{ + if (!gst_is_initialized()) + { + GError* error = NULL; + gst_init_check(NULL, NULL, &error); + + GStreamerPtr err(error); + + if (err) + { + cv::util::throw_error( + std::runtime_error(std::string("GStreamer initializaton error! Details: ") + + err->message)); + } + } + + // FIXME: GStreamer libs which have same MAJOR and MINOR versions are API and ABI compatible. + // If GStreamer runtime MAJOR version differs from the version the application was + // compiled with, will it fail on the linkage stage? If so, the code below isn't needed. + guint major, minor, micro, nano; + gst_version(&major, &minor, µ, &nano); + if (GST_VERSION_MAJOR != major) + { + cv::util::throw_error( + std::runtime_error(std::string("Incompatible GStreamer version: compiled with ") + + std::to_string(GST_VERSION_MAJOR) + '.' + + std::to_string(GST_VERSION_MINOR) + '.' + + std::to_string(GST_VERSION_MICRO) + '.' + + std::to_string(GST_VERSION_NANO) + + ", but runtime has " + + std::to_string(major) + '.' + std::to_string(minor) + '.' + + std::to_string(micro) + '.' + std::to_string(nano) + '.')); + } +} + +GStreamerEnv::~GStreamerEnv() +{ + gst_deinit(); +} + +#else // HAVE_GSTREAMER + +const GStreamerEnv& GStreamerEnv::init() +{ + GAPI_Assert(false && "Built without GStreamer support!"); +} + +GStreamerEnv::GStreamerEnv() +{ + GAPI_Assert(false && "Built without GStreamer support!"); +} + +GStreamerEnv::~GStreamerEnv() +{ + // No need an assert here. The assert raise C4722 warning. Constructor have already got assert. +} + +#endif // HAVE_GSTREAMER + +} // namespace gst +} // namespace wip +} // namespace gapi +} // namespace cv diff --git a/modules/gapi/src/streaming/gstreamer/gstreamerenv.hpp b/modules/gapi/src/streaming/gstreamer/gstreamerenv.hpp new file mode 100644 index 0000000000..04f7f0c16b --- /dev/null +++ b/modules/gapi/src/streaming/gstreamer/gstreamerenv.hpp @@ -0,0 +1,37 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMERENV_HPP +#define OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMERENV_HPP + +namespace cv { +namespace gapi { +namespace wip { +namespace gst { + +/*! + * \brief The GStreamerEnv class + * Initializes gstreamer once in the whole process + * + * + * @note You need to build OpenCV with GStreamer support to use this class. + */ +class GStreamerEnv +{ +public: + static const GStreamerEnv& init(); + +private: + GStreamerEnv(); + ~GStreamerEnv(); +}; + +} // namespace gst +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMERENV_HPP diff --git a/modules/gapi/src/streaming/gstreamer/gstreamerpipeline.cpp b/modules/gapi/src/streaming/gstreamer/gstreamerpipeline.cpp new file mode 100644 index 0000000000..6687076c7e --- /dev/null +++ b/modules/gapi/src/streaming/gstreamer/gstreamerpipeline.cpp @@ -0,0 +1,112 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include "gstreamer_pipeline_facade.hpp" +#include "gstreamerpipeline_priv.hpp" +#include + +#ifdef HAVE_GSTREAMER +#include +#endif // HAVE_GSTREAMER + +namespace cv { +namespace gapi { +namespace wip { +namespace gst { + +#ifdef HAVE_GSTREAMER + +GStreamerPipeline::Priv::Priv(const std::string& pipelineDesc): + m_pipeline(std::make_shared(pipelineDesc)) +{ + std::vector appsinks = + m_pipeline->getElementsByFactoryName("appsink"); + + for (std::size_t i = 0ul; i < appsinks.size(); ++i) + { + auto* appsink = appsinks[i]; + GAPI_Assert(appsink != nullptr); + GStreamerPtr name(gst_element_get_name(appsink)); + auto result = m_appsinkNamesToUse.insert({ name.get(), true /* free */ }); + GAPI_Assert(std::get<1>(result) && "Each appsink name must be unique!"); + } +} + +IStreamSource::Ptr GStreamerPipeline::Priv::getStreamingSource( + const std::string& appsinkName, const GStreamerSource::OutputType outputType) +{ + auto appsinkNameIt = m_appsinkNamesToUse.find(appsinkName); + if (appsinkNameIt == m_appsinkNamesToUse.end()) + { + cv::util::throw_error(std::logic_error(std::string("There is no appsink element in the " + "pipeline with the name '") + appsinkName + "'.")); + } + + if (!appsinkNameIt->second) + { + cv::util::throw_error(std::logic_error(std::string("appsink element with the name '") + + appsinkName + "' has been already used to create a GStreamerSource!")); + } + + m_appsinkNamesToUse[appsinkName] = false /* not free */; + + IStreamSource::Ptr src; + try { + src = cv::gapi::wip::make_src(m_pipeline, appsinkName, + outputType); + } + catch(...) { + m_appsinkNamesToUse[appsinkName] = true; /* free */ + cv::util::throw_error(std::runtime_error(std::string("Error during creation of ") + + "GStreamerSource on top of '" + appsinkName + "' appsink element!")); + } + + return src; +} + +GStreamerPipeline::Priv::~Priv() { } + +#else // HAVE_GSTREAMER + +GStreamerPipeline::Priv::Priv(const std::string&) +{ + GAPI_Assert(false && "Built without GStreamer support!"); +} + +IStreamSource::Ptr GStreamerPipeline::Priv::getStreamingSource(const std::string&, + const GStreamerSource::OutputType) +{ + // No need an assert here. The assert raise C4702 warning. Constructor have already got assert. + return nullptr; +} + +GStreamerPipeline::Priv::~Priv() +{ + // No need an assert here. The assert raise C4722 warning. Constructor have already got assert. +} + +#endif // HAVE_GSTREAMER + +GStreamerPipeline::GStreamerPipeline(const std::string& pipelineDesc): + m_priv(new Priv(pipelineDesc)) { } + +IStreamSource::Ptr GStreamerPipeline::getStreamingSource( + const std::string& appsinkName, const GStreamerSource::OutputType outputType) +{ + return m_priv->getStreamingSource(appsinkName, outputType); +} + +GStreamerPipeline::~GStreamerPipeline() +{ } + +GStreamerPipeline::GStreamerPipeline(std::unique_ptr priv): + m_priv(std::move(priv)) +{ } + +} // namespace gst +} // namespace wip +} // namespace gapi +} // namespace cv diff --git a/modules/gapi/src/streaming/gstreamer/gstreamerpipeline_priv.hpp b/modules/gapi/src/streaming/gstreamer/gstreamerpipeline_priv.hpp new file mode 100644 index 0000000000..4b10d1011c --- /dev/null +++ b/modules/gapi/src/streaming/gstreamer/gstreamerpipeline_priv.hpp @@ -0,0 +1,58 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMERPIPELINE_PRIV_HPP +#define OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMERPIPELINE_PRIV_HPP + +#include + +#include +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace gst { + +#ifdef HAVE_GSTREAMER + +class GStreamerPipeline::Priv +{ +public: + explicit Priv(const std::string& pipeline); + + IStreamSource::Ptr getStreamingSource(const std::string& appsinkName, + const GStreamerSource::OutputType outputType); + + virtual ~Priv(); + +protected: + std::shared_ptr m_pipeline; + std::unordered_map m_appsinkNamesToUse; +}; + +#else // HAVE_GSTREAMER + +class GStreamerPipeline::Priv +{ +public: + explicit Priv(const std::string& pipeline); + + IStreamSource::Ptr getStreamingSource(const std::string& appsinkName, + const GStreamerSource::OutputType outputType); + + virtual ~Priv(); +}; + +#endif // HAVE_GSTREAMER + +} // namespace gst +} // namespace wip +} // namespace gapi +} // namespace cv + + +#endif // OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMERPIPELINE_PRIV_HPP diff --git a/modules/gapi/src/streaming/gstreamer/gstreamerptr.hpp b/modules/gapi/src/streaming/gstreamer/gstreamerptr.hpp new file mode 100644 index 0000000000..6e6bba37e3 --- /dev/null +++ b/modules/gapi/src/streaming/gstreamer/gstreamerptr.hpp @@ -0,0 +1,177 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMERPTR_HPP +#define OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMERPTR_HPP + +#include + +#include + +#ifdef HAVE_GSTREAMER +#include +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace gst { + +template static inline void GStreamerPtrUnrefObject(T* ptr) +{ + if (ptr) + { + gst_object_unref(G_OBJECT(ptr)); + } +} + +template static inline void GStreamerPtrRelease(T* ptr); + +template<> inline void GStreamerPtrRelease(GError* ptr) +{ + if (ptr) + { + g_error_free(ptr); + } +} + +template<> inline void GStreamerPtrRelease(GstElement* ptr) +{ + GStreamerPtrUnrefObject(ptr); +} + +template<> inline void GStreamerPtrRelease(GstElementFactory* ptr) +{ + GStreamerPtrUnrefObject(ptr); +} + +template<> inline void GStreamerPtrRelease(GstPad* ptr) +{ + GStreamerPtrUnrefObject(ptr); +} + +template<> inline void GStreamerPtrRelease(GstBus* ptr) +{ + GStreamerPtrUnrefObject(ptr); +} + +template<> inline void GStreamerPtrRelease(GstAllocator* ptr) +{ + GStreamerPtrUnrefObject(ptr); +} + +template<> inline void GStreamerPtrRelease(GstVideoInfo* ptr) +{ + if (ptr) + { + gst_video_info_free(ptr); + } +} + +template<> inline void GStreamerPtrRelease(GstCaps* ptr) +{ + if (ptr) + { + gst_caps_unref(ptr); + } +} + +template<> inline void GStreamerPtrRelease(GstMemory* ptr) +{ + if (ptr) + { + gst_memory_unref(ptr); + } +} + +template<> inline void GStreamerPtrRelease(GstBuffer* ptr) +{ + if (ptr) + { + gst_buffer_unref(ptr); + } +} + +template<> inline void GStreamerPtrRelease(GstSample* ptr) +{ + if (ptr) + { + gst_sample_unref(ptr); + } +} + +template<> inline void GStreamerPtrRelease(GstMessage* ptr) +{ + if (ptr) + { + gst_message_unref(ptr); + } +} + +template<> inline void GStreamerPtrRelease(GstIterator* ptr) +{ + if (ptr) + { + gst_iterator_free(ptr); + } +} + +template<> inline void GStreamerPtrRelease(GstQuery* ptr) +{ + if (ptr) + { + gst_query_unref(ptr); + } +} + +template<> inline void GStreamerPtrRelease(char* ptr) +{ + if (ptr) + { + g_free(ptr); + } +} + +// NOTE: The main concept of this class is to be owner of some passed to it piece of memory. +// (be owner = free this memory or reduce reference count to it after use). +// More specifically, GStreamerPtr is designed to own memory returned from GStreamer/GLib +// functions, which are marked as [transfer full] in documentation. +// [transfer full] means that function fully transfers ownership of returned memory to the +// receiving piece of code. +// +// Memory ownership and ownership transfer concept: +// https://developer.gnome.org/programming-guidelines/stable/memory-management.html.en#g-clear-object + +// NOTE: GStreamerPtr can only own strong references, not floating ones. +// For floating references please call g_object_ref_sink(reference) before wrapping +// it with GStreamerPtr. +// See https://developer.gnome.org/gobject/stable/gobject-The-Base-Object-Type.html#floating-ref +// for floating references. +// NOTE: GStreamerPtr doesn't support pointers to arrays, only pointers to single objects. +template class GStreamerPtr : + public std::unique_ptr)> +{ + using BaseClass = std::unique_ptr)>; + +public: + constexpr GStreamerPtr() noexcept : BaseClass(nullptr, GStreamerPtrRelease) { } + constexpr GStreamerPtr(std::nullptr_t) noexcept : BaseClass(nullptr, GStreamerPtrRelease) { } + explicit GStreamerPtr(typename BaseClass::pointer p) noexcept : + BaseClass(p, GStreamerPtrRelease) { } + + GStreamerPtr& operator=(T* p) noexcept { *this = std::move(GStreamerPtr(p)); return *this; } + + inline operator T*() noexcept { return this->get(); } + // There is no const correctness in GStreamer C API + inline operator /*const*/ T*() const noexcept { return (T*)this->get(); } +}; + +} // namespace gst +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_GSTREAMER +#endif // OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMERPTR_HPP diff --git a/modules/gapi/src/streaming/gstreamer/gstreamersource.cpp b/modules/gapi/src/streaming/gstreamer/gstreamersource.cpp new file mode 100644 index 0000000000..661125657c --- /dev/null +++ b/modules/gapi/src/streaming/gstreamer/gstreamersource.cpp @@ -0,0 +1,383 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include "gstreamer_buffer_utils.hpp" + +#include "gstreamer_media_adapter.hpp" + +#include "gstreamersource_priv.hpp" +#include + +#include + +#include + +#include + +#ifdef HAVE_GSTREAMER +#include +#include +#include +#endif // HAVE_GSTREAMER + +namespace cv { +namespace gapi { +namespace wip { +namespace gst { + +#ifdef HAVE_GSTREAMER + +constexpr char NV12_CAPS_STRING[] = + "video/x-raw,format=NV12;video/x-raw(memory:DMABuf),format=NV12"; + +namespace { +GstPadProbeReturn appsinkQueryCallback(GstPad*, GstPadProbeInfo* info, gpointer) +{ + GstQuery *query = GST_PAD_PROBE_INFO_QUERY(info); + + if (GST_QUERY_TYPE(query) != GST_QUERY_ALLOCATION) + return GST_PAD_PROBE_OK; + + gst_query_add_allocation_meta(query, GST_VIDEO_META_API_TYPE, NULL); + + return GST_PAD_PROBE_HANDLED; +} +} // anonymous namespace + +GStreamerSource::Priv::Priv(const std::string& pipelineDesc, + const GStreamerSource::OutputType outputType) : + m_pipeline(std::make_shared(pipelineDesc)), + m_outputType(outputType) +{ + GAPI_Assert((m_outputType == GStreamerSource::OutputType::FRAME || + m_outputType == GStreamerSource::OutputType::MAT) + && "Unsupported output type for GStreamerSource!"); + + auto appsinks = m_pipeline->getElementsByFactoryName("appsink"); + GAPI_Assert(1ul == appsinks.size() && + "GStreamerSource can accept pipeline with only 1 appsink element inside!\n" + "Please, note, that amount of sink elements of other than appsink type is not limited.\n"); + + m_appsink = GST_ELEMENT(gst_object_ref(appsinks[0])); + + configureAppsink(); +} + +GStreamerSource::Priv::Priv(std::shared_ptr pipeline, + const std::string& appsinkName, + const GStreamerSource::OutputType outputType) : + m_pipeline(pipeline), + m_outputType(outputType) +{ + GAPI_Assert((m_outputType == GStreamerSource::OutputType::FRAME || + m_outputType == GStreamerSource::OutputType::MAT) + && "Unsupported output type for GStreamerSource!"); + + m_appsink = (GST_ELEMENT(gst_object_ref(m_pipeline->getElementByName(appsinkName)))); + configureAppsink(); +} + +bool GStreamerSource::Priv::pull(cv::gapi::wip::Data& data) +{ + bool result = false; + switch(m_outputType) { + case GStreamerSource::OutputType::FRAME: { + cv::MediaFrame frame; + result = retrieveFrame(frame); + if (result) { + data = frame; + } + break; + } + case GStreamerSource::OutputType::MAT: { + cv::Mat mat; + result = retrieveFrame(mat); + if (result) { + data = mat; + } + break; + } + } + + if (result) { + data.meta[cv::gapi::streaming::meta_tag::timestamp] = computeTimestamp(); + data.meta[cv::gapi::streaming::meta_tag::seq_id] = m_frameId++; + } + + return result; +} + +GMetaArg GStreamerSource::Priv::descr_of() noexcept +{ + // Prepare frame metadata if it wasn't prepared yet. + prepareVideoMeta(); + + switch(m_outputType) { + case GStreamerSource::OutputType::FRAME: { + return GMetaArg { m_mediaFrameMeta }; + } + case GStreamerSource::OutputType::MAT: { + return GMetaArg { m_matMeta }; + } + } + + return GMetaArg { }; +} + +void GStreamerSource::Priv::configureAppsink() { + // NOTE: appsink element should be configured before pipeline launch. + GAPI_Assert(!m_pipeline->isPlaying()); + + // TODO: is 1 single buffer really high enough? + gst_app_sink_set_max_buffers(GST_APP_SINK(m_appsink.get()), 1); + + // Do not emit signals: all calls will be synchronous and blocking. + gst_app_sink_set_emit_signals(GST_APP_SINK(m_appsink.get()), FALSE); + + GStreamerPtr nv12Caps(gst_caps_from_string(NV12_CAPS_STRING)); + + GStreamerPtr appsinkPad(gst_element_get_static_pad(m_appsink, "sink")); + GStreamerPtr peerCaps(gst_pad_peer_query_caps(appsinkPad, NULL)); + if (!gst_caps_can_intersect(peerCaps, nv12Caps)) { + cv::util::throw_error( + std::logic_error("appsink element can only consume video-frame in NV12 format in " + "GStreamerSource")); + } + + gst_app_sink_set_caps(GST_APP_SINK(m_appsink.get()), nv12Caps); + + gst_pad_add_probe(appsinkPad, GST_PAD_PROBE_TYPE_QUERY_DOWNSTREAM, appsinkQueryCallback, + NULL, NULL); +} + +void GStreamerSource::Priv::prepareVideoMeta() +{ + if (!m_isMetaPrepared) { + m_pipeline->completePreroll(); + + GStreamerPtr prerollSample( +#if GST_VERSION_MINOR >= 10 + gst_app_sink_try_pull_preroll(GST_APP_SINK(m_appsink.get()), 5 * GST_SECOND)); +#else // GST_VERSION_MINOR < 10 + // TODO: This function may cause hang with some pipelines, need to check whether these + // pipelines are really wrong or not? + gst_app_sink_pull_preroll(GST_APP_SINK(m_appsink.get()))); +#endif // GST_VERSION_MINOR >= 10 + GAPI_Assert(prerollSample != nullptr); + + GstCaps* prerollCaps(gst_sample_get_caps(prerollSample)); + GAPI_Assert(prerollCaps != nullptr); + + const GstStructure* structure = gst_caps_get_structure(prerollCaps, 0); + + // Width and height held in GstCaps structure are format-independent width and height + // of the frame. So the actual height of the output buffer in NV12 format will be + // height * 3/2. + int width = 0; + int height = 0; + if (!gst_structure_get_int(structure, "width", &width) || + !gst_structure_get_int(structure, "height", &height)) + { + cv::util::throw_error(std::logic_error("Cannot query video width/height.")); + } + + switch(m_outputType) { + case GStreamerSource::OutputType::FRAME: { + // Construct metadata for media frame. + m_mediaFrameMeta = GFrameDesc { cv::MediaFormat::NV12, cv::Size(width, height) }; + break; + } + case GStreamerSource::OutputType::MAT: { + // Construct metadata for BGR mat. + m_matMeta = GMatDesc { CV_8U, 3, cv::Size(width, height), false }; + break; + } + } + + // Fill GstVideoInfo structure to work further with GstVideoFrame class. + if (!gst_video_info_from_caps(&m_videoInfo, prerollCaps)) { + cv::util::throw_error(std::logic_error("preroll sample has invalid caps.")); + } + GAPI_Assert(GST_VIDEO_INFO_N_PLANES(&m_videoInfo) == 2); + GAPI_Assert(GST_VIDEO_INFO_FORMAT(&m_videoInfo) == GST_VIDEO_FORMAT_NV12); + + m_isMetaPrepared = true; + } +} + +int64_t GStreamerSource::Priv::computeTimestamp() +{ + GAPI_Assert(m_buffer && "Pulled buffer is nullptr!"); + + int64_t timestamp { }; + + GstClockTime pts = GST_BUFFER_PTS(m_buffer); + if (GST_CLOCK_TIME_IS_VALID(pts)) { + timestamp = GST_TIME_AS_USECONDS(pts); + } else { + const auto now = std::chrono::system_clock::now(); + const auto dur = std::chrono::duration_cast + (now.time_since_epoch()); + timestamp = int64_t{dur.count()}; + } + + return timestamp; +} + +bool GStreamerSource::Priv::pullBuffer() +{ + // Start the pipeline if it is not in the playing state yet + if (!m_isPipelinePlaying) { + m_pipeline->play(); + m_isPipelinePlaying = true; + } + + // Bail out if EOS + if (gst_app_sink_is_eos(GST_APP_SINK(m_appsink.get()))) + { + return false; + } + + m_sample = gst_app_sink_pull_sample(GST_APP_SINK(m_appsink.get())); + + // TODO: GAPI_Assert because of already existed check for EOS? + if (m_sample == nullptr) + { + return false; + } + + m_buffer = gst_sample_get_buffer(m_sample); + GAPI_Assert(m_buffer && "Grabbed sample has no buffer!"); + + return true; +} + +bool GStreamerSource::Priv::retrieveFrame(cv::Mat& data) +{ + // Prepare metadata if it isn't prepared yet. + prepareVideoMeta(); + + bool result = pullBuffer(); + if (!result) + { + return false; + } + + // TODO: Use RAII for map/unmap + GstVideoFrame videoFrame; + gstreamer_utils::mapBufferToFrame(*m_buffer, m_videoInfo, videoFrame, GST_MAP_READ); + + try + { + // m_matMeta holds width and height for 8U BGR frame, but actual + // frame m_buffer we request from GStreamer pipeline has 8U NV12 format. + // Constructing y and uv cv::Mat-s from such a m_buffer: + GAPI_Assert((uint8_t*)GST_VIDEO_FRAME_PLANE_DATA(&videoFrame, 1) == + (uint8_t*)GST_VIDEO_FRAME_PLANE_DATA(&videoFrame, 0) + + GST_VIDEO_FRAME_PLANE_OFFSET(&videoFrame, 1)); + + cv::Mat y(m_matMeta.size, CV_8UC1, + (uint8_t*)GST_VIDEO_FRAME_PLANE_DATA(&videoFrame, 0) + + GST_VIDEO_FRAME_PLANE_OFFSET(&videoFrame, 0), + GST_VIDEO_FRAME_PLANE_STRIDE(&videoFrame, 0)); + cv::Mat uv(m_matMeta.size / 2, CV_8UC2, + (uint8_t*)GST_VIDEO_FRAME_PLANE_DATA(&videoFrame, 0) + + GST_VIDEO_FRAME_PLANE_OFFSET(&videoFrame, 1), + GST_VIDEO_FRAME_PLANE_STRIDE(&videoFrame, 1)); + + cv::cvtColorTwoPlane(y, uv, data, cv::COLOR_YUV2BGR_NV12); + } + catch (...) + { + gst_video_frame_unmap(&videoFrame); + cv::util::throw_error(std::runtime_error("NV12 buffer conversion to BGR is failed!")); + } + gst_video_frame_unmap(&videoFrame); + + return true; +} + +bool GStreamerSource::Priv::retrieveFrame(cv::MediaFrame& data) +{ + // Prepare metadata if it isn't prepared yet. + prepareVideoMeta(); + + bool result = pullBuffer(); + if (!result) + { + return false; + } + + data = cv::MediaFrame::Create(m_mediaFrameMeta, &m_videoInfo, + m_buffer); + + return true; +} + +GStreamerSource::Priv::~Priv() { } + +#else // HAVE_GSTREAMER + +GStreamerSource::Priv::Priv(const std::string&, const GStreamerSource::OutputType) +{ + GAPI_Assert(false && "Built without GStreamer support!"); +} + +GStreamerSource::Priv::Priv(std::shared_ptr, const std::string&, + const GStreamerSource::OutputType) +{ + GAPI_Assert(false && "Built without GStreamer support!"); +} + +bool GStreamerSource::Priv::pull(cv::gapi::wip::Data&) +{ + // No need an assert here. Constructor have already got assert. + return false; +} + +GMetaArg GStreamerSource::Priv::descr_of() const noexcept +{ + // No need an assert here. The assert raise C4702 warning. Constructor have already got assert. + return GMetaArg{}; +} + +GStreamerSource::Priv::~Priv() +{ + // No need an assert here. The assert raise C4722 warning. Constructor have already got assert. +} + +#endif // HAVE_GSTREAMER + +GStreamerSource::GStreamerSource(const std::string& pipeline, + const GStreamerSource::OutputType outputType): + m_priv(new Priv(pipeline, outputType)) { } + +GStreamerSource::GStreamerSource(std::shared_ptr pipeline, + const std::string& appsinkName, + const GStreamerSource::OutputType outputType): + m_priv(new Priv(pipeline, appsinkName, outputType)) { } + +bool GStreamerSource::pull(cv::gapi::wip::Data& data) +{ + return m_priv->pull(data); +} + +GMetaArg GStreamerSource::descr_of() const +{ + return m_priv->descr_of(); +} + +GStreamerSource::~GStreamerSource() +{ } + +GStreamerSource::GStreamerSource(std::unique_ptr priv): + m_priv(std::move(priv)) +{ } + +} // namespace gst +} // namespace wip +} // namespace gapi +} // namespace cv diff --git a/modules/gapi/src/streaming/gstreamer/gstreamersource_priv.hpp b/modules/gapi/src/streaming/gstreamer/gstreamersource_priv.hpp new file mode 100644 index 0000000000..b0940c48a3 --- /dev/null +++ b/modules/gapi/src/streaming/gstreamer/gstreamersource_priv.hpp @@ -0,0 +1,94 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMERSOURCE_PRIV_HPP +#define OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMERSOURCE_PRIV_HPP + +#include "gstreamerptr.hpp" +#include "gstreamer_pipeline_facade.hpp" +#include + +#include + +#ifdef HAVE_GSTREAMER +#include +#include +#endif // HAVE_GSTREAMER + +namespace cv { +namespace gapi { +namespace wip { +namespace gst { + +#ifdef HAVE_GSTREAMER + +class GStreamerSource::Priv +{ +public: + Priv(const std::string& pipeline, const GStreamerSource::OutputType outputType); + + Priv(std::shared_ptr pipeline, const std::string& appsinkName, + const GStreamerSource::OutputType outputType); + + bool pull(cv::gapi::wip::Data& data); + + // non-const in difference with GStreamerSource, because contains delayed meta initialization + GMetaArg descr_of() noexcept; + + virtual ~Priv(); + +protected: + // Shares: + std::shared_ptr m_pipeline; + + // Owns: + GStreamerPtr m_appsink; + GStreamerPtr m_sample; + GstBuffer* m_buffer = nullptr; // Actual frame memory holder + GstVideoInfo m_videoInfo; // Information about Video frame + + GStreamerSource::OutputType m_outputType = GStreamerSource::OutputType::MAT; + + GMatDesc m_matMeta; + GFrameDesc m_mediaFrameMeta; + + bool m_isMetaPrepared = false; + bool m_isPipelinePlaying = false; + + int64_t m_frameId = 0L; + +protected: + void configureAppsink(); + void prepareVideoMeta(); + + int64_t computeTimestamp(); + + bool pullBuffer(); + bool retrieveFrame(cv::Mat& data); + bool retrieveFrame(cv::MediaFrame& data); +}; + +#else // HAVE_GSTREAMER + +class GStreamerSource::Priv +{ +public: + Priv(const std::string& pipeline, const GStreamerSource::OutputType outputType); + Priv(std::shared_ptr pipeline, const std::string& appsinkName, + const GStreamerSource::OutputType outputType); + bool pull(cv::gapi::wip::Data& data); + GMetaArg descr_of() const noexcept; + virtual ~Priv(); +}; + +#endif // HAVE_GSTREAMER + +} // namespace gst +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // OPENCV_GAPI_STREAMING_GSTREAMER_GSTREAMERSOURCE_PRIV_HPP diff --git a/modules/gapi/test/streaming/gapi_gstreamer_pipeline_facade_int_tests.cpp b/modules/gapi/test/streaming/gapi_gstreamer_pipeline_facade_int_tests.cpp new file mode 100644 index 0000000000..7b809a8847 --- /dev/null +++ b/modules/gapi/test/streaming/gapi_gstreamer_pipeline_facade_int_tests.cpp @@ -0,0 +1,188 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include "../test/common/gapi_tests_common.hpp" + +#include "../src/streaming/gstreamer/gstreamer_pipeline_facade.hpp" +#include "../src/streaming/gstreamer/gstreamerptr.hpp" + +#include + +#include + +#ifdef HAVE_GSTREAMER +#include + +namespace opencv_test +{ + +TEST(GStreamerPipelineFacadeTest, GetElsByFactoryNameUnitTest) +{ + auto comparator = [](GstElement* element, const std::string& factoryName) { + cv::gapi::wip::gst::GStreamerPtr name( + gst_object_get_name(GST_OBJECT(gst_element_get_factory(element)))); + return name && (0 == strcmp(name, factoryName.c_str())); + }; + + cv::gapi::wip::gst::GStreamerPipelineFacade + pipelineFacade("videotestsrc is-live=true pattern=colors num-buffers=10 ! " + "videorate ! videoscale ! " + "video/x-raw,width=1920,height=1080,framerate=3/1 ! " + "appsink name=sink1 " + "videotestsrc is-live=true pattern=colors num-buffers=10 ! " + "videorate ! videoscale ! " + "video/x-raw,width=1920,height=1080,framerate=3/1 ! " + "appsink name=sink2"); + + auto videotestsrcs = pipelineFacade.getElementsByFactoryName("videotestsrc"); + EXPECT_EQ(2u, videotestsrcs.size()); + for (auto&& src : videotestsrcs) { + EXPECT_TRUE(comparator(src, "videotestsrc")); + } + + auto appsinks = pipelineFacade.getElementsByFactoryName("appsink"); + EXPECT_EQ(2u, appsinks.size()); + for (auto&& sink : appsinks) { + EXPECT_TRUE(comparator(sink, "appsink")); + } +} + +TEST(GStreamerPipelineFacadeTest, GetElByNameUnitTest) +{ + auto comparator = [](GstElement* element, const std::string& elementName) { + cv::gapi::wip::gst::GStreamerPtr name(gst_element_get_name(element)); + return name && (0 == strcmp(name, elementName.c_str())); + }; + + cv::gapi::wip::gst::GStreamerPipelineFacade + pipelineFacade("videotestsrc is-live=true pattern=colors num-buffers=10 ! " + "videorate ! videoscale ! " + "video/x-raw,width=1920,height=1080,framerate=3/1 ! " + "appsink name=sink1 " + "videotestsrc is-live=true pattern=colors num-buffers=10 ! " + "videorate ! videoscale ! " + "video/x-raw,width=1920,height=1080,framerate=3/1 ! " + "appsink name=sink2"); + + auto appsink1 = pipelineFacade.getElementByName("sink1"); + GAPI_Assert(appsink1 != nullptr); + EXPECT_TRUE(comparator(appsink1, "sink1")); + auto appsink2 = pipelineFacade.getElementByName("sink2"); + GAPI_Assert(appsink2 != nullptr); + EXPECT_TRUE(comparator(appsink2, "sink2")); +} + +TEST(GStreamerPipelineFacadeTest, CompletePrerollUnitTest) +{ + cv::gapi::wip::gst::GStreamerPipelineFacade + pipelineFacade("videotestsrc is-live=true pattern=colors num-buffers=10 ! " + "videorate ! videoscale ! " + "video/x-raw,width=1920,height=1080,framerate=3/1 ! " + "appsink name=sink1 " + "videotestsrc is-live=true pattern=colors num-buffers=10 ! " + "videorate ! videoscale ! " + "video/x-raw,width=1920,height=1080,framerate=3/1 ! " + "appsink name=sink2"); + + auto appsink = pipelineFacade.getElementByName("sink1"); + pipelineFacade.completePreroll(); + + cv::gapi::wip::gst::GStreamerPtr prerollSample( +#if GST_VERSION_MINOR >= 10 + gst_app_sink_try_pull_preroll(GST_APP_SINK(appsink), 5 * GST_SECOND) +#else // GST_VERSION_MINOR < 10 + // TODO: This function may cause hang with some pipelines, need to check whether these + // pipelines are really wrong or not? + gst_app_sink_pull_preroll(GST_APP_SINK(appsink)) +#endif // GST_VERSION_MINOR >= 10 + ); + GAPI_Assert(prerollSample != nullptr); +} + +TEST(GStreamerPipelineFacadeTest, PlayUnitTest) +{ + cv::gapi::wip::gst::GStreamerPipelineFacade + pipelineFacade("videotestsrc is-live=true pattern=colors num-buffers=10 ! " + "videorate ! videoscale ! " + "video/x-raw,width=1920,height=1080,framerate=3/1 ! " + "appsink name=sink1 " + "videotestsrc is-live=true pattern=colors num-buffers=10 ! " + "videorate ! videoscale ! " + "video/x-raw,width=1920,height=1080,framerate=3/1 ! " + "appsink name=sink2"); + + auto appsink = pipelineFacade.getElementByName("sink2"); + + pipelineFacade.play(); + + cv::gapi::wip::gst::PipelineState state; + GstStateChangeReturn status = + gst_element_get_state(appsink, &state.current, &state.pending, 5 * GST_SECOND); + EXPECT_EQ(GST_STATE_CHANGE_SUCCESS, status); + EXPECT_EQ(GST_STATE_PLAYING, state.current); + EXPECT_EQ(GST_STATE_VOID_PENDING, state.pending); +} + +TEST(GStreamerPipelineFacadeTest, IsPlayingUnitTest) +{ + cv::gapi::wip::gst::GStreamerPipelineFacade + pipelineFacade("videotestsrc is-live=true pattern=colors num-buffers=10 ! " + "videorate ! videoscale ! " + "video/x-raw,width=1920,height=1080,framerate=3/1 ! " + "appsink name=sink1 " + "videotestsrc is-live=true pattern=colors num-buffers=10 ! " + "videorate ! videoscale ! " + "video/x-raw,width=1920,height=1080,framerate=3/1 ! " + "appsink name=sink2"); + + EXPECT_EQ(false, pipelineFacade.isPlaying()); + pipelineFacade.play(); + EXPECT_EQ(true, pipelineFacade.isPlaying()); +} + +TEST(GStreamerPipelineFacadeTest, MTSafetyUnitTest) +{ + cv::gapi::wip::gst::GStreamerPipelineFacade + pipelineFacade("videotestsrc is-live=true pattern=colors num-buffers=10 ! " + "videorate ! videoscale ! " + "video/x-raw,width=1920,height=1080,framerate=3/1 ! " + "appsink name=sink1 " + "videotestsrc is-live=true pattern=colors num-buffers=10 ! " + "videorate ! videoscale ! " + "video/x-raw,width=1920,height=1080,framerate=3/1 ! " + "appsink name=sink2"); + + auto prerollRoutine = [&pipelineFacade](){ pipelineFacade.completePreroll(); }; + auto playRoutine = [&pipelineFacade](){ pipelineFacade.play(); }; + auto isPlayingRoutine = [&pipelineFacade](){ pipelineFacade.isPlaying(); }; + + using f = std::function; + + auto routinesLauncher = [](const f& r1, const f& r2, const f& r3) { + std::vector routines { r1, r2, r3 }; + std::vector threads { }; + + for (auto&& r : routines) { + threads.emplace_back(r); + } + + for (auto&& t : threads) { + t.join(); + } + }; + + routinesLauncher(prerollRoutine, playRoutine, isPlayingRoutine); + routinesLauncher(prerollRoutine, isPlayingRoutine, playRoutine); + routinesLauncher(isPlayingRoutine, prerollRoutine, playRoutine); + routinesLauncher(playRoutine, prerollRoutine, isPlayingRoutine); + routinesLauncher(playRoutine, isPlayingRoutine, prerollRoutine); + routinesLauncher(isPlayingRoutine, playRoutine, prerollRoutine); + + EXPECT_TRUE(true); +} +} // namespace opencv_test + +#endif // HAVE_GSTREAMER diff --git a/modules/gapi/test/streaming/gapi_gstreamersource_tests.cpp b/modules/gapi/test/streaming/gapi_gstreamersource_tests.cpp new file mode 100644 index 0000000000..0478d2dc1d --- /dev/null +++ b/modules/gapi/test/streaming/gapi_gstreamersource_tests.cpp @@ -0,0 +1,401 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include "../test/common/gapi_tests_common.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include + +#ifdef HAVE_GSTREAMER + +namespace opencv_test +{ + +struct GStreamerSourceTest : public TestWithParam> +{ }; + +TEST_P(GStreamerSourceTest, AccuracyTest) +{ + std::string pipeline; + cv::Size expectedFrameSize; + std::size_t streamLength { }; + std::tie(pipeline, expectedFrameSize, streamLength) = GetParam(); + + // Graph declaration: + cv::GMat in; + auto out = cv::gapi::copy(in); + cv::GComputation c(cv::GIn(in), cv::GOut(out)); + + // Graph compilation for streaming mode: + auto ccomp = c.compileStreaming(); + + EXPECT_TRUE(ccomp); + EXPECT_FALSE(ccomp.running()); + + // GStreamer streaming source configuration: + ccomp.setSource(pipeline); + + // Start of streaming: + ccomp.start(); + EXPECT_TRUE(ccomp.running()); + + // Streaming - pulling of frames until the end: + cv::Mat in_mat_gapi; + + EXPECT_TRUE(ccomp.pull(cv::gout(in_mat_gapi))); + EXPECT_TRUE(!in_mat_gapi.empty()); + EXPECT_EQ(expectedFrameSize, in_mat_gapi.size()); + EXPECT_EQ(CV_8UC3, in_mat_gapi.type()); + + std::size_t framesCount = 1UL; + while (ccomp.pull(cv::gout(in_mat_gapi))) { + EXPECT_TRUE(!in_mat_gapi.empty()); + EXPECT_EQ(expectedFrameSize, in_mat_gapi.size()); + EXPECT_EQ(CV_8UC3, in_mat_gapi.type()); + + framesCount++; + } + + EXPECT_FALSE(ccomp.running()); + ccomp.stop(); + + EXPECT_FALSE(ccomp.running()); + + EXPECT_EQ(streamLength, framesCount); +} + +TEST_P(GStreamerSourceTest, TimestampsTest) +{ + std::string pipeline; + std::size_t streamLength { }; + std::tie(pipeline, std::ignore, streamLength) = GetParam(); + + // Graph declaration: + cv::GMat in; + cv::GMat copied = cv::gapi::copy(in); + cv::GOpaque outId = cv::gapi::streaming::seq_id(copied); + cv::GOpaque outTs = cv::gapi::streaming::timestamp(copied); + cv::GComputation c(cv::GIn(in), cv::GOut(outId, outTs)); + + // Graph compilation for streaming mode: + auto ccomp = c.compileStreaming(); + + EXPECT_TRUE(ccomp); + EXPECT_FALSE(ccomp.running()); + + // GStreamer streaming source configuration: + ccomp.setSource(pipeline); + + // Start of streaming: + ccomp.start(); + EXPECT_TRUE(ccomp.running()); + + // Streaming - pulling of frames until the end: + int64_t seqId; + int64_t timestamp; + + std::vector allSeqIds; + std::vector allTimestamps; + + while (ccomp.pull(cv::gout(seqId, timestamp))) { + allSeqIds.push_back(seqId); + allTimestamps.push_back(timestamp); + } + + EXPECT_FALSE(ccomp.running()); + ccomp.stop(); + + EXPECT_FALSE(ccomp.running()); + + EXPECT_EQ(0L, allSeqIds.front()); + EXPECT_EQ(int64_t(streamLength) - 1, allSeqIds.back()); + EXPECT_EQ(streamLength, allSeqIds.size()); + EXPECT_TRUE(std::is_sorted(allSeqIds.begin(), allSeqIds.end())); + EXPECT_EQ(allSeqIds.size(), std::set(allSeqIds.begin(), allSeqIds.end()).size()); + + EXPECT_EQ(streamLength, allTimestamps.size()); + EXPECT_TRUE(std::is_sorted(allTimestamps.begin(), allTimestamps.end())); +} + +G_TYPED_KERNEL(GGstFrameCopyToNV12, (GFrame)>, + "org.opencv.test.gstframe_copy_to_nv12") +{ + static std::tuple outMeta(GFrameDesc desc) { + GMatDesc y { CV_8U, 1, desc.size, false }; + GMatDesc uv { CV_8U, 2, desc.size / 2, false }; + + return std::make_tuple(y, uv); + } +}; + +GAPI_OCV_KERNEL(GOCVGstFrameCopyToNV12, GGstFrameCopyToNV12) +{ + static void run(const cv::MediaFrame& in, cv::Mat& y, cv::Mat& uv) + { + auto view = in.access(cv::MediaFrame::Access::R); + cv::Mat ly(y.size(), y.type(), view.ptr[0], view.stride[0]); + cv::Mat luv(uv.size(), uv.type(), view.ptr[1], view.stride[1]); + + ly.copyTo(y); + luv.copyTo(uv); + } +}; + +TEST_P(GStreamerSourceTest, GFrameTest) +{ + std::string pipeline; + cv::Size expectedFrameSize; + std::size_t streamLength { }; + std::tie(pipeline, expectedFrameSize, streamLength) = GetParam(); + + // Graph declaration: + cv::GFrame in; + cv::GMat copiedY, copiedUV; + std::tie(copiedY, copiedUV) = GGstFrameCopyToNV12::on(in); + cv::GComputation c(cv::GIn(in), cv::GOut(copiedY, copiedUV)); + + // Graph compilation for streaming mode: + auto ccomp = c.compileStreaming(cv::compile_args(cv::gapi::kernels())); + + EXPECT_TRUE(ccomp); + EXPECT_FALSE(ccomp.running()); + + // GStreamer streaming source configuration: + ccomp.setSource + (pipeline, cv::gapi::wip::GStreamerSource::OutputType::FRAME); + + // Start of streaming: + ccomp.start(); + EXPECT_TRUE(ccomp.running()); + + // Streaming - pulling of frames until the end: + cv::Mat y_mat, uv_mat; + + EXPECT_TRUE(ccomp.pull(cv::gout(y_mat, uv_mat))); + EXPECT_TRUE(!y_mat.empty()); + EXPECT_TRUE(!uv_mat.empty()); + + cv::Size expectedYSize = expectedFrameSize; + cv::Size expectedUVSize = expectedFrameSize / 2; + + EXPECT_EQ(expectedYSize, y_mat.size()); + EXPECT_EQ(expectedUVSize, uv_mat.size()); + + EXPECT_EQ(CV_8UC1, y_mat.type()); + EXPECT_EQ(CV_8UC2, uv_mat.type()); + + std::size_t framesCount = 1UL; + while (ccomp.pull(cv::gout(y_mat, uv_mat))) { + EXPECT_TRUE(!y_mat.empty()); + EXPECT_TRUE(!uv_mat.empty()); + + EXPECT_EQ(expectedYSize, y_mat.size()); + EXPECT_EQ(expectedUVSize, uv_mat.size()); + + EXPECT_EQ(CV_8UC1, y_mat.type()); + EXPECT_EQ(CV_8UC2, uv_mat.type()); + + framesCount++; + } + + EXPECT_FALSE(ccomp.running()); + ccomp.stop(); + + EXPECT_FALSE(ccomp.running()); + + EXPECT_EQ(streamLength, framesCount); +} + +// FIXME: Need to launch with sudo. May be infrastructure problems. +// TODO: It is needed to add tests for streaming from native KMB camera: kmbcamsrc +// GStreamer element. +INSTANTIATE_TEST_CASE_P(CameraEmulatingPipeline, GStreamerSourceTest, + Combine(Values("videotestsrc is-live=true pattern=colors num-buffers=10 ! " + "videorate ! videoscale ! " + "video/x-raw,width=1920,height=1080,framerate=3/1 ! " + "appsink"), + Values(cv::Size(1920, 1080)), + Values(10UL))); + +INSTANTIATE_TEST_CASE_P(FileEmulatingPipeline, GStreamerSourceTest, + Combine(Values("videotestsrc pattern=colors num-buffers=10 ! " + "videorate ! videoscale ! " + "video/x-raw,width=640,height=420,framerate=3/1 ! " + "appsink"), + Values(cv::Size(640, 420)), + Values(10UL))); + +INSTANTIATE_TEST_CASE_P(MultipleLiveSources, GStreamerSourceTest, + Combine(Values("videotestsrc is-live=true pattern=colors num-buffers=10 ! " + "videoscale ! video/x-raw,width=1280,height=720 ! appsink " + "videotestsrc is-live=true pattern=colors num-buffers=10 ! " + "fakesink"), + Values(cv::Size(1280, 720)), + Values(10UL))); + +INSTANTIATE_TEST_CASE_P(MultipleNotLiveSources, GStreamerSourceTest, + Combine(Values("videotestsrc pattern=colors num-buffers=10 ! " + "videoscale ! video/x-raw,width=1280,height=720 ! appsink " + "videotestsrc pattern=colors num-buffers=10 ! " + "fakesink"), + Values(cv::Size(1280, 720)), + Values(10UL))); + + +TEST(GStreamerMultiSourceSmokeTest, Test) +{ + // Graph declaration: + cv::GMat in1, in2; + auto out = cv::gapi::add(in1, in2); + cv::GComputation c(cv::GIn(in1, in2), cv::GOut(out)); + + // Graph compilation for streaming mode: + auto ccomp = c.compileStreaming(); + + EXPECT_TRUE(ccomp); + EXPECT_FALSE(ccomp.running()); + + cv::gapi::wip::GStreamerPipeline + pipeline("videotestsrc is-live=true pattern=colors num-buffers=10 ! " + "videorate ! videoscale ! " + "video/x-raw,width=1920,height=1080,framerate=3/1 ! " + "appsink name=sink1 " + "videotestsrc is-live=true pattern=colors num-buffers=10 ! " + "videorate ! videoscale ! " + "video/x-raw,width=1920,height=1080,framerate=3/1 ! " + "appsink name=sink2"); + + // GStreamer streaming sources configuration: + auto src1 = pipeline.getStreamingSource("sink1"); + auto src2 = pipeline.getStreamingSource("sink2"); + + ccomp.setSource(cv::gin(src1, src2)); + + // Start of streaming: + ccomp.start(); + EXPECT_TRUE(ccomp.running()); + + // Streaming - pulling of frames until the end: + cv::Mat in_mat_gapi; + + EXPECT_TRUE(ccomp.pull(cv::gout(in_mat_gapi))); + EXPECT_TRUE(!in_mat_gapi.empty()); + EXPECT_EQ(CV_8UC3, in_mat_gapi.type()); + + while (ccomp.pull(cv::gout(in_mat_gapi))) { + EXPECT_TRUE(!in_mat_gapi.empty()); + EXPECT_EQ(CV_8UC3, in_mat_gapi.type()); + } + + EXPECT_FALSE(ccomp.running()); + ccomp.stop(); + + EXPECT_FALSE(ccomp.running()); +} + +struct GStreamerMultiSourceTest : + public TestWithParam> +{ }; + +TEST_P(GStreamerMultiSourceTest, ImageDataTest) +{ + std::string pathToLeftIm = findDataFile("cv/stereomatching/datasets/tsukuba/im6.png"); + std::string pathToRightIm = findDataFile("cv/stereomatching/datasets/tsukuba/im2.png"); + + std::string pipelineToReadImage("filesrc location=LOC ! pngdec ! videoconvert ! " + "videoscale ! video/x-raw,format=NV12 ! appsink"); + + cv::gapi::wip::GStreamerSource leftImageProvider( + std::regex_replace(pipelineToReadImage, std::regex("LOC"), pathToLeftIm)); + cv::gapi::wip::GStreamerSource rightImageProvider( + std::regex_replace(pipelineToReadImage, std::regex("LOC"), pathToRightIm)); + + cv::gapi::wip::Data leftImData, rightImData; + leftImageProvider.pull(leftImData); + rightImageProvider.pull(rightImData); + + cv::Mat leftRefMat = cv::util::get(leftImData); + cv::Mat rightRefMat = cv::util::get(rightImData); + + // Retrieve test parameters: + std::tuple params = GetParam(); + cv::GComputation extractImage = std::move(std::get<0>(params)); + cv::gapi::wip::GStreamerSource::OutputType outputType = std::get<1>(params); + + // Graph compilation for streaming mode: + auto compiled = + extractImage.compileStreaming(); + + EXPECT_TRUE(compiled); + EXPECT_FALSE(compiled.running()); + + cv::gapi::wip::GStreamerPipeline + pipeline(std::string("multifilesrc location=" + pathToLeftIm + " index=0 loop=true ! " + "pngdec ! videoconvert ! videoscale ! video/x-raw,format=NV12 ! " + "appsink name=sink1 ") + + std::string("multifilesrc location=" + pathToRightIm + " index=0 loop=true ! " + "pngdec ! videoconvert ! videoscale ! video/x-raw,format=NV12 ! " + "appsink name=sink2")); + + // GStreamer streaming sources configuration: + auto src1 = pipeline.getStreamingSource("sink1", outputType); + auto src2 = pipeline.getStreamingSource("sink2", outputType); + + compiled.setSource(cv::gin(src1, src2)); + + // Start of streaming: + compiled.start(); + EXPECT_TRUE(compiled.running()); + + // Streaming - pulling of frames: + cv::Mat in_mat1, in_mat2; + + std::size_t counter { }, limit { 10 }; + while(compiled.pull(cv::gout(in_mat1, in_mat2)) && (counter < limit)) { + EXPECT_EQ(0, cv::norm(in_mat1, leftRefMat, cv::NORM_INF)); + EXPECT_EQ(0, cv::norm(in_mat2, rightRefMat, cv::NORM_INF)); + ++counter; + } + + compiled.stop(); + + EXPECT_FALSE(compiled.running()); +} + +INSTANTIATE_TEST_CASE_P(GStreamerMultiSourceViaGMatsTest, GStreamerMultiSourceTest, + Combine(Values(cv::GComputation([]() + { + cv::GMat in1, in2; + return cv::GComputation(cv::GIn(in1, in2), + cv::GOut(cv::gapi::copy(in1), + cv::gapi::copy(in2))); + })), + Values(cv::gapi::wip::GStreamerSource::OutputType::MAT))); + +INSTANTIATE_TEST_CASE_P(GStreamerMultiSourceViaGFramesTest, GStreamerMultiSourceTest, + Combine(Values(cv::GComputation([]() + { + cv::GFrame in1, in2; + return cv::GComputation(cv::GIn(in1, in2), + cv::GOut(cv::gapi::streaming::BGR(in1), + cv::gapi::streaming::BGR(in2))); + })), + Values(cv::gapi::wip::GStreamerSource::OutputType::FRAME))); +} // namespace opencv_test + +#endif // HAVE_GSTREAMER From e608adea6091275317c45d20803b3d5dd76aa3c9 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Mon, 6 Dec 2021 19:33:59 +0300 Subject: [PATCH 139/226] add ArgMax and ArgMin layers --- .../dnn/include/opencv2/dnn/all_layers.hpp | 10 ++ modules/dnn/src/init.cpp | 1 + modules/dnn/src/layers/arg_layer.cpp | 120 ++++++++++++++++++ modules/dnn/src/onnx/onnx_importer.cpp | 10 ++ modules/dnn/test/test_onnx_importer.cpp | 9 ++ 5 files changed, 150 insertions(+) create mode 100644 modules/dnn/src/layers/arg_layer.cpp diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index 0bec9e35e2..45935279f5 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -284,6 +284,16 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + + /** @brief ArgMax/ArgMin layer + * @note returns indices as floats, which means the supported range is [-2^24; 2^24] + */ + class CV_EXPORTS ArgLayer : public Layer + { + public: + static Ptr create(const LayerParams& params); + }; + class CV_EXPORTS PoolingLayer : public Layer { public: diff --git a/modules/dnn/src/init.cpp b/modules/dnn/src/init.cpp index affaa1a7e1..443d1eaef4 100644 --- a/modules/dnn/src/init.cpp +++ b/modules/dnn/src/init.cpp @@ -123,6 +123,7 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(Identity, BlankLayer); CV_DNN_REGISTER_LAYER_CLASS(Silence, BlankLayer); CV_DNN_REGISTER_LAYER_CLASS(Const, ConstLayer); + CV_DNN_REGISTER_LAYER_CLASS(Arg, ArgLayer); CV_DNN_REGISTER_LAYER_CLASS(Crop, CropLayer); CV_DNN_REGISTER_LAYER_CLASS(Eltwise, EltwiseLayer); diff --git a/modules/dnn/src/layers/arg_layer.cpp b/modules/dnn/src/layers/arg_layer.cpp new file mode 100644 index 0000000000..94af45882a --- /dev/null +++ b/modules/dnn/src/layers/arg_layer.cpp @@ -0,0 +1,120 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" + + +namespace cv { namespace dnn { + +class ArgLayerImpl CV_FINAL : public ArgLayer +{ +public: + enum class ArgOp + { + MIN = 0, + MAX = 1, + }; + + ArgLayerImpl(const LayerParams& params) + { + setParamsFrom(params); + + axis = params.get("axis", 0); + keepdims = (params.get("keepdims", 1) == 1); + select_last_index = (params.get("select_last_index", 0) == 1); + + const std::string& argOp = params.get("op"); + + if (argOp == "max") + { + op = ArgOp::MAX; + } + else if (argOp == "min") + { + op = ArgOp::MIN; + } + else + { + CV_Error(Error::StsBadArg, "Unsupported operation"); + } + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV && preferableTarget == DNN_TARGET_CPU; + } + + void handleKeepDims(MatShape& shape, const int axis_) const + { + if (keepdims) + { + shape[axis_] = 1; + } + else + { + shape.erase(shape.begin() + axis_); + } + } + + virtual bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + MatShape inpShape = inputs[0]; + + const int axis_ = normalize_axis(axis, inpShape); + handleKeepDims(inpShape, axis_); + outputs.assign(1, inpShape); + + return false; + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + CV_Assert_N(inputs.size() == 1, outputs.size() == 1); + std::vector outShape = shape(outputs[0]); + Mat output(outShape, CV_32SC1); + + switch (op) + { + case ArgOp::MIN: + cv::reduceArgMin(inputs[0], output, axis, select_last_index); + break; + case ArgOp::MAX: + cv::reduceArgMax(inputs[0], output, axis, select_last_index); + break; + default: + CV_Error(Error::StsBadArg, "Unsupported operation."); + } + + output = output.reshape(1, outShape); + output.convertTo(outputs[0], CV_32FC1); + } + +private: + // The axis in which to compute the arg indices. Accepted range is [-r, r-1] where r = rank(data). + int axis; + // Keep the reduced dimension or not + bool keepdims; + // Whether to select the first or the last index or Max/Min. + bool select_last_index; + // Operation to be performed + ArgOp op; +}; + +Ptr ArgLayer::create(const LayerParams& params) +{ + return Ptr(new ArgLayerImpl(params)); +} + +}} // namespace cv::dnn diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index b0d7d4b913..85c4479c6f 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -100,6 +100,7 @@ private: const DispatchMap dispatch; static const DispatchMap buildDispatchMap(); + void parseArg (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseMaxPool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseAveragePool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseReduce (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); @@ -768,6 +769,14 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto) } } +void ONNXImporter::parseArg(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + const std::string& layer_type = node_proto.op_type(); + layerParams.type = "Arg"; + layerParams.set("op", layer_type == "ArgMax" ? "max" : "min"); + addLayer(layerParams, node_proto); +} + void setCeilMode(LayerParams& layerParams) { // auto_pad attribute is deprecated and uses ceil @@ -2986,6 +2995,7 @@ const ONNXImporter::DispatchMap ONNXImporter::buildDispatchMap() { DispatchMap dispatch; + dispatch["ArgMax"] = dispatch["ArgMin"] = &ONNXImporter::parseArg; dispatch["MaxPool"] = &ONNXImporter::parseMaxPool; dispatch["AveragePool"] = &ONNXImporter::parseAveragePool; dispatch["GlobalAveragePool"] = dispatch["GlobalMaxPool"] = dispatch["ReduceMean"] = dispatch["ReduceSum"] = diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index f2deaf1a4e..bfea8550c0 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -355,6 +355,15 @@ TEST_P(Test_ONNX_layers, Min) testONNXModels("min", npy, 0, 0, false, true, 2); } +TEST_P(Test_ONNX_layers, ArgLayer) +{ + if (backend != DNN_BACKEND_OPENCV || target != DNN_TARGET_CPU) + throw SkipTestException("Only CPU is supported"); // FIXIT use tags + + testONNXModels("argmax"); + testONNXModels("argmin"); +} + TEST_P(Test_ONNX_layers, Scale) { if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) From 51d7ba7446b8819c10e7151ba287a4e8d91b7ecc Mon Sep 17 00:00:00 2001 From: OrestChura Date: Tue, 7 Dec 2021 04:18:36 +0300 Subject: [PATCH 140/226] Bring updates from openvino version of standalone files --- modules/gapi/doc/10-hld-overview.md | 2 +- .../gapi/include/opencv2/gapi/own/cvdefs.hpp | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/modules/gapi/doc/10-hld-overview.md b/modules/gapi/doc/10-hld-overview.md index 557bf08b12..6de6efa921 100644 --- a/modules/gapi/doc/10-hld-overview.md +++ b/modules/gapi/doc/10-hld-overview.md @@ -142,7 +142,7 @@ Graph execution is triggered in two ways: Both methods are polimorphic and take a variadic number of arguments, with validity checks performed in runtime. If a number, shapes, and -formats of passed data objects differ from expected, a run-time +formats of passed data objects differ from expected, a runtime exception is thrown. G-API also provides _typed_ wrappers to move these checks to the compile time -- see `cv::GComputationT<>`. diff --git a/modules/gapi/include/opencv2/gapi/own/cvdefs.hpp b/modules/gapi/include/opencv2/gapi/own/cvdefs.hpp index b0c91d3530..d3bef98e98 100644 --- a/modules/gapi/include/opencv2/gapi/own/cvdefs.hpp +++ b/modules/gapi/include/opencv2/gapi/own/cvdefs.hpp @@ -20,11 +20,12 @@ typedef char schar; typedef unsigned short ushort; +#define CV_USRTYPE1 (void)"CV_USRTYPE1 support has been dropped in OpenCV 4.0" + #define CV_CN_MAX 512 #define CV_CN_SHIFT 3 #define CV_DEPTH_MAX (1 << CV_CN_SHIFT) - #define CV_8U 0 #define CV_8S 1 #define CV_16U 2 @@ -32,7 +33,7 @@ typedef unsigned short ushort; #define CV_32S 4 #define CV_32F 5 #define CV_64F 6 -#define CV_USRTYPE1 7 +#define CV_16F 7 #define CV_MAT_DEPTH_MASK (CV_DEPTH_MAX - 1) #define CV_MAT_DEPTH(flags) ((flags) & CV_MAT_DEPTH_MASK) @@ -70,6 +71,12 @@ typedef unsigned short ushort; #define CV_32SC4 CV_MAKETYPE(CV_32S,4) #define CV_32SC(n) CV_MAKETYPE(CV_32S,(n)) +#define CV_16FC1 CV_MAKETYPE(CV_16F,1) +#define CV_16FC2 CV_MAKETYPE(CV_16F,2) +#define CV_16FC3 CV_MAKETYPE(CV_16F,3) +#define CV_16FC4 CV_MAKETYPE(CV_16F,4) +#define CV_16FC(n) CV_MAKETYPE(CV_16F,(n)) + #define CV_32FC1 CV_MAKETYPE(CV_32F,1) #define CV_32FC2 CV_MAKETYPE(CV_32F,2) #define CV_32FC3 CV_MAKETYPE(CV_32F,3) @@ -106,10 +113,10 @@ typedef unsigned short ushort; #define CV_SUBMAT_FLAG (1 << CV_SUBMAT_FLAG_SHIFT) #define CV_IS_SUBMAT(flags) ((flags) & CV_MAT_SUBMAT_FLAG) -///** Size of each channel item, +//** Size of each channel item, // 0x8442211 = 1000 0100 0100 0010 0010 0001 0001 ~ array of sizeof(arr_type_elem) */ -//#define CV_ELEM_SIZE1(type) \ -// ((((sizeof(size_t)<<28)|0x8442211) >> CV_MAT_DEPTH(type)*4) & 15) +#define CV_ELEM_SIZE1(type) \ + ((((sizeof(size_t)<<28)|0x8442211) >> CV_MAT_DEPTH(type)*4) & 15) #define CV_MAT_TYPE(flags) ((flags) & CV_MAT_TYPE_MASK) From 3a4b61579d892f410915fe4f583bceeef2714f81 Mon Sep 17 00:00:00 2001 From: Anna Khakimova Date: Mon, 6 Dec 2021 14:56:20 +0300 Subject: [PATCH 141/226] GAPI GPU: fix for tests failure. --- modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp b/modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp index 025ea5331d..fa63f1e208 100644 --- a/modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp +++ b/modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp @@ -54,14 +54,14 @@ INSTANTIATE_TEST_CASE_P(MulPerfTestGPU, MulPerfTest, Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(MulDoublePerfTestGPU, MulDoublePerfTest, - Combine(Values(AbsExact().to_compare_f()), + Combine(Values(Tolerance_FloatRel_IntAbs(1e-5, 1).to_compare_f()), Values( szSmall128, szVGA, sz720p, sz1080p ), Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), Values( -1, CV_8U, CV_16U, CV_32F ), Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(MulCPerfTestGPU, MulCPerfTest, - Combine(Values(AbsExact().to_compare_f()), + Combine(Values(Tolerance_FloatRel_IntAbs(1e-5, 1).to_compare_f()), Values( szSmall128, szVGA, sz720p, sz1080p ), Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), Values( -1, CV_8U, CV_16U, CV_32F ), From 65392d5e6b0957b6e08678ff54f106a08e1e6bd3 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 7 Dec 2021 05:47:54 +0000 Subject: [PATCH 142/226] cmake: fix OPENGL_LIBRARIES handling --- cmake/OpenCVFindLibsGUI.cmake | 1 - modules/core/CMakeLists.txt | 1 + samples/cpp/CMakeLists.txt | 1 + samples/opengl/CMakeLists.txt | 2 +- 4 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cmake/OpenCVFindLibsGUI.cmake b/cmake/OpenCVFindLibsGUI.cmake index 5e7db6c199..1d7e86722a 100644 --- a/cmake/OpenCVFindLibsGUI.cmake +++ b/cmake/OpenCVFindLibsGUI.cmake @@ -101,7 +101,6 @@ if(WITH_OPENGL) find_package (OpenGL QUIET) if(OPENGL_FOUND) set(HAVE_OPENGL TRUE) - list(APPEND OPENCV_LINKER_LIBS ${OPENGL_LIBRARIES}) if(QT_QTOPENGL_FOUND) set(HAVE_QT_OPENGL TRUE) else() diff --git a/modules/core/CMakeLists.txt b/modules/core/CMakeLists.txt index a84d7fc3ad..beba9f804e 100644 --- a/modules/core/CMakeLists.txt +++ b/modules/core/CMakeLists.txt @@ -107,6 +107,7 @@ ocv_create_module(${extra_libs}) ocv_target_link_libraries(${the_module} PRIVATE "${ZLIB_LIBRARIES}" "${OPENCL_LIBRARIES}" "${VA_LIBRARIES}" + "${OPENGL_LIBRARIES}" "${LAPACK_LIBRARIES}" "${CPUFEATURES_LIBRARIES}" "${HALIDE_LIBRARIES}" "${ITT_LIBRARIES}" "${OPENCV_HAL_LINKER_LIBS}" diff --git a/samples/cpp/CMakeLists.txt b/samples/cpp/CMakeLists.txt index db16785dcc..6ae9586fd4 100644 --- a/samples/cpp/CMakeLists.txt +++ b/samples/cpp/CMakeLists.txt @@ -56,6 +56,7 @@ foreach(sample_filename ${cpp_samples}) endif() if(HAVE_OPENGL AND sample_filename MATCHES "detect_mser") target_compile_definitions(${tgt} PRIVATE HAVE_OPENGL) + ocv_target_link_libraries(${tgt} PRIVATE "${OPENGL_LIBRARIES}") endif() if(sample_filename MATCHES "simd_") # disabled intentionally - demonstration purposes only diff --git a/samples/opengl/CMakeLists.txt b/samples/opengl/CMakeLists.txt index 1e5d68dfe5..158151c300 100644 --- a/samples/opengl/CMakeLists.txt +++ b/samples/opengl/CMakeLists.txt @@ -23,7 +23,7 @@ if(BUILD_EXAMPLES AND OCV_DEPENDENCIES_FOUND) endif() foreach(sample_filename ${all_samples}) ocv_define_sample(tgt ${sample_filename} opengl) - ocv_target_link_libraries(${tgt} PRIVATE ${OPENCV_LINKER_LIBS} ${OPENCV_OPENGL_SAMPLES_REQUIRED_DEPS}) + ocv_target_link_libraries(${tgt} PRIVATE "${OPENGL_LIBRARIES}" "${OPENCV_OPENGL_SAMPLES_REQUIRED_DEPS}") if(sample_filename STREQUAL "opengl_interop.cpp") ocv_target_link_libraries(${tgt} PRIVATE ${X11_LIBRARIES}) ocv_target_include_directories(${tgt} ${X11_INCLUDE_DIR}) From 5c91f5b71dc22613764d37712cbd119bf06774fc Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Wed, 8 Dec 2021 10:09:33 +0300 Subject: [PATCH 143/226] Merge pull request #21049 from sivanov-work:vpl_dx11_merge G-API: oneVPL merge DX11 acceleration * Merge DX11 initial * Fold conditions row in MACRO in utils * Inject DeviceSelector * Turn on DeviceSelector in DX11 * Change sharedLock logic & Move FMT checking in FrameAdapter c-tor * Move out NumSuggestFrame to configure params * Drain file source fix * Fix compilation * Force zero initializetion of SharedLock * Fix some compiler warnings * Fix integer comparison warnings * Fix integers in sample * Integrate Demux * Fix compilation * Add predefined names for some CfgParam * Trigger CI * Fix MultithreadCtx bug, Add Dx11 GetBlobParam(), Get rif of ATL CComPtr * Fix UT: remove unit test with deprecated video from opencv_extra * Add creators for most usable CfgParam * Eliminate some warnings * Fix warning in GAPI_Assert * Apply comments * Add VPL wrapped header with MSVC pragma to get rid of global warning masking --- modules/gapi/CMakeLists.txt | 7 + .../gapi/streaming/onevpl/cfg_params.hpp | 65 ++- .../gapi_streaming_source_perf_tests.cpp | 30 +- .../gapi/samples/onevpl_infer_single_roi.cpp | 26 +- .../gapi/src/backends/render/grenderocv.cpp | 38 +- .../onevpl/accelerators/accel_policy_cpu.cpp | 125 ++++- .../onevpl/accelerators/accel_policy_cpu.hpp | 6 +- .../onevpl/accelerators/accel_policy_dx11.cpp | 364 ++++++++++++-- .../onevpl/accelerators/accel_policy_dx11.hpp | 46 +- .../accelerators/accel_policy_interface.hpp | 19 +- .../accelerators/dx11_alloc_resource.cpp | 404 +++++++++++++++ .../accelerators/dx11_alloc_resource.hpp | 151 ++++++ .../surface/cpu_frame_adapter.cpp | 41 +- .../surface/cpu_frame_adapter.hpp | 1 + .../surface/dx11_frame_adapter.cpp | 232 +++++++++ .../surface/dx11_frame_adapter.hpp | 63 +++ .../onevpl/accelerators/surface/surface.cpp | 16 +- .../onevpl/accelerators/surface/surface.hpp | 40 +- .../accelerators/surface/surface_pool.hpp | 6 +- .../accelerators/utils/elastic_barrier.hpp | 296 +++++++++++ .../onevpl/accelerators/utils/shared_lock.cpp | 95 ++++ .../onevpl/accelerators/utils/shared_lock.hpp | 47 ++ .../onevpl/cfg_param_device_selector.cpp | 44 +- .../gapi/src/streaming/onevpl/cfg_params.cpp | 28 ++ .../streaming/onevpl/cfg_params_parser.cpp | 55 ++- .../streaming/onevpl/cfg_params_parser.hpp | 13 +- .../onevpl/data_provider_defines.hpp | 3 +- .../onevpl/data_provider_dispatcher.cpp | 7 +- .../data_provider_interface_exception.cpp | 3 +- .../demux/async_mfp_demux_data_provider.hpp | 2 +- .../engine/decode/decode_engine_legacy.cpp | 310 ++++++------ .../engine/decode/decode_engine_legacy.hpp | 10 +- .../onevpl/engine/decode/decode_session.cpp | 10 +- .../onevpl/engine/decode/decode_session.hpp | 11 +- .../onevpl/engine/engine_session.hpp | 4 +- .../onevpl/engine/processing_engine_base.cpp | 6 +- .../onevpl/engine/processing_engine_base.hpp | 7 +- .../streaming/onevpl/file_data_provider.cpp | 6 +- .../src/streaming/onevpl/onevpl_export.hpp | 25 + .../gapi/src/streaming/onevpl/source_priv.cpp | 122 ++--- .../gapi/src/streaming/onevpl/source_priv.hpp | 15 +- modules/gapi/src/streaming/onevpl/utils.cpp | 466 +++++++----------- modules/gapi/src/streaming/onevpl/utils.hpp | 16 +- .../common/gapi_streaming_tests_common.hpp | 86 ++++ .../test/streaming/gapi_streaming_tests.cpp | 97 +--- .../streaming/gapi_streaming_utils_test.cpp | 348 +++++++++++++ .../gapi_streaming_vpl_core_test.cpp | 262 +++++++++- .../gapi_streaming_vpl_data_provider.cpp | 6 +- .../gapi_streaming_vpl_device_selector.cpp | 17 +- 49 files changed, 3231 insertions(+), 866 deletions(-) create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/dx11_alloc_resource.cpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/dx11_alloc_resource.hpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/surface/dx11_frame_adapter.cpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/surface/dx11_frame_adapter.hpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/utils/elastic_barrier.hpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/utils/shared_lock.cpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/utils/shared_lock.hpp create mode 100644 modules/gapi/src/streaming/onevpl/onevpl_export.hpp create mode 100644 modules/gapi/test/common/gapi_streaming_tests_common.hpp create mode 100644 modules/gapi/test/streaming/gapi_streaming_utils_test.cpp diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 855ce27088..cc83606694 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -174,10 +174,13 @@ set(gapi_srcs src/streaming/onevpl/utils.cpp src/streaming/onevpl/data_provider_interface_exception.cpp src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp + src/streaming/onevpl/accelerators/surface/dx11_frame_adapter.cpp src/streaming/onevpl/accelerators/surface/surface.cpp src/streaming/onevpl/accelerators/surface/surface_pool.cpp + src/streaming/onevpl/accelerators/utils/shared_lock.cpp src/streaming/onevpl/accelerators/accel_policy_cpu.cpp src/streaming/onevpl/accelerators/accel_policy_dx11.cpp + src/streaming/onevpl/accelerators/dx11_alloc_resource.cpp src/streaming/onevpl/engine/engine_session.cpp src/streaming/onevpl/engine/processing_engine_base.cpp src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp @@ -282,12 +285,16 @@ if(HAVE_GAPI_ONEVPL) if(TARGET opencv_test_gapi) ocv_target_compile_definitions(opencv_test_gapi PRIVATE -DHAVE_ONEVPL) ocv_target_link_libraries(opencv_test_gapi PRIVATE ${VPL_IMPORTED_TARGETS}) + if(MSVC) + target_compile_options(opencv_test_gapi PUBLIC "/wd4201") + endif() if(HAVE_D3D11 AND HAVE_OPENCL) ocv_target_include_directories(opencv_test_gapi SYSTEM PRIVATE ${OPENCL_INCLUDE_DIRS}) endif() endif() ocv_target_compile_definitions(${the_module} PRIVATE -DHAVE_ONEVPL) ocv_target_link_libraries(${the_module} PRIVATE ${VPL_IMPORTED_TARGETS}) + if(HAVE_D3D11 AND HAVE_OPENCL) ocv_target_include_directories(${the_module} SYSTEM PRIVATE ${OPENCL_INCLUDE_DIRS}) endif() diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/cfg_params.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/cfg_params.hpp index 9dc5ead7d7..0f35200f49 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/onevpl/cfg_params.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/cfg_params.hpp @@ -46,9 +46,72 @@ struct GAPI_EXPORTS CfgParam { double_t, void*, std::string>; + /** + * @brief frames_pool_size_name + * + * Special configuration parameter name for onevp::GSource: + * + * @note frames_pool_size_name allows to allocate surfaces pool appropriate size to keep + * decoded frames in accelerator memory ready before + * they would be consumed by onevp::GSource::pull operation. If you see + * a lot of WARNING about lack of free surface then it's time to increase + * frames_pool_size_name but be aware of accelerator free memory volume. + * If not set then MFX implementation use + * mfxFrameAllocRequest::NumFrameSuggested behavior + * + */ + static constexpr const char *frames_pool_size_name() { return "frames_pool_size"; } + static CfgParam create_frames_pool_size(uint64_t value); /** - * Create onevp::GSource configuration parameter. + * @brief acceleration_mode_name + * + * Special configuration parameter names for onevp::GSource: + * + * @note acceleration_mode_name allows to activate hardware acceleration & + * device memory management. + * Supported values: + * - MFX_ACCEL_MODE_VIA_D3D11 Will activate DX11 acceleration and will produces + * MediaFrames with data allocated in DX11 device memory + * + * If not set then MFX implementation will use default acceleration behavior: + * all decoding operation uses default GPU resources but MediaFrame produces + * data allocated by using host RAM + * + */ + static constexpr const char *acceleration_mode_name() { return "mfxImplDescription.AccelerationMode"; } + static CfgParam create_acceleration_mode(uint32_t value); + static CfgParam create_acceleration_mode(const char* value); + + /** + * @brief decoder_id_name + * + * Special configuration parameter names for onevp::GSource: + * + * @note decoder_id_name allows to specify VPL decoder type which MUST present + * in case of RAW video input data and MUST NOT present as CfgParam if video + * stream incapsulated into container(*.mp4, *.mkv and so on). In latter case + * onevp::GSource will determine it automatically + * Supported values: + * - MFX_CODEC_AVC + * - MFX_CODEC_HEVC + * - MFX_CODEC_MPEG2 + * - MFX_CODEC_VC1 + * - MFX_CODEC_CAPTURE + * - MFX_CODEC_VP9 + * - MFX_CODEC_AV1 + * + */ + static constexpr const char *decoder_id_name() { return "mfxImplDescription.mfxDecoderDescription.decoder.CodecID"; } + static CfgParam create_decoder_id(uint32_t value); + static CfgParam create_decoder_id(const char* value); + + static constexpr const char *implementation_name() { return "mfxImplDescription.Impl"; } + static CfgParam create_implementation(uint32_t value); + static CfgParam create_implementation(const char* value); + + /** + * Create generic onevp::GSource configuration parameter. * *@param name name of parameter. *@param value value of parameter. diff --git a/modules/gapi/perf/streaming/gapi_streaming_source_perf_tests.cpp b/modules/gapi/perf/streaming/gapi_streaming_source_perf_tests.cpp index ffefa24259..7d06ad068b 100644 --- a/modules/gapi/perf/streaming/gapi_streaming_source_perf_tests.cpp +++ b/modules/gapi/perf/streaming/gapi_streaming_source_perf_tests.cpp @@ -18,16 +18,19 @@ using namespace perf; const std::string files[] = { "highgui/video/big_buck_bunny.h265", "highgui/video/big_buck_bunny.h264", + "highgui/video/sample_322x242_15frames.yuv420p.libx265.mp4", }; const std::string codec[] = { "MFX_CODEC_HEVC", - "MFX_CODEC_AVC" + "MFX_CODEC_AVC", + "", }; using source_t = std::string; using codec_t = std::string; -using source_description_t = std::tuple; +using accel_mode_t = std::string; +using source_description_t = std::tuple; class OneVPLSourcePerfTest : public TestPerfParams {}; class VideoCapSourcePerfTest : public TestPerfParams {}; @@ -39,12 +42,20 @@ PERF_TEST_P_(OneVPLSourcePerfTest, TestPerformance) const auto params = GetParam(); source_t src = findDataFile(get<0>(params)); codec_t type = get<1>(params); + accel_mode_t mode = get<2>(params); std::vector cfg_params { - CfgParam::create("mfxImplDescription.Impl", "MFX_IMPL_TYPE_HARDWARE"), - CfgParam::create("mfxImplDescription.mfxDecoderDescription.decoder.CodecID", type), + CfgParam::create_implementation("MFX_IMPL_TYPE_HARDWARE"), }; + if (!type.empty()) { + cfg_params.push_back(CfgParam::create_decoder_id(type.c_str())); + } + + if (!mode.empty()) { + cfg_params.push_back(CfgParam::create_acceleration_mode(mode.c_str())); + } + auto source_ptr = cv::gapi::wip::make_onevpl_src(src, cfg_params); cv::gapi::wip::Data out; @@ -72,12 +83,17 @@ PERF_TEST_P_(VideoCapSourcePerfTest, TestPerformance) } INSTANTIATE_TEST_CASE_P(Streaming, OneVPLSourcePerfTest, - Values(source_description_t(files[0], codec[0]), - source_description_t(files[1], codec[1]))); + Values(source_description_t(files[0], codec[0], ""), + source_description_t(files[0], codec[0], "MFX_ACCEL_MODE_VIA_D3D11"), + source_description_t(files[1], codec[1], ""), + source_description_t(files[1], codec[1], "MFX_ACCEL_MODE_VIA_D3D11"), + source_description_t(files[2], codec[2], ""), + source_description_t(files[2], codec[2], "MFX_ACCEL_MODE_VIA_D3D11"))); INSTANTIATE_TEST_CASE_P(Streaming, VideoCapSourcePerfTest, Values(files[0], - files[1])); + files[1], + files[2])); } // namespace opencv_test #endif // HAVE_ONEVPL diff --git a/modules/gapi/samples/onevpl_infer_single_roi.cpp b/modules/gapi/samples/onevpl_infer_single_roi.cpp index fec0f0d043..1ab06de739 100644 --- a/modules/gapi/samples/onevpl_infer_single_roi.cpp +++ b/modules/gapi/samples/onevpl_infer_single_roi.cpp @@ -38,12 +38,14 @@ const std::string about = "This is an OpenCV-based version of oneVPLSource decoder example"; const std::string keys = - "{ h help | | Print this help message }" - "{ input | | Path to the input demultiplexed video file }" - "{ output | | Path to the output RAW video file. Use .avi extension }" - "{ facem | face-detection-adas-0001.xml | Path to OpenVINO IE face detection model (.xml) }" - "{ faced | CPU | Target device for face detection model (e.g. CPU, GPU, VPU, ...) }" - "{ cfg_params | :;: | Semicolon separated list of oneVPL mfxVariants which is used for configuring source (see `MFXSetConfigFilterProperty` by https://spec.oneapi.io/versions/latest/elements/oneVPL/source/index.html) }"; + "{ h help | | Print this help message }" + "{ input | | Path to the input demultiplexed video file }" + "{ output | | Path to the output RAW video file. Use .avi extension }" + "{ facem | face-detection-adas-0001.xml | Path to OpenVINO IE face detection model (.xml) }" + "{ faced | AUTO | Target device for face detection model (e.g. AUTO, GPU, VPU, ...) }" + "{ cfg_params | :;: | Semicolon separated list of oneVPL mfxVariants which is used for configuring source (see `MFXSetConfigFilterProperty` by https://spec.oneapi.io/versions/latest/elements/oneVPL/source/index.html) }" + "{ streaming_queue_capacity | 1 | Streaming executor queue capacity. Calculated automaticaly if 0 }" + "{ frames_pool_size | 0 | OneVPL source applies this parameter as preallocated frames pool size}"; namespace { @@ -194,6 +196,8 @@ int main(int argc, char *argv[]) { std::string file_path = cmd.get("input"); const std::string output = cmd.get("output"); const auto face_model_path = cmd.get("facem"); + const auto streaming_queue_capacity = cmd.get("streaming_queue_capacity"); + const auto source_queue_capacity = cmd.get("frames_pool_size"); // check ouput file extension if (!output.empty()) { @@ -217,6 +221,10 @@ int main(int argc, char *argv[]) { return -1; } + if (source_queue_capacity != 0) { + source_cfgs.push_back(cv::gapi::wip::onevpl::CfgParam::create_frames_pool_size(source_queue_capacity)); + } + const std::string& device_id = cmd.get("faced"); auto face_net = cv::gapi::ie::Params { face_model_path, // path to topology IR @@ -298,6 +306,10 @@ int main(int argc, char *argv[]) { < custom::OCVLocateROI , custom::OCVBBoxes>(); auto networks = cv::gapi::networks(face_net); + auto face_detection_args = cv::compile_args(networks, kernels); + if (streaming_queue_capacity != 0) { + face_detection_args += cv::compile_args(cv::gapi::streaming::queue_capacity{ streaming_queue_capacity }); + } // Create source cv::Ptr cap; @@ -331,7 +343,7 @@ int main(int argc, char *argv[]) { cv::GStreamingCompiled pipeline; try { pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)) - .compileStreaming(cv::compile_args(kernels, networks)); + .compileStreaming(std::move(face_detection_args)); } catch (const std::exception& ex) { std::cerr << "Exception occured during pipeline construction: " << ex.what() << std::endl; return -1; diff --git a/modules/gapi/src/backends/render/grenderocv.cpp b/modules/gapi/src/backends/render/grenderocv.cpp index 2652284668..da0e5831a1 100644 --- a/modules/gapi/src/backends/render/grenderocv.cpp +++ b/modules/gapi/src/backends/render/grenderocv.cpp @@ -128,15 +128,12 @@ GAPI_OCV_KERNEL_ST(RenderFrameOCVImpl, cv::gapi::wip::draw::GRenderFrame, Render out = in; auto desc = out.desc(); - auto w_out = out.access(cv::MediaFrame::Access::W); + cv::Mat upsample_uv, yuv; + { + auto r_in = in.access(cv::MediaFrame::Access::R); - auto out_y = cv::Mat(desc.size, CV_8UC1, w_out.ptr[0], w_out.stride[0]); - auto out_uv = cv::Mat(desc.size / 2, CV_8UC2, w_out.ptr[1], w_out.stride[1]); - - auto r_in = in.access(cv::MediaFrame::Access::R); - - auto in_y = cv::Mat(desc.size, CV_8UC1, r_in.ptr[0], r_in.stride[0]); - auto in_uv = cv::Mat(desc.size / 2, CV_8UC2, r_in.ptr[1], r_in.stride[1]); + auto in_y = cv::Mat(desc.size, CV_8UC1, r_in.ptr[0], r_in.stride[0]); + auto in_uv = cv::Mat(desc.size / 2, CV_8UC2, r_in.ptr[1], r_in.stride[1]); /* FIXME How to render correctly on NV12 format ? * @@ -157,19 +154,26 @@ GAPI_OCV_KERNEL_ST(RenderFrameOCVImpl, cv::gapi::wip::draw::GRenderFrame, Render * */ - // NV12 -> YUV - cv::Mat upsample_uv, yuv; - cv::resize(in_uv, upsample_uv, in_uv.size() * 2, cv::INTER_LINEAR); - cv::merge(std::vector{in_y, upsample_uv}, yuv); + // NV12 -> YUV + cv::resize(in_uv, upsample_uv, in_uv.size() * 2, cv::INTER_LINEAR); + cv::merge(std::vector{in_y, upsample_uv}, yuv); + } cv::gapi::wip::draw::drawPrimitivesOCVYUV(yuv, prims, state.ftpr); // YUV -> NV12 - cv::Mat out_u, out_v, uv_plane; - std::vector chs = { out_y, out_u, out_v }; - cv::split(yuv, chs); - cv::merge(std::vector{chs[1], chs[2]}, uv_plane); - cv::resize(uv_plane, out_uv, uv_plane.size() / 2, cv::INTER_LINEAR); + { + auto w_out = out.access(cv::MediaFrame::Access::W); + + auto out_y = cv::Mat(desc.size, CV_8UC1, w_out.ptr[0], w_out.stride[0]); + auto out_uv = cv::Mat(desc.size / 2, CV_8UC2, w_out.ptr[1], w_out.stride[1]); + + cv::Mat out_u, out_v, uv_plane; + std::vector chs = { out_y, out_u, out_v }; + cv::split(yuv, chs); + cv::merge(std::vector{chs[1], chs[2]}, uv_plane); + cv::resize(uv_plane, out_uv, uv_plane.size() / 2, cv::INTER_LINEAR); + } } static void setup(const cv::GFrameDesc& /* in_nv12 */, diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp index 033fb9eb15..2cdf1c2b44 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp @@ -21,8 +21,100 @@ namespace cv { namespace gapi { namespace wip { namespace onevpl { +namespace utils { +mfxU32 GetSurfaceSize_(mfxU32 FourCC, mfxU32 width, mfxU32 height) { + mfxU32 nbytes = 0; -VPLCPUAccelerationPolicy::VPLCPUAccelerationPolicy() { + mfxU32 half_width = width / 2; + mfxU32 half_height = height / 2; + switch (FourCC) { + case MFX_FOURCC_I420: + case MFX_FOURCC_NV12: + nbytes = width * height + 2 * half_width * half_height; + break; + case MFX_FOURCC_I010: + case MFX_FOURCC_P010: + nbytes = width * height + 2 * half_width * half_height; + nbytes *= 2; + break; + case MFX_FOURCC_RGB4: + nbytes = width * height * 4; + break; + default: + break; + } + + return nbytes; +} + +surface_ptr_t create_surface_RGB4_(mfxFrameInfo frameInfo, + std::shared_ptr out_buf_ptr, + size_t out_buf_ptr_offset, + size_t out_buf_size) +{ + mfxU8* buf = reinterpret_cast(out_buf_ptr.get()); + mfxU16 surfW = frameInfo.Width * 4; + mfxU16 surfH = frameInfo.Height; + (void)surfH; + + // TODO more intelligent check + if (out_buf_size <= out_buf_ptr_offset) { + GAPI_LOG_WARNING(nullptr, "Not enough buffer, ptr: " << out_buf_ptr << + ", size: " << out_buf_size << + ", offset: " << out_buf_ptr_offset << + ", W: " << surfW << + ", H: " << surfH); + GAPI_Assert(false && "Invalid offset"); + } + + std::unique_ptr handle(new mfxFrameSurface1); + memset(handle.get(), 0, sizeof(mfxFrameSurface1)); + + handle->Info = frameInfo; + handle->Data.B = buf + out_buf_ptr_offset; + handle->Data.G = handle->Data.B + 1; + handle->Data.R = handle->Data.B + 2; + handle->Data.A = handle->Data.B + 3; + handle->Data.Pitch = surfW; + + return Surface::create_surface(std::move(handle), out_buf_ptr); +} + +surface_ptr_t create_surface_other_(mfxFrameInfo frameInfo, + std::shared_ptr out_buf_ptr, + size_t out_buf_ptr_offset, + size_t out_buf_size) +{ + mfxU8* buf = reinterpret_cast(out_buf_ptr.get()); + mfxU16 surfH = frameInfo.Height; + mfxU16 surfW = (frameInfo.FourCC == MFX_FOURCC_P010) ? frameInfo.Width * 2 : frameInfo.Width; + + // TODO more intelligent check + if (out_buf_size <= + out_buf_ptr_offset + (surfW * surfH) + ((surfW / 2) * (surfH / 2))) { + GAPI_LOG_WARNING(nullptr, "Not enough buffer, ptr: " << out_buf_ptr << + ", size: " << out_buf_size << + ", offset: " << out_buf_ptr_offset << + ", W: " << surfW << + ", H: " << surfH); + GAPI_Assert(false && "Invalid offset"); + } + + std::unique_ptr handle(new mfxFrameSurface1); + memset(handle.get(), 0, sizeof(mfxFrameSurface1)); + + handle->Info = frameInfo; + handle->Data.Y = buf + out_buf_ptr_offset; + handle->Data.U = buf + out_buf_ptr_offset + (surfW * surfH); + handle->Data.V = handle->Data.U + ((surfW / 2) * (surfH / 2)); + handle->Data.Pitch = surfW; + + return Surface::create_surface(std::move(handle), out_buf_ptr); +} +} // namespace utils + +VPLCPUAccelerationPolicy::VPLCPUAccelerationPolicy(device_selector_ptr_t selector) : + VPLAccelerationPolicy(selector) { GAPI_LOG_INFO(nullptr, "created"); } @@ -117,6 +209,37 @@ VPLCPUAccelerationPolicy::create_surface_pool(size_t pool_size, size_t surface_s return preallocated_pool_memory_ptr; } +VPLCPUAccelerationPolicy::pool_key_t +VPLCPUAccelerationPolicy::create_surface_pool(const mfxFrameAllocRequest& alloc_request, mfxVideoParam& param) { + + // External (application) allocation of decode surfaces + GAPI_LOG_DEBUG(nullptr, "Query mfxFrameAllocRequest.NumFrameSuggested: " << alloc_request.NumFrameSuggested << + ", mfxFrameAllocRequest.Type: " << alloc_request.Type); + + mfxU32 singleSurfaceSize = utils::GetSurfaceSize_(param.mfx.FrameInfo.FourCC, + param.mfx.FrameInfo.Width, + param.mfx.FrameInfo.Height); + if (!singleSurfaceSize) { + throw std::runtime_error("Cannot determine surface size for: fourCC: " + + std::to_string(param.mfx.FrameInfo.FourCC) + + ", width: " + std::to_string(param.mfx.FrameInfo.Width) + + ", height: " + std::to_string(param.mfx.FrameInfo.Height)); + } + + const auto &frameInfo = param.mfx.FrameInfo; + auto surface_creator = + [&frameInfo] (std::shared_ptr out_buf_ptr, size_t out_buf_ptr_offset, + size_t out_buf_size) -> surface_ptr_t { + return (frameInfo.FourCC == MFX_FOURCC_RGB4) ? + utils::create_surface_RGB4_(frameInfo, out_buf_ptr, out_buf_ptr_offset, + out_buf_size) : + utils::create_surface_other_(frameInfo, out_buf_ptr, out_buf_ptr_offset, + out_buf_size);}; + + return create_surface_pool(alloc_request.NumFrameSuggested, + singleSurfaceSize, surface_creator); +} + VPLCPUAccelerationPolicy::surface_weak_ptr_t VPLCPUAccelerationPolicy::get_free_surface(pool_key_t key) { auto pool_it = pool_table.find(key); if (pool_it == pool_table.end()) { diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp index 35165a55c6..fdc0afd4bf 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp @@ -13,7 +13,6 @@ #include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS #ifdef HAVE_ONEVPL -#include #include "streaming/onevpl/accelerators/accel_policy_interface.hpp" #include "streaming/onevpl/accelerators/surface/surface_pool.hpp" @@ -25,14 +24,15 @@ namespace onevpl { // GAPI_EXPORTS for tests struct GAPI_EXPORTS VPLCPUAccelerationPolicy final : public VPLAccelerationPolicy { - VPLCPUAccelerationPolicy(); + VPLCPUAccelerationPolicy(device_selector_ptr_t selector); ~VPLCPUAccelerationPolicy(); using pool_t = CachedPool; void init(session_t session) override; void deinit(session_t session) override; - pool_key_t create_surface_pool(size_t pool_size, size_t surface_size_bytes, surface_ptr_ctr_t creator) override; + pool_key_t create_surface_pool(size_t pool_size, size_t surface_size_bytes, surface_ptr_ctr_t creator); + pool_key_t create_surface_pool(const mfxFrameAllocRequest& alloc_request, mfxVideoParam& param) override; surface_weak_ptr_t get_free_surface(pool_key_t key) override; size_t get_free_surface_count(pool_key_t key) const override; size_t get_surface_count(pool_key_t key) const override; diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp index 8365dd20e3..f528190ad5 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp @@ -5,8 +5,10 @@ // Copyright (C) 2021 Intel Corporation #ifdef HAVE_ONEVPL +#include + #include "streaming/onevpl/accelerators/accel_policy_dx11.hpp" -#include "streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp" +#include "streaming/onevpl/accelerators/surface/dx11_frame_adapter.hpp" #include "streaming/onevpl/accelerators/surface/surface.hpp" #include "streaming/onevpl/utils.hpp" #include "logger.hpp" @@ -30,28 +32,60 @@ namespace gapi { namespace wip { namespace onevpl { -VPLDX11AccelerationPolicy::VPLDX11AccelerationPolicy() : - hw_handle(nullptr) +VPLDX11AccelerationPolicy::VPLDX11AccelerationPolicy(device_selector_ptr_t selector) : + VPLAccelerationPolicy(selector), + hw_handle(), + device_context(), + allocator() { -#ifdef CPU_ACCEL_ADAPTER - adapter.reset(new VPLCPUAccelerationPolicy); -#endif + // setup dx11 device + IDeviceSelector::DeviceScoreTable devices = get_device_selector()->select_devices(); + GAPI_Assert(devices.size() == 1 && "Multiple(or zero) acceleration devices case is unsupported"); + AccelType accel_type = devices.begin()->second.get_type(); + GAPI_Assert(accel_type == AccelType::DX11 && + "Unexpected device AccelType while is waiting AccelType::DX11"); + + hw_handle = reinterpret_cast(devices.begin()->second.get_ptr()); + + // setup dx11 context + IDeviceSelector::DeviceContexts contexts = get_device_selector()->select_context(); + GAPI_Assert(contexts.size() == 1 && "Multiple(or zero) acceleration context case is unsupported"); + accel_type = contexts.begin()->get_type(); + GAPI_Assert(accel_type == AccelType::DX11 && + "Unexpected context AccelType while is waiting AccelType::DX11"); + device_context = reinterpret_cast(contexts.begin()->get_ptr()); + + // setup dx11 allocator + memset(&allocator, 0, sizeof(mfxFrameAllocator)); + allocator.Alloc = alloc_cb; + allocator.Lock = lock_cb; + allocator.Unlock = unlock_cb; + allocator.GetHDL = get_hdl_cb; + allocator.Free = free_cb; + allocator.pthis = this; } VPLDX11AccelerationPolicy::~VPLDX11AccelerationPolicy() { - if (hw_handle) - { - GAPI_LOG_INFO(nullptr, "VPLDX11AccelerationPolicy release ID3D11Device"); - hw_handle->Release(); + for (auto& allocation_pair : allocation_table) { + allocation_pair.second.reset(); } + GAPI_LOG_INFO(nullptr, "destroyed"); } void VPLDX11AccelerationPolicy::init(session_t session) { - mfxStatus sts = MFXVideoCORE_GetHandle(session, MFX_HANDLE_D3D11_DEVICE, reinterpret_cast(&hw_handle)); + mfxStatus sts = MFXVideoCORE_SetHandle(session, MFX_HANDLE_D3D11_DEVICE, + static_cast(hw_handle)); if (sts != MFX_ERR_NONE) { - throw std::logic_error("Cannot create VPLDX11AccelerationPolicy, MFXVideoCORE_GetHandle error: " + + throw std::logic_error("Cannot create VPLDX11AccelerationPolicy, MFXVideoCORE_SetHandle error: " + + mfxstatus_to_string(sts)); + } + + sts = MFXVideoCORE_SetFrameAllocator(session, &allocator); + if (sts != MFX_ERR_NONE) + { + throw std::logic_error("Cannot create VPLDX11AccelerationPolicy, MFXVideoCORE_SetFrameAllocator error: " + mfxstatus_to_string(sts)); } @@ -63,53 +97,289 @@ void VPLDX11AccelerationPolicy::deinit(session_t session) { } VPLDX11AccelerationPolicy::pool_key_t -VPLDX11AccelerationPolicy::create_surface_pool(size_t pool_size, size_t surface_size_bytes, - surface_ptr_ctr_t creator) { - GAPI_LOG_DEBUG(nullptr, "pool size: " << pool_size << ", surface size bytes: " << surface_size_bytes); +VPLDX11AccelerationPolicy::create_surface_pool(const mfxFrameAllocRequest& alloc_req, + mfxVideoParam& param) { + param.IOPattern = MFX_IOPATTERN_OUT_VIDEO_MEMORY; -#ifdef CPU_ACCEL_ADAPTER - return adapter->create_surface_pool(pool_size, surface_size_bytes, creator); -#endif - (void)pool_size; - (void)surface_size_bytes; - (void)creator; - throw std::runtime_error("VPLDX11AccelerationPolicy::create_surface_pool() is not implemented"); + // allocate textures by explicit request + mfxFrameAllocResponse mfxResponse; + mfxStatus sts = on_alloc(&alloc_req, &mfxResponse); + if (sts != MFX_ERR_NONE) + { + throw std::logic_error("Cannot create allocated memory for surfaces, error: " + + mfxstatus_to_string(sts)); + } + + // get reference pointer + auto table_it = allocation_table.find(alloc_req.AllocId); + GAPI_DbgAssert (allocation_table.end() != table_it); + + mfxU16 numSurfaces = alloc_req.NumFrameSuggested; + + // NB: create pool with numSurfaces reservation + pool_t pool(numSurfaces); + for (int i = 0; i < numSurfaces; i++) { + std::unique_ptr handle(new mfxFrameSurface1 {}); + handle->Info = param.mfx.FrameInfo; + handle->Data.MemId = mfxResponse.mids[i]; + + pool.push_back(Surface::create_surface(std::move(handle), table_it->second)); + } + + // remember pool by key + pool_key_t key = reinterpret_cast(table_it->second.get()); + GAPI_LOG_INFO(nullptr, "New pool allocated, key: " << key << + ", surface count: " << pool.total_size()); + try { + if (!pool_table.emplace(key, std::move(pool)).second) { + throw std::runtime_error(std::string("VPLDX11AccelerationPolicy::create_surface_pool - ") + + "cannot insert pool, table size: " + std::to_string(pool_table.size())); + } + } catch (const std::exception&) { + throw; + } + return key; } -VPLDX11AccelerationPolicy::surface_weak_ptr_t VPLDX11AccelerationPolicy::get_free_surface(pool_key_t key) -{ -#ifdef CPU_ACCEL_ADAPTER - return adapter->get_free_surface(key); -#endif - (void)key; - throw std::runtime_error("VPLDX11AccelerationPolicy::get_free_surface() is not implemented"); +VPLDX11AccelerationPolicy::surface_weak_ptr_t VPLDX11AccelerationPolicy::get_free_surface(pool_key_t key) { + auto pool_it = pool_table.find(key); + if (pool_it == pool_table.end()) { + std::stringstream ss; + ss << "key is not found: " << key << ", table size: " << pool_table.size(); + const std::string& str = ss.str(); + GAPI_LOG_WARNING(nullptr, str); + throw std::runtime_error(std::string(__FUNCTION__) + " - " + str); + } + + pool_t& requested_pool = pool_it->second; + return requested_pool.find_free(); } -size_t VPLDX11AccelerationPolicy::get_free_surface_count(pool_key_t key) const { -#ifdef CPU_ACCEL_ADAPTER - return adapter->get_free_surface_count(key); -#endif - (void)key; - throw std::runtime_error("get_free_surface_count() is not implemented"); +size_t VPLDX11AccelerationPolicy::get_free_surface_count(pool_key_t) const { + GAPI_Assert(false && "get_free_surface_count() is not implemented"); } -size_t VPLDX11AccelerationPolicy::get_surface_count(pool_key_t key) const { -#ifdef CPU_ACCEL_ADAPTER - return adapter->get_surface_count(key); -#endif - (void)key; - throw std::runtime_error("VPLDX11AccelerationPolicy::get_surface_count() is not implemented"); +size_t VPLDX11AccelerationPolicy::get_surface_count(pool_key_t) const { + GAPI_Assert(false && "VPLDX11AccelerationPolicy::get_surface_count() is not implemented"); } cv::MediaFrame::AdapterPtr VPLDX11AccelerationPolicy::create_frame_adapter(pool_key_t key, mfxFrameSurface1* surface) { + auto pool_it = pool_table.find(key); + if (pool_it == pool_table.end()) { + std::stringstream ss; + ss << "key is not found: " << key << ", table size: " << pool_table.size(); + const std::string& str = ss.str(); + GAPI_LOG_WARNING(nullptr, str); + throw std::runtime_error(std::string(__FUNCTION__) + " - " + str); + } -#ifdef CPU_ACCEL_ADAPTER - return adapter->create_frame_adapter(key, surface); -#endif - (void)key; - (void)surface; - throw std::runtime_error("VPLDX11AccelerationPolicy::create_frame_adapter() is not implemented"); + pool_t& requested_pool = pool_it->second; + return cv::MediaFrame::AdapterPtr{new VPLMediaFrameDX11Adapter(requested_pool.find_by_handle(surface))}; +} + +mfxStatus VPLDX11AccelerationPolicy::alloc_cb(mfxHDL pthis, mfxFrameAllocRequest *request, + mfxFrameAllocResponse *response) { + if (!pthis) { + return MFX_ERR_MEMORY_ALLOC; + } + + VPLDX11AccelerationPolicy *self = static_cast(pthis); + + return self->on_alloc(request, response); +} + +mfxStatus VPLDX11AccelerationPolicy::lock_cb(mfxHDL pthis, mfxMemId mid, mfxFrameData *ptr) { + VPLDX11AccelerationPolicy *self = static_cast(pthis); + GAPI_LOG_DEBUG(nullptr, "called from: " << self ? "Policy" : "Resource"); + cv::util::suppress_unused_warning(self); + return on_lock(mid, ptr); +} + +mfxStatus VPLDX11AccelerationPolicy::unlock_cb(mfxHDL pthis, mfxMemId mid, mfxFrameData *ptr) { + VPLDX11AccelerationPolicy *self = static_cast(pthis); + GAPI_LOG_DEBUG(nullptr, "called from: " << self ? "Policy" : "Resource"); + cv::util::suppress_unused_warning(self); + return on_unlock(mid, ptr); +} + +mfxStatus VPLDX11AccelerationPolicy::get_hdl_cb(mfxHDL pthis, mfxMemId mid, mfxHDL *handle) { + VPLDX11AccelerationPolicy *self = static_cast(pthis); + + GAPI_LOG_DEBUG(nullptr, "called from: " << self ? "Policy" : "Resource"); + cv::util::suppress_unused_warning(self); + return on_get_hdl(mid, handle); +} + +mfxStatus VPLDX11AccelerationPolicy::free_cb(mfxHDL pthis, mfxFrameAllocResponse *response) { + if (!pthis) { + return MFX_ERR_MEMORY_ALLOC; + } + + VPLDX11AccelerationPolicy *self = static_cast(pthis); + return self->on_free(response); +} + +mfxStatus VPLDX11AccelerationPolicy::on_alloc(const mfxFrameAllocRequest *request, + mfxFrameAllocResponse *response) { + GAPI_LOG_DEBUG(nullptr, "Requested allocation id: " << std::to_string(request->AllocId) << + ", type: " << ext_mem_frame_type_to_cstr(request->Type) << + ", size: " << request->Info.Width << "x" << request->Info.Height << + ", frames minimum count: " << request->NumFrameMin << + ", frames suggested count: " << request->NumFrameSuggested); + auto table_it = allocation_table.find(request->AllocId); + if (allocation_table.end() != table_it) { + GAPI_LOG_WARNING(nullptr, "Allocation already exists, id: " + std::to_string(request->AllocId) + + ". Total allocation size: " + std::to_string(allocation_table.size())); + + // TODO cache + allocation_t &resources_array = table_it->second; + response->AllocId = request->AllocId; + GAPI_DbgAssert(static_cast(std::numeric_limits::max()) > resources_array->size() && + "Invalid num frames: overflow"); + response->NumFrameActual = static_cast(resources_array->size()); + response->mids = reinterpret_cast(resources_array->data()); + + return MFX_ERR_NONE; + } + + DXGI_FORMAT colorFormat = VPLMediaFrameDX11Adapter::get_dx11_color_format(request->Info.FourCC); + + if (DXGI_FORMAT_UNKNOWN == colorFormat || colorFormat != DXGI_FORMAT_NV12) { + GAPI_LOG_WARNING(nullptr, "Unsupported fourcc :" << request->Info.FourCC); + return MFX_ERR_UNSUPPORTED; + } + + D3D11_TEXTURE2D_DESC desc = { 0 }; + + desc.Width = request->Info.Width; + desc.Height = request->Info.Height; + + desc.MipLevels = 1; + // single texture with subresources + desc.ArraySize = request->NumFrameSuggested; + desc.Format = colorFormat; + desc.SampleDesc.Count = 1; + desc.Usage = D3D11_USAGE_DEFAULT; + desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED; + desc.BindFlags = D3D11_BIND_DECODER; + + if (request->Type & MFX_MEMTYPE_SHARED_RESOURCE) { + desc.BindFlags |= D3D11_BIND_SHADER_RESOURCE; + desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED; + } + + ComPtrGuard main_texture = createCOMPtrGuard(); + HRESULT err = S_OK; + { + ID3D11Texture2D *pTexture2D = nullptr; + err = hw_handle->CreateTexture2D(&desc, nullptr, &pTexture2D); + if (FAILED(err)) { + GAPI_LOG_WARNING(nullptr, "Cannot create texture, error: " + std::to_string(HRESULT_CODE(err))); + return MFX_ERR_MEMORY_ALLOC; + } + main_texture.reset(pTexture2D); + } + + // create staging texture to read it from + desc.ArraySize = 1; + desc.Usage = D3D11_USAGE_STAGING; + desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ | D3D11_CPU_ACCESS_WRITE; + desc.BindFlags = 0; + desc.MiscFlags = 0; + std::vector> staging_textures; + staging_textures.reserve(request->NumFrameSuggested); + for (int i = 0; i < request->NumFrameSuggested; i++ ) { + ID3D11Texture2D *staging_texture_2d = nullptr; + err = hw_handle->CreateTexture2D(&desc, NULL, &staging_texture_2d); + if (FAILED(err)) { + GAPI_LOG_WARNING(nullptr, "Cannot create staging texture, error: " + std::to_string(HRESULT_CODE(err))); + return MFX_ERR_MEMORY_ALLOC; + } + staging_textures.push_back(createCOMPtrGuard(staging_texture_2d)); + } + + // for multiple subresources initialize allocation array + auto cand_resource_it = allocation_table.end(); + { + // insert into global table + auto inserted_it = + allocation_table.emplace(request->AllocId, + DX11AllocationRecord::create(request->NumFrameSuggested, + device_context, + allocator, + std::move(main_texture), + std::move(staging_textures))); + if (!inserted_it.second) { + GAPI_LOG_WARNING(nullptr, "Cannot assign allocation by id: " + std::to_string(request->AllocId) + + " - aldeady exist. Total allocation size: " + std::to_string(allocation_table.size())); + return MFX_ERR_MEMORY_ALLOC; + } + + GAPI_LOG_DEBUG(nullptr, "allocation by id: " << request->AllocId << + " was created, total allocations count: " << allocation_table.size()); + cand_resource_it = inserted_it.first; + } + + // fill out response + GAPI_DbgAssert(cand_resource_it != allocation_table.end() && "Invalid cand_resource_it"); + + allocation_t &resources_array = cand_resource_it->second; + response->AllocId = request->AllocId; + response->NumFrameActual = request->NumFrameSuggested; + response->mids = reinterpret_cast(resources_array->data()); + + return MFX_ERR_NONE; +} + +mfxStatus VPLDX11AccelerationPolicy::on_lock(mfxMemId mid, mfxFrameData *ptr) { + DX11AllocationRecord::AllocationId data = reinterpret_cast(mid); + if (!data) { + GAPI_LOG_WARNING(nullptr, "Allocation record is empty"); + return MFX_ERR_LOCK_MEMORY; + } + + return data->acquire_access(ptr); +} + +mfxStatus VPLDX11AccelerationPolicy::on_unlock(mfxMemId mid, mfxFrameData *ptr) { + DX11AllocationRecord::AllocationId data = reinterpret_cast(mid); + if (!data) { + return MFX_ERR_LOCK_MEMORY; + } + + return data->release_access(ptr); +} + +mfxStatus VPLDX11AccelerationPolicy::on_get_hdl(mfxMemId mid, mfxHDL *handle) { + DX11AllocationRecord::AllocationId data = reinterpret_cast(mid); + if (!data) { + return MFX_ERR_INVALID_HANDLE; + } + + mfxHDLPair *pPair = reinterpret_cast(handle); + + pPair->first = data->get_texture_ptr(); + pPair->second = static_cast(reinterpret_cast( + static_cast(data->get_subresource()))); + + GAPI_LOG_DEBUG(nullptr, "texture : " << pPair->first << ", sub id: " << pPair->second); + return MFX_ERR_NONE; +} + +mfxStatus VPLDX11AccelerationPolicy::on_free(mfxFrameAllocResponse *response) { + GAPI_LOG_DEBUG(nullptr, "Allocations count before: " << allocation_table.size() << + ", requested id: " << response->AllocId); + + auto table_it = allocation_table.find(response->AllocId); + if (allocation_table.end() == table_it) { + GAPI_LOG_WARNING(nullptr, "Cannot find allocation id: " + std::to_string(response->AllocId) + + ". Total allocation size: " + std::to_string(allocation_table.size())); + return MFX_ERR_MEMORY_ALLOC; + } + + allocation_table.erase(table_it); + return MFX_ERR_NONE; } } // namespace onevpl } // namespace wip diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp index 946640d886..e053089587 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp @@ -6,18 +6,14 @@ #ifndef GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_DX11_HPP #define GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_DX11_HPP +#include #include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS -//TODO Remove the next MACRO right after DX11 implementation -#define CPU_ACCEL_ADAPTER #ifdef HAVE_ONEVPL -#include #include "streaming/onevpl/accelerators/accel_policy_interface.hpp" - -#ifdef CPU_ACCEL_ADAPTER -#include "streaming/onevpl/accelerators/accel_policy_cpu.hpp" -#endif +#include "streaming/onevpl/accelerators/surface/surface_pool.hpp" +#include "streaming/onevpl/accelerators/dx11_alloc_resource.hpp" #ifdef HAVE_DIRECTX #ifdef HAVE_D3D11 @@ -39,30 +35,52 @@ namespace onevpl { struct GAPI_EXPORTS VPLDX11AccelerationPolicy final: public VPLAccelerationPolicy { // GAPI_EXPORTS for tests - VPLDX11AccelerationPolicy(); + VPLDX11AccelerationPolicy(device_selector_ptr_t selector); ~VPLDX11AccelerationPolicy(); + using pool_t = CachedPool; + void init(session_t session) override; void deinit(session_t session) override; - pool_key_t create_surface_pool(size_t pool_size, size_t surface_size_bytes, surface_ptr_ctr_t creator) override; + pool_key_t create_surface_pool(const mfxFrameAllocRequest& alloc_request, + mfxVideoParam& param) override; surface_weak_ptr_t get_free_surface(pool_key_t key) override; size_t get_free_surface_count(pool_key_t key) const override; size_t get_surface_count(pool_key_t key) const override; cv::MediaFrame::AdapterPtr create_frame_adapter(pool_key_t key, - mfxFrameSurface1* surface) override; - + mfxFrameSurface1* surface) override; private: ID3D11Device *hw_handle; + ID3D11DeviceContext* device_context; -#ifdef CPU_ACCEL_ADAPTER - std::unique_ptr adapter; -#endif + mfxFrameAllocator allocator; + static mfxStatus MFX_CDECL alloc_cb(mfxHDL pthis, + mfxFrameAllocRequest *request, + mfxFrameAllocResponse *response); + static mfxStatus MFX_CDECL lock_cb(mfxHDL pthis, mfxMemId mid, mfxFrameData *ptr); + static mfxStatus MFX_CDECL unlock_cb(mfxHDL pthis, mfxMemId mid, mfxFrameData *ptr); + static mfxStatus MFX_CDECL get_hdl_cb(mfxHDL pthis, mfxMemId mid, mfxHDL *handle); + static mfxStatus MFX_CDECL free_cb(mfxHDL pthis, mfxFrameAllocResponse *response); + + virtual mfxStatus on_alloc(const mfxFrameAllocRequest *request, + mfxFrameAllocResponse *response); + static mfxStatus on_lock(mfxMemId mid, mfxFrameData *ptr); + static mfxStatus on_unlock(mfxMemId mid, mfxFrameData *ptr); + static mfxStatus on_get_hdl(mfxMemId mid, mfxHDL *handle); + virtual mfxStatus on_free(mfxFrameAllocResponse *response); + + using alloc_id_t = mfxU32; + using allocation_t = std::shared_ptr; + std::map allocation_table; + + std::map pool_table; }; } // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv +#undef NOMINMAX #endif // HAVE_D3D11 #endif // HAVE_DIRECTX diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp index 31ee91535c..a9059c29ef 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp @@ -12,9 +12,10 @@ #include #include +#include #ifdef HAVE_ONEVPL -#include +#include "streaming/onevpl/onevpl_export.hpp" namespace cv { namespace gapi { @@ -24,7 +25,10 @@ namespace onevpl { class Surface; struct VPLAccelerationPolicy { - virtual ~VPLAccelerationPolicy() {} + using device_selector_ptr_t = std::shared_ptr; + + VPLAccelerationPolicy(device_selector_ptr_t selector) : device_selector(selector) {} + virtual ~VPLAccelerationPolicy() = default; using pool_key_t = void*; @@ -36,6 +40,13 @@ struct VPLAccelerationPolicy size_t out_buf_ptr_offset, size_t out_buf_ptr_size)>; + device_selector_ptr_t get_device_selector() { + return device_selector; + } + const device_selector_ptr_t get_device_selector() const { + return device_selector; + } + virtual void init(session_t session) = 0; virtual void deinit(session_t session) = 0; @@ -43,7 +54,7 @@ struct VPLAccelerationPolicy // for existing workspace in existing pool (see realloc) // thus it is not implemented, // PLEASE provide initial memory area large enough - virtual pool_key_t create_surface_pool(size_t pool_size, size_t surface_size_bytes, surface_ptr_ctr_t creator) = 0; + virtual pool_key_t create_surface_pool(const mfxFrameAllocRequest& alloc_request, mfxVideoParam& param) = 0; virtual surface_weak_ptr_t get_free_surface(pool_key_t key) = 0; virtual size_t get_free_surface_count(pool_key_t key) const = 0; @@ -51,6 +62,8 @@ struct VPLAccelerationPolicy virtual cv::MediaFrame::AdapterPtr create_frame_adapter(pool_key_t key, mfxFrameSurface1* surface) = 0; +private: + device_selector_ptr_t device_selector; }; } // namespace onevpl } // namespace wip diff --git a/modules/gapi/src/streaming/onevpl/accelerators/dx11_alloc_resource.cpp b/modules/gapi/src/streaming/onevpl/accelerators/dx11_alloc_resource.cpp new file mode 100644 index 0000000000..3bbfb25b0a --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/dx11_alloc_resource.cpp @@ -0,0 +1,404 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifdef HAVE_ONEVPL +#include "streaming/onevpl/accelerators/dx11_alloc_resource.hpp" +#include "streaming/onevpl/accelerators/utils/shared_lock.hpp" +#include "logger.hpp" + +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +LockAdapter::LockAdapter(mfxFrameAllocator origin_allocator) : + lockable_allocator(origin_allocator), + impl() { + GAPI_DbgAssert((lockable_allocator.Lock && lockable_allocator.Unlock) && + "Cannot create LockAdapter for empty origin allocator"); + + // abandon unusable c-allocator interfaces + // because LockAdapter requires Lock & Unlock only + lockable_allocator.Alloc = nullptr; + lockable_allocator.Free = nullptr; + lockable_allocator.pthis = nullptr; +} + +size_t LockAdapter::read_lock(mfxMemId mid, mfxFrameData &data) { + size_t prev_lock_count = 0; + if (impl) { + prev_lock_count = impl->shared_lock(); + } + + // dispatch to VPL allocator using READ access mode + mfxStatus sts = MFX_ERR_LOCK_MEMORY; + try { + sts = lockable_allocator.Lock(nullptr, mid, &data); + } catch(...) { + } + + // adapter will throw error if VPL frame allocator fails + if (sts != MFX_ERR_NONE) { + impl->unlock_shared(); + GAPI_Assert(false && "Cannot lock frame on READ using VPL allocator"); + } + + return prev_lock_count; +} + +size_t LockAdapter::unlock_read(mfxMemId mid, mfxFrameData &data) { + GAPI_DbgAssert(!impl || !is_write_acquired() && + "Reject `unlock_read` in `write_lock` state"); + lockable_allocator.Unlock(nullptr, mid, &data); + return impl ? impl->unlock_shared() : 0; +} + +void LockAdapter::write_lock(mfxMemId mid, mfxFrameData &data) { + if (impl) { + // TODO consider using `try_lock` in loop with limited iteration count + // to prevent dead-lock with WARN at least notification + impl->lock(); + } + + // dispatch to VPL allocator using READ access mode + mfxStatus sts = MFX_ERR_LOCK_MEMORY; + try { + sts = lockable_allocator.Lock(nullptr, mid, &data); + } catch(...) { + } + + // adapter will throw error if VPL frame allocator fails + if (sts != MFX_ERR_NONE) { + impl->unlock(); + GAPI_Assert(false && "Cannot lock frame on WRITE using VPL allocator"); + } +} + +bool LockAdapter::is_write_acquired() { + if(!impl) return true; + return impl->owns(); +} + +void LockAdapter::unlock_write(mfxMemId mid, mfxFrameData &data) { + GAPI_DbgAssert(is_write_acquired() && + "Reject `unlock_write` for unlocked state"); + lockable_allocator.Unlock(nullptr, mid, &data); + if (impl) { + impl->unlock(); + } +} + +SharedLock* LockAdapter::set_adaptee(SharedLock* new_impl) { + SharedLock* old_impl = impl; + impl = new_impl; + return old_impl; +} + +SharedLock* LockAdapter::get_adaptee() { + return impl; +} + +NativeHandleAdapter::NativeHandleAdapter(mfxFrameAllocator origin_allocator) : + native_handle_getter(origin_allocator) { + GAPI_DbgAssert(native_handle_getter.GetHDL && + "Cannot create NativeHandleAdapter for empty origin allocator"); + + // abandon unusable c-allocator interfaces + // because NativeHandleAdapter requires `GetHDL` only + native_handle_getter.Alloc = nullptr; + native_handle_getter.Free = nullptr; + native_handle_getter.Lock = nullptr; + native_handle_getter.Unlock = nullptr; + native_handle_getter.pthis = nullptr; +} + +void NativeHandleAdapter::get_handle(mfxMemId mid, mfxHDL& out) { + if (native_handle_getter.GetHDL(nullptr, mid, &out) != MFX_ERR_NONE) { + GAPI_Assert(nullptr && "Cannot get native handle for resourse by mid"); + } +} + +DX11AllocationItem::DX11AllocationItem(std::weak_ptr parent, + ID3D11DeviceContext* origin_ctx, + mfxFrameAllocator origin_allocator, + ComSharedPtrGuard tex_ptr, + subresource_id_t subtex_id, + ComPtrGuard&& staging_tex_ptr) : + LockAdapter(origin_allocator), + NativeHandleAdapter(origin_allocator), + shared_device_context(origin_ctx), + texture_ptr(tex_ptr), + subresource_id(subtex_id), + staging_texture_ptr(std::move(staging_tex_ptr)), + observer(parent) { + GAPI_DbgAssert(texture_ptr && + "Cannot create DX11AllocationItem for empty texture"); + GAPI_DbgAssert(staging_texture_ptr && + "Cannot create DX11AllocationItem for empty staging texture"); + GAPI_DbgAssert(observer.lock() && + "Cannot create DX11AllocationItem for empty parent"); + + shared_device_context->AddRef(); +} + +DX11AllocationItem::~DX11AllocationItem() { + release(); + observer.reset(); + if (shared_device_context) { + shared_device_context->Release(); + } +} + +void DX11AllocationItem::release() { + auto parent = observer.lock(); + GAPI_LOG_DEBUG(nullptr, "texture: " << texture_ptr << + ", subresource id: " << subresource_id << + ", parent: " << parent.get()); + cv::util::suppress_unused_warning(parent); +} + +ID3D11Texture2D* DX11AllocationItem::get_texture_ptr() { + return texture_ptr.get(); +} + +ID3D11Texture2D* DX11AllocationItem::get_staging_texture_ptr() { + return staging_texture_ptr.get(); +} + +DX11AllocationItem::subresource_id_t DX11AllocationItem::get_subresource() const { + return subresource_id; +} + +ID3D11DeviceContext* DX11AllocationItem::get_device_ctx_ptr() { + return shared_device_context;//shared_device_context.get(); +} + +void DX11AllocationItem::on_first_in_impl(mfxFrameData *ptr) { + D3D11_MAP mapType = D3D11_MAP_READ; + UINT mapFlags = D3D11_MAP_FLAG_DO_NOT_WAIT; + + shared_device_context->CopySubresourceRegion(get_staging_texture_ptr(), 0, + 0, 0, 0, + get_texture_ptr(), + get_subresource(), + nullptr); + HRESULT err = S_OK; + D3D11_MAPPED_SUBRESOURCE lockedRect {}; + do { + err = shared_device_context->Map(get_staging_texture_ptr(), 0, mapType, mapFlags, &lockedRect); + if (S_OK != err && DXGI_ERROR_WAS_STILL_DRAWING != err) { + GAPI_LOG_WARNING(nullptr, "Cannot Map staging texture in device context, error: " << std::to_string(HRESULT_CODE(err))); + GAPI_Assert(false && "Cannot Map staging texture in device context"); + } + } while (DXGI_ERROR_WAS_STILL_DRAWING == err); + + if (FAILED(err)) { + GAPI_LOG_WARNING(nullptr, "Cannot lock frame"); + GAPI_Assert(false && "Cannot lock frame"); + return ; + } + + D3D11_TEXTURE2D_DESC desc {}; + get_texture_ptr()->GetDesc(&desc); + switch (desc.Format) { + case DXGI_FORMAT_NV12: + ptr->Pitch = (mfxU16)lockedRect.RowPitch; + ptr->Y = (mfxU8 *)lockedRect.pData; + ptr->UV = (mfxU8 *)lockedRect.pData + desc.Height * lockedRect.RowPitch; + + GAPI_Assert(ptr->Y && ptr->UV && "DXGI_FORMAT_NV12 locked frame data is nullptr"); + break; + default: + GAPI_LOG_WARNING(nullptr, "Unknown DXGI format: " << desc.Format); + return; + } +} + +void DX11AllocationItem::on_last_out_impl(mfxFrameData *ptr) { + shared_device_context->Unmap(get_staging_texture_ptr(), 0); + if (ptr) { + ptr->Pitch = 0; + ptr->U = ptr->V = ptr->Y = 0; + ptr->A = ptr->R = ptr->G = ptr->B = 0; + } +} + +mfxStatus DX11AllocationItem::acquire_access(mfxFrameData *ptr) { + if (is_write_acquired()) { + return exclusive_access_acquire_unsafe(ptr); + } + return shared_access_acquire_unsafe(ptr); +} + +mfxStatus DX11AllocationItem::release_access(mfxFrameData *ptr) { + if (is_write_acquired()) { + return exclusive_access_release_unsafe(ptr); + } + return shared_access_release_unsafe(ptr); +} + +mfxStatus DX11AllocationItem::shared_access_acquire_unsafe(mfxFrameData *ptr) { + GAPI_LOG_DEBUG(nullptr, "acquire READ lock: " << this); + GAPI_LOG_DEBUG(nullptr, "texture: " << get_texture_ptr() << + ", sub id: " << get_subresource()); + // shared access requires elastic barrier + // first-in visited thread uses resource mapping on host memory + // subsequent threads reuses mapped memory + // + // exclusive access is prohibited while any one shared access has been obtained + visit_in(ptr); + + if (!(ptr->Y && (ptr->UV || (ptr->U && ptr->V)))) { + GAPI_LOG_WARNING(nullptr, "No any data obtained: " << this); + return MFX_ERR_LOCK_MEMORY; + } + GAPI_LOG_DEBUG(nullptr, "READ access granted: " << this); + return MFX_ERR_NONE; +} + +mfxStatus DX11AllocationItem::shared_access_release_unsafe(mfxFrameData *ptr) { + GAPI_LOG_DEBUG(nullptr, "releasing READ lock: " << this); + GAPI_LOG_DEBUG(nullptr, "texture: " << get_texture_ptr() << + ", sub id: " << get_subresource()); + // releasing shared access requires elastic barrier + // last-out thread must make memory unmapping then and only then no more + // read access is coming. If another read-access goes into critical section + // (or waiting for acees) we must drop off unmapping procedure + visit_out(ptr); + + GAPI_LOG_DEBUG(nullptr, "access on READ released: " << this); + return MFX_ERR_NONE; +} + +mfxStatus DX11AllocationItem::exclusive_access_acquire_unsafe(mfxFrameData *ptr) { + GAPI_LOG_DEBUG(nullptr, "acquire WRITE lock: " << this); + GAPI_LOG_DEBUG(nullptr, "texture: " << get_texture_ptr() << + ", sub id: " << get_subresource()); + D3D11_MAP mapType = D3D11_MAP_WRITE; + UINT mapFlags = D3D11_MAP_FLAG_DO_NOT_WAIT; + + HRESULT err = S_OK; + D3D11_MAPPED_SUBRESOURCE lockedRect {}; + do { + err = get_device_ctx_ptr()->Map(get_staging_texture_ptr(), 0, mapType, mapFlags, &lockedRect); + if (S_OK != err && DXGI_ERROR_WAS_STILL_DRAWING != err) { + GAPI_LOG_WARNING(nullptr, "Cannot Map staging texture in device context, error: " << std::to_string(HRESULT_CODE(err))); + return MFX_ERR_LOCK_MEMORY; + } + } while (DXGI_ERROR_WAS_STILL_DRAWING == err); + + if (FAILED(err)) { + GAPI_LOG_WARNING(nullptr, "Cannot lock frame"); + return MFX_ERR_LOCK_MEMORY; + } + + D3D11_TEXTURE2D_DESC desc {}; + get_texture_ptr()->GetDesc(&desc); + switch (desc.Format) { + case DXGI_FORMAT_NV12: + ptr->Pitch = (mfxU16)lockedRect.RowPitch; + ptr->Y = (mfxU8 *)lockedRect.pData; + ptr->UV = (mfxU8 *)lockedRect.pData + desc.Height * lockedRect.RowPitch; + if (!ptr->Y || !ptr->UV) { + GAPI_LOG_WARNING(nullptr, "DXGI_FORMAT_NV12 locked frame data is nullptr"); + return MFX_ERR_LOCK_MEMORY; + } + break; + default: + GAPI_LOG_WARNING(nullptr, "Unknown DXGI format: " << desc.Format); + return MFX_ERR_LOCK_MEMORY; + } + + GAPI_LOG_DEBUG(nullptr, "WRITE access granted: " << this); + return MFX_ERR_NONE; +} + +mfxStatus DX11AllocationItem::exclusive_access_release_unsafe(mfxFrameData *ptr) { + GAPI_LOG_DEBUG(nullptr, "releasing WRITE lock: " << this); + GAPI_LOG_DEBUG(nullptr, "texture: " << get_texture_ptr() << + ", sub id: " << get_subresource()); + + get_device_ctx_ptr()->Unmap(get_staging_texture_ptr(), 0); + + get_device_ctx_ptr()->CopySubresourceRegion(get_texture_ptr(), + get_subresource(), + 0, 0, 0, + get_staging_texture_ptr(), 0, + nullptr); + + if (ptr) { + ptr->Pitch = 0; + ptr->U = ptr->V = ptr->Y = 0; + ptr->A = ptr->R = ptr->G = ptr->B = 0; + } + GAPI_LOG_DEBUG(nullptr, "access on WRITE released: " << this); + return MFX_ERR_NONE; +} + +DX11AllocationRecord::DX11AllocationRecord() = default; + +DX11AllocationRecord::~DX11AllocationRecord() { + GAPI_LOG_DEBUG(nullptr, "record: " << this << + ", subresources count: " << resources.size()); + + for (AllocationId id : resources) { + delete id; + } + resources.clear(); + + GAPI_LOG_DEBUG(nullptr, "release final referenced texture: " << texture_ptr.get()); +} + +void DX11AllocationRecord::init(unsigned int items, + ID3D11DeviceContext* origin_ctx, + mfxFrameAllocator origin_allocator, + ComPtrGuard&& texture, + std::vector> &&staging_textures) { + GAPI_DbgAssert(items != 0 && "Cannot create DX11AllocationRecord with empty items"); + GAPI_DbgAssert(items == staging_textures.size() && "Allocation items count and staging size are not equal"); + GAPI_DbgAssert(origin_ctx && + "Cannot create DX11AllocationItem for empty origin_ctx"); + auto shared_allocator_copy = origin_allocator; + GAPI_DbgAssert((shared_allocator_copy.Lock && shared_allocator_copy.Unlock) && + "Cannot create DX11AllocationItem for empty origin allocator"); + + // abandon unusable c-allocator interfaces + shared_allocator_copy.Alloc = nullptr; + shared_allocator_copy.Free = nullptr; + shared_allocator_copy.pthis = nullptr; + + + GAPI_LOG_DEBUG(nullptr, "subresources count: " << items << ", text: " << texture.get()); + resources.reserve(items); + // no AddRef here, because DX11AllocationRecord receive ownership it here + texture_ptr = createCOMSharedPtrGuard(std::move(texture)); + for(unsigned int i = 0; i < items; i++) { + resources.emplace_back(new DX11AllocationItem(get_ptr(), origin_ctx, shared_allocator_copy, + texture_ptr, i, std::move(staging_textures[i]))); + } +} + +DX11AllocationRecord::Ptr DX11AllocationRecord::get_ptr() { + return shared_from_this(); +} + +DX11AllocationRecord::AllocationId* DX11AllocationRecord::data() { + return resources.data(); +} + +size_t DX11AllocationRecord::size() const { + return resources.size(); +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/accelerators/dx11_alloc_resource.hpp b/modules/gapi/src/streaming/onevpl/accelerators/dx11_alloc_resource.hpp new file mode 100644 index 0000000000..46ddff86a4 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/dx11_alloc_resource.hpp @@ -0,0 +1,151 @@ +#ifndef GAPI_STREAMING_ONEVPL_ACCEL_DX11_ALLOC_RESOURCE_HPP +#define GAPI_STREAMING_ONEVPL_ACCEL_DX11_ALLOC_RESOURCE_HPP + +#include + +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS +#include + +#ifdef HAVE_ONEVPL +#include "streaming/onevpl/onevpl_export.hpp" +#include "streaming/onevpl/accelerators/utils/elastic_barrier.hpp" +#include "streaming/onevpl/utils.hpp" + +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 +#pragma comment(lib,"d3d11.lib") + +#define D3D11_NO_HELPERS +#define NOMINMAX +#include +#include +#include +#include "opencv2/core/directx.hpp" +#ifdef HAVE_OPENCL +#include +#endif // HAVE_OPENCL +#undef D3D11_NO_HELPERS +#undef NOMINMAX + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +class SharedLock; +// GAPI_EXPORTS for tests +struct GAPI_EXPORTS LockAdapter { + LockAdapter(mfxFrameAllocator origin_allocator); + + size_t read_lock(mfxMemId mid, mfxFrameData &data); + size_t unlock_read(mfxMemId mid, mfxFrameData &data); + + void write_lock(mfxMemId mid, mfxFrameData &data); + bool is_write_acquired(); + void unlock_write(mfxMemId mid, mfxFrameData &data); + + SharedLock* set_adaptee(SharedLock* new_impl); + SharedLock* get_adaptee(); +private: + LockAdapter(const LockAdapter&) = delete; + LockAdapter(LockAdapter&&) = delete; + LockAdapter& operator= (const LockAdapter&) = delete; + LockAdapter& operator= (LockAdapter&&) = delete; + + mfxFrameAllocator lockable_allocator; + SharedLock* impl; +}; + +struct GAPI_EXPORTS NativeHandleAdapter { + NativeHandleAdapter(mfxFrameAllocator origin_allocator); + + void get_handle(mfxMemId mid, mfxHDL& out); +private: + mfxFrameAllocator native_handle_getter; +}; + +struct DX11AllocationRecord; +struct DX11AllocationItem : public LockAdapter, + public NativeHandleAdapter, + public elastic_barrier { + using subresource_id_t = unsigned int; + + friend struct DX11AllocationRecord; + friend class elastic_barrier; + ~DX11AllocationItem(); + + void release(); + ID3D11Texture2D* get_texture_ptr(); + ID3D11Texture2D* get_staging_texture_ptr(); + DX11AllocationItem::subresource_id_t get_subresource() const; + + ID3D11DeviceContext* get_device_ctx_ptr(); + + // public transactional access to resources. + // implements dispatching through different access acquisition modes. + // current acquisition mode determined by `LockAdapter` with `is_write_acquired()` + mfxStatus acquire_access(mfxFrameData *ptr); + mfxStatus release_access(mfxFrameData *ptr); +private: + DX11AllocationItem(std::weak_ptr parent, + ID3D11DeviceContext* origin_ctx, + mfxFrameAllocator origin_allocator, + ComSharedPtrGuard texture_ptr, + subresource_id_t subresource_id, + ComPtrGuard&& staging_tex_ptr); + + // elastic barrier interface impl + void on_first_in_impl(mfxFrameData *ptr); + void on_last_out_impl(mfxFrameData *ptr); + + mfxStatus shared_access_acquire_unsafe(mfxFrameData *ptr); + mfxStatus shared_access_release_unsafe(mfxFrameData *ptr); + mfxStatus exclusive_access_acquire_unsafe(mfxFrameData *ptr); + mfxStatus exclusive_access_release_unsafe(mfxFrameData *ptr); + + ID3D11DeviceContext* shared_device_context; + + ComSharedPtrGuard texture_ptr; + subresource_id_t subresource_id = 0; + ComPtrGuard staging_texture_ptr; + std::weak_ptr observer; +}; + +struct DX11AllocationRecord : public std::enable_shared_from_this { + + using Ptr = std::shared_ptr; + + ~DX11AllocationRecord(); + + template + static Ptr create(Args&& ...args) { + std::shared_ptr record(new DX11AllocationRecord); + record->init(std::forward(args)...); + return record; + } + + Ptr get_ptr(); + + // Raw ptr is required as a part of VPL `Mid` c-interface + // which requires contiguous memory + using AllocationId = DX11AllocationItem*; + AllocationId* data(); + size_t size() const; +private: + DX11AllocationRecord(); + void init(unsigned int items, ID3D11DeviceContext* origin_ctx, + mfxFrameAllocator origin_allocator, + ComPtrGuard&& texture, std::vector> &&staging_textures); + + std::vector resources; + ComSharedPtrGuard texture_ptr; +}; + +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_ACCEL_DX11_ALLOC_RESOURCE_HPP diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp index d3020ab168..39094c9bc3 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp @@ -9,12 +9,7 @@ #include "logger.hpp" #ifdef HAVE_ONEVPL - -#if (MFX_VERSION >= 2000) -#include -#endif - -#include +#include "streaming/onevpl/onevpl_export.hpp" namespace cv { namespace gapi { @@ -25,12 +20,25 @@ VPLMediaFrameCPUAdapter::VPLMediaFrameCPUAdapter(std::shared_ptr surfac parent_surface_ptr(surface) { GAPI_Assert(parent_surface_ptr && "Surface is nullptr"); - parent_surface_ptr->obtain_lock(); - GAPI_LOG_DEBUG(nullptr, "surface: " << parent_surface_ptr->get_handle() << ", w: " << parent_surface_ptr->get_info().Width << ", h: " << parent_surface_ptr->get_info().Height << ", p: " << parent_surface_ptr->get_data().Pitch); + const Surface::info_t& info = parent_surface_ptr->get_info(); + switch(info.FourCC) + { + case MFX_FOURCC_I420: + throw std::runtime_error("MediaFrame doesn't support I420 type"); + break; + case MFX_FOURCC_NV12: + frame_desc.fmt = MediaFormat::NV12; + break; + default: + throw std::runtime_error("MediaFrame unknown 'fmt' type: " + std::to_string(info.FourCC)); + } + + frame_desc.size = cv::Size{info.Width, info.Height}; + parent_surface_ptr->obtain_lock(); } VPLMediaFrameCPUAdapter::~VPLMediaFrameCPUAdapter() { @@ -42,22 +50,7 @@ VPLMediaFrameCPUAdapter::~VPLMediaFrameCPUAdapter() { } cv::GFrameDesc VPLMediaFrameCPUAdapter::meta() const { - GFrameDesc desc; - const Surface::info_t& info = parent_surface_ptr->get_info(); - switch(info.FourCC) - { - case MFX_FOURCC_I420: - throw std::runtime_error("MediaFrame doesn't support I420 type"); - break; - case MFX_FOURCC_NV12: - desc.fmt = MediaFormat::NV12; - break; - default: - throw std::runtime_error("MediaFrame unknown 'fmt' type: " + std::to_string(info.FourCC)); - } - - desc.size = cv::Size{info.Width, info.Height}; - return desc; + return frame_desc; } MediaFrame::View VPLMediaFrameCPUAdapter::access(MediaFrame::Access) { diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp index 04a9bdc275..1c51ad7473 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp @@ -33,6 +33,7 @@ public: void deserialize(cv::gapi::s11n::IIStream&) override; private: std::shared_ptr parent_surface_ptr; + GFrameDesc frame_desc; }; } // namespace onevpl } // namespace wip diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/dx11_frame_adapter.cpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/dx11_frame_adapter.cpp new file mode 100644 index 0000000000..04cf10c8d7 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/dx11_frame_adapter.cpp @@ -0,0 +1,232 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include "streaming/onevpl/accelerators/surface/dx11_frame_adapter.hpp" +#include "streaming/onevpl/accelerators/dx11_alloc_resource.hpp" +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "logger.hpp" + +#ifdef HAVE_ONEVPL +#include "streaming/onevpl/onevpl_export.hpp" + +#ifdef HAVE_INF_ENGINE +// For IE classes (ParamMap, etc) +#include +#endif // HAVE_INF_ENGINE + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +void lock_mid(mfxMemId mid, mfxFrameData &data, MediaFrame::Access mode) { + LockAdapter* alloc_data = reinterpret_cast(mid); + if (mode == MediaFrame::Access::R) { + alloc_data->read_lock(mid, data); + } else { + alloc_data->write_lock(mid, data); + } +} + +void unlock_mid(mfxMemId mid, mfxFrameData &data, MediaFrame::Access mode) { + LockAdapter* alloc_data = reinterpret_cast(data.MemId); + if (mode == MediaFrame::Access::R) { + alloc_data->unlock_read(mid, data); + } else { + alloc_data->unlock_write(mid, data); + } +} + +VPLMediaFrameDX11Adapter::VPLMediaFrameDX11Adapter(std::shared_ptr surface): + parent_surface_ptr(surface) { + GAPI_Assert(parent_surface_ptr && "Surface is nullptr"); + + const Surface::info_t& info = parent_surface_ptr->get_info(); + Surface::data_t& data = parent_surface_ptr->get_data(); + GAPI_LOG_DEBUG(nullptr, "surface: " << parent_surface_ptr->get_handle() << + ", w: " << info.Width << ", h: " << info.Height << + ", p: " << data.Pitch); + switch(info.FourCC) + { + case MFX_FOURCC_I420: + throw std::runtime_error("MediaFrame doesn't support I420 type"); + break; + case MFX_FOURCC_NV12: + frame_desc.fmt = MediaFormat::NV12; + break; + default: + throw std::runtime_error("MediaFrame unknown 'fmt' type: " + std::to_string(info.FourCC)); + } + frame_desc.size = cv::Size{info.Width, info.Height}; + + LockAdapter* alloc_data = reinterpret_cast(data.MemId); + alloc_data->set_adaptee(this); + + parent_surface_ptr->obtain_lock(); +} + +VPLMediaFrameDX11Adapter::~VPLMediaFrameDX11Adapter() { + // Each VPLMediaFrameDX11Adapter releases mfx surface counter + // The last VPLMediaFrameDX11Adapter releases shared Surface pointer + // The last surface pointer releases workspace memory + Surface::data_t& data = parent_surface_ptr->get_data(); + LockAdapter* alloc_data = reinterpret_cast(data.MemId); + alloc_data->set_adaptee(nullptr); + + parent_surface_ptr->release_lock(); +} + +cv::GFrameDesc VPLMediaFrameDX11Adapter::meta() const { + return frame_desc; +} + +MediaFrame::View VPLMediaFrameDX11Adapter::access(MediaFrame::Access mode) { + Surface::data_t& data = parent_surface_ptr->get_data(); + const Surface::info_t& info = parent_surface_ptr->get_info(); + void* frame_id = reinterpret_cast(this); + + GAPI_LOG_DEBUG(nullptr, "START lock frame in surface: " << parent_surface_ptr->get_handle() << + ", frame id: " << frame_id); + + // lock MT + lock_mid(data.MemId, data, mode); + + GAPI_LOG_DEBUG(nullptr, "FINISH lock frame in surface: " << parent_surface_ptr->get_handle() << + ", frame id: " << frame_id); + using stride_t = typename cv::MediaFrame::View::Strides::value_type; + stride_t pitch = static_cast(data.Pitch); + + // NB: make copy for some copyable object, because access release may be happened + // after source/pool destruction, so we need a copy + auto parent_surface_ptr_copy = parent_surface_ptr; + switch(info.FourCC) { + case MFX_FOURCC_I420: + { + GAPI_Assert(data.Y && data.U && data.V && "MFX_FOURCC_I420 frame data is nullptr"); + cv::MediaFrame::View::Ptrs pp = { data.Y, data.U, data.V, nullptr }; + cv::MediaFrame::View::Strides ss = { pitch, pitch / 2, pitch / 2, 0u }; + return cv::MediaFrame::View(std::move(pp), std::move(ss), + [parent_surface_ptr_copy, + frame_id, mode] () { + parent_surface_ptr_copy->obtain_lock(); + + auto& data = parent_surface_ptr_copy->get_data(); + GAPI_LOG_DEBUG(nullptr, "START unlock frame in surface: " << parent_surface_ptr_copy->get_handle() << + ", frame id: " << frame_id); + unlock_mid(data.MemId, data, mode); + + GAPI_LOG_DEBUG(nullptr, "FINISH unlock frame in surface: " << parent_surface_ptr_copy->get_handle() << + ", frame id: " << frame_id); + + parent_surface_ptr_copy->release_lock(); + }); + } + case MFX_FOURCC_NV12: + { + if (!data.Y || !data.UV) { + GAPI_LOG_WARNING(nullptr, "Empty data detected!!! for surface: " << parent_surface_ptr->get_handle() << + ", frame id: " << frame_id); + } + GAPI_Assert(data.Y && data.UV && "MFX_FOURCC_NV12 frame data is nullptr"); + cv::MediaFrame::View::Ptrs pp = { data.Y, data.UV, nullptr, nullptr }; + cv::MediaFrame::View::Strides ss = { pitch, pitch, 0u, 0u }; + return cv::MediaFrame::View(std::move(pp), std::move(ss), + [parent_surface_ptr_copy, + frame_id, mode] () { + parent_surface_ptr_copy->obtain_lock(); + + auto& data = parent_surface_ptr_copy->get_data(); + GAPI_LOG_DEBUG(nullptr, "START unlock frame in surface: " << parent_surface_ptr_copy->get_handle() << + ", frame id: " << frame_id); + unlock_mid(data.MemId, data, mode); + + GAPI_LOG_DEBUG(nullptr, "FINISH unlock frame in surface: " << parent_surface_ptr_copy->get_handle() << + ", frame id: " << frame_id); + parent_surface_ptr_copy->release_lock(); + }); + } + break; + default: + throw std::runtime_error("MediaFrame unknown 'fmt' type: " + std::to_string(info.FourCC)); + } +} + +cv::util::any VPLMediaFrameDX11Adapter::blobParams() const { +#ifdef HAVE_INF_ENGINE + GAPI_Assert(false && "VPLMediaFrameDX11Adapter::blobParams() is not fully operable " + "in G-API streaming. Please waiting for future PRs"); + + Surface::data_t& data = parent_surface_ptr->get_data(); + NativeHandleAdapter* native_handle_getter = reinterpret_cast(data.MemId); + + mfxHDLPair handle{}; + native_handle_getter->get_handle(data.MemId, reinterpret_cast(handle)); + + InferenceEngine::ParamMap params{{"SHARED_MEM_TYPE", "VA_SURFACE"}, + {"DEV_OBJECT_HANDLE", handle.first}, + {"COLOR_FORMAT", InferenceEngine::ColorFormat::NV12}, + {"VA_PLANE", + static_cast( + reinterpret_cast( + reinterpret_cast( + handle.second)))}};//, + const Surface::info_t& info = parent_surface_ptr->get_info(); + InferenceEngine::TensorDesc tdesc({InferenceEngine::Precision::U8, + {1, 3, static_cast(info.Height), + static_cast(info.Width)}, + InferenceEngine::Layout::NCHW}); + return std::make_pair(tdesc, params); +#else + GAPI_Assert(false && "VPLMediaFrameDX11Adapter::blobParams() is not implemented"); +#endif // HAVE_INF_ENGINE +} + +void VPLMediaFrameDX11Adapter::serialize(cv::gapi::s11n::IOStream&) { + GAPI_Assert(false && "VPLMediaFrameDX11Adapter::serialize() is not implemented"); +} + +void VPLMediaFrameDX11Adapter::deserialize(cv::gapi::s11n::IIStream&) { + GAPI_Assert(false && "VPLMediaFrameDX11Adapter::deserialize() is not implemented"); +} + +DXGI_FORMAT VPLMediaFrameDX11Adapter::get_dx11_color_format(uint32_t mfx_fourcc) { + switch (mfx_fourcc) { + case MFX_FOURCC_NV12: + return DXGI_FORMAT_NV12; + + case MFX_FOURCC_YUY2: + return DXGI_FORMAT_YUY2; + + case MFX_FOURCC_RGB4: + return DXGI_FORMAT_B8G8R8A8_UNORM; + + case MFX_FOURCC_P8: + case MFX_FOURCC_P8_TEXTURE: + return DXGI_FORMAT_P8; + + case MFX_FOURCC_ARGB16: + case MFX_FOURCC_ABGR16: + return DXGI_FORMAT_R16G16B16A16_UNORM; + + case MFX_FOURCC_P010: + return DXGI_FORMAT_P010; + + case MFX_FOURCC_A2RGB10: + return DXGI_FORMAT_R10G10B10A2_UNORM; + + case DXGI_FORMAT_AYUV: + case MFX_FOURCC_AYUV: + return DXGI_FORMAT_AYUV; + + default: + return DXGI_FORMAT_UNKNOWN; + } +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/dx11_frame_adapter.hpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/dx11_frame_adapter.hpp new file mode 100644 index 0000000000..ca6602353b --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/dx11_frame_adapter.hpp @@ -0,0 +1,63 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_DX11_FRAME_ADAPTER_HPP +#define GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_DX11_FRAME_ADAPTER_HPP +#include + +#include +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +#include "streaming/onevpl/accelerators/utils/shared_lock.hpp" +#ifdef HAVE_ONEVPL +#include "streaming/onevpl/onevpl_export.hpp" + +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 + #define D3D11_NO_HELPERS + #define NOMINMAX + #include + #include + #include "opencv2/core/directx.hpp" + #ifdef HAVE_OPENCL + #include + #endif + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +class Surface; +class VPLMediaFrameDX11Adapter final: public cv::MediaFrame::IAdapter, + public SharedLock { +public: + // GAPI_EXPORTS for tests + GAPI_EXPORTS VPLMediaFrameDX11Adapter(std::shared_ptr assoc_surface); + GAPI_EXPORTS ~VPLMediaFrameDX11Adapter(); + cv::GFrameDesc meta() const override; + MediaFrame::View access(MediaFrame::Access) override; + + // The default implementation does nothing + cv::util::any blobParams() const override; + void serialize(cv::gapi::s11n::IOStream&) override; + void deserialize(cv::gapi::s11n::IIStream&) override; + + static DXGI_FORMAT get_dx11_color_format(uint32_t mfx_fourcc); +private: + std::shared_ptr parent_surface_ptr; + mfxFrameAllocator allocator; + GFrameDesc frame_desc; +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#undef NOMINMAX +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_DX11_FRAME_ADAPTER_HPP diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp index 3f5fd00305..c09dc80338 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp @@ -20,18 +20,20 @@ Surface::Surface(std::unique_ptr&& surf, std::shared_ptr associa mirrored_locked_count() { GAPI_Assert(mfx_surface && "Surface is nullptr"); - mirrored_locked_count.store(mfx_surface->Data.Locked); GAPI_LOG_DEBUG(nullptr, "create surface: " << mfx_surface << ", locked count: " << mfx_surface->Data.Locked); } Surface::~Surface() { GAPI_LOG_DEBUG(nullptr, "destroy surface: " << mfx_surface << - ", worspace memory counter: " << workspace_memory_ptr.use_count()); + ", worspace memory counter: " << + workspace_memory_ptr.use_count()); } std::shared_ptr Surface::create_surface(std::unique_ptr&& surf, std::shared_ptr accociated_memory) { + Surface::info_t& info = surf->Info; + info.FourCC = MFX_FOURCC_NV12; surface_ptr_t ret {new Surface(std::move(surf), accociated_memory)}; return ret; } @@ -48,14 +50,16 @@ const Surface::data_t& Surface::get_data() const { return mfx_surface->Data; } +Surface::data_t& Surface::get_data() { + return const_cast(static_cast(this)->get_data()); +} + size_t Surface::get_locks_count() const { - return mirrored_locked_count.load(); + return mirrored_locked_count.load() + mfx_surface->Data.Locked; } size_t Surface::obtain_lock() { size_t locked_count = mirrored_locked_count.fetch_add(1); - GAPI_Assert(locked_count < std::numeric_limits::max() && "Too many references "); - mfx_surface->Data.Locked = static_cast(locked_count + 1); GAPI_LOG_DEBUG(nullptr, "surface: " << mfx_surface.get() << ", locked times: " << locked_count + 1); return locked_count; // return preceding value @@ -63,9 +67,7 @@ size_t Surface::obtain_lock() { size_t Surface::release_lock() { size_t locked_count = mirrored_locked_count.fetch_sub(1); - GAPI_Assert(locked_count < std::numeric_limits::max() && "Too many references "); GAPI_Assert(locked_count && "Surface lock counter is invalid"); - mfx_surface->Data.Locked = static_cast(locked_count - 1); GAPI_LOG_DEBUG(nullptr, "surface: " << mfx_surface.get() << ", locked times: " << locked_count - 1); return locked_count; // return preceding value diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp index 828e5cb1c7..4f93312e24 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp @@ -13,12 +13,7 @@ #include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS #ifdef HAVE_ONEVPL -#if (MFX_VERSION >= 2000) -#include -#endif - -#include - +#include "streaming/onevpl/onevpl_export.hpp" namespace cv { namespace gapi { @@ -45,24 +40,21 @@ namespace onevpl { * - @ref Surface::obtain_lock() against @ref Surface::release_lock() * - @ref Surface::release_lock() against @ref Surface::release_lock() */ -class Surface { - using handle_t = mfxFrameSurface1; - - std::shared_ptr workspace_memory_ptr; - std::unique_ptr mfx_surface; - std::atomic mirrored_locked_count; +class GAPI_EXPORTS Surface final { // GAPI_EXPORTS for tests public: + using handle_t = mfxFrameSurface1; using info_t = mfxFrameInfo; using data_t = mfxFrameData; - // GAPI_EXPORTS for tests - GAPI_EXPORTS static std::shared_ptr create_surface(std::unique_ptr&& surf, - std::shared_ptr accociated_memory); - GAPI_EXPORTS ~Surface(); - GAPI_EXPORTS handle_t* get_handle() const; - GAPI_EXPORTS const info_t& get_info() const; - GAPI_EXPORTS const data_t& get_data() const; + static std::shared_ptr create_surface(std::unique_ptr&& surf, + std::shared_ptr accociated_memory); + ~Surface(); + + handle_t* get_handle() const; + const info_t& get_info() const; + const data_t& get_data() const; + data_t& get_data(); /** * Extract value thread-safe lock counter (see @ref Surface description). @@ -71,7 +63,7 @@ public: * * @return fetched locks count. */ - GAPI_EXPORTS size_t get_locks_count() const; + size_t get_locks_count() const; /** * Atomically increase value of thread-safe lock counter (see @ref Surface description). @@ -80,7 +72,7 @@ public: * * @return locks count just before its increasing. */ - GAPI_EXPORTS size_t obtain_lock(); + size_t obtain_lock(); /** * Atomically decrease value of thread-safe lock counter (see @ref Surface description). @@ -89,9 +81,13 @@ public: * * @return locks count just before its decreasing. */ - GAPI_EXPORTS size_t release_lock(); + size_t release_lock(); private: Surface(std::unique_ptr&& surf, std::shared_ptr accociated_memory); + + std::shared_ptr workspace_memory_ptr; + std::unique_ptr mfx_surface; + std::atomic mirrored_locked_count; }; using surface_ptr_t = std::shared_ptr; diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.hpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.hpp index a1b8d192bc..a5084a81ea 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.hpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.hpp @@ -8,11 +8,7 @@ #include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS #ifdef HAVE_ONEVPL -#if (MFX_VERSION >= 2000) -#include -#endif - -#include +#include "streaming/onevpl/onevpl_export.hpp" namespace cv { namespace gapi { diff --git a/modules/gapi/src/streaming/onevpl/accelerators/utils/elastic_barrier.hpp b/modules/gapi/src/streaming/onevpl/accelerators/utils/elastic_barrier.hpp new file mode 100644 index 0000000000..827392f8be --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/utils/elastic_barrier.hpp @@ -0,0 +1,296 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ACCELERATORS_UTILS_ELASTIC_BARRIER_HPP +#define GAPI_STREAMING_ONEVPL_ACCELERATORS_UTILS_ELASTIC_BARRIER_HPP +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +template +class elastic_barrier { +public: + using self_t = Impl; + elastic_barrier() : + incoming_requests(), + outgoing_requests(), + pending_requests(), + reinit(false) { + } + + self_t* get_self() { + return static_cast(this); + } + + template + void visit_in (Args&& ...args) { + on_lock(std::forward(args)...); + } + + template + void visit_out (Args&& ...args) { + on_unlock(std::forward(args)...); + } + +protected: + ~elastic_barrier() = default; + +private: + std::atomic incoming_requests; + std::atomic outgoing_requests; + std::atomic pending_requests; + std::atomic reinit; + + template + void on_first_in(Args&& ...args) { + get_self()->on_first_in_impl(std::forward(args)...); + } + + template + void on_last_out(Args&& ...args) { + get_self()->on_last_out_impl(std::forward(args)...); + } + + template + void on_lock(Args&& ...args) { + // Read access is more complex + // each `incoming` request must check in before acquire resource + size_t thread_id = incoming_requests.fetch_add(1); + if (thread_id == 0) { + /* + * only one `incoming` request is allowable to init resource + * at first time + * let's filter out the first one by `thread_id` + * + * The first one `incoming` request becomes main `incoming` request + * */ + if (outgoing_requests.load() == 0) { + get_self()->on_first_in(std::forward(args)...); + /* + * The main `incoming` request finished resource initialization + * and became `outgoing` + * + * Non empty `outgoing` count means that + * other further `incoming` (or busy-wait) requests + * are getting on with its job without resource initialization, + * because main `incoming` request has already initialized it at here + * */ + outgoing_requests.fetch_add(1); + return; + } + return; + } else { + /* + * CASE 1) + * + * busy wait for others `incoming` requests for resource initialization + * besides main `incoming` request which are getting on + * resource initialization at this point + * + * */ + + // OR + + /* + * CASE 2) + * + * busy wait for ALL `incoming` request for resource initialization + * including main `incoming` request. It will happen if + * new `incoming` requests had came here while resource was getting on deinit + * in `on_unlock` in another processing thread. + * In this case no actual main `incoming` request is available and + * all `incoming` requests must be in busy-wait stare + * + * */ + + // Each `incoming` request became `busy-wait` request + size_t busy_thread_id = pending_requests.fetch_add(1); + + /* + * CASE 1) + * + * Non empty `outgoing` requests count means that other further `incoming` or + * `busy-wait` request are getting on with its job + * without resource initialization because + * main thread has already initialized it at here + * */ + while (outgoing_requests.load() == 0) { + + // OR + + /* + * CASE 2) + * + * In case of NO master `incoming `request is available and doesn't + * provide resource initialization. All `incoming` requests must be in + * busy-wait state. + * If it is not true then CASE 1) is going on + * + * OR + * + * `on_unlock` is in deinitialization phase in another thread. + * Both cases mean busy-wait state here + * */ + if (pending_requests.load() == incoming_requests.load()) { + /* + * CASE 2) ONLY + * + * It will happen if 'on_unlock` in another thread + * finishes its execution only + * + * `on_unlock` in another thread might finished with either + * deinitialization action or without deinitialization action + * (the call off deinitialization case) + * + * We must not continue at here (without reinit) + * if deinitialization happens in `on_unlock` in another thread. + * So try it on + * */ + + // only single `busy-wait` request must make sure about possible + // deinitialization. So first `busy-wait` request becomes + // main `busy-wait` request + if (busy_thread_id == 0) { + bool expected_reinit = true; + if (!reinit.compare_exchange_strong(expected_reinit, false)) { + /* + * deinitialization called off in `on_unlock` + * because new `incoming` request had appeared at here before + * `on_unlock` started deinit procedure in another thread. + * So no reinit required because no deinit had happended + * + * main `busy-wait` request must break busy-wait state + * and become `outgoing` request. + * Non empty `outgoing` count means that other + * further `incoming` requests or + * `busy-wait` requests are getting on with its job + * without resource initialization/reinitialization + * because no deinit happened in `on_unlock` + * in another thread + * */ + break; //just quit busy loop + } else { + /* Deinitialization had happened in `on_unlock` + * in another thread right before + * new `incoming` requests appeared. + * So main `busy-wait` request must start reinit procedure + */ + get_self()->on_first_in(std::forward(args)...); + + /* + * Main `busy-wait` request has finished reinit procedure + * and becomes `outgong` request. + * Non empty `outgoing` count means that other + * further `incoming` requests or + * `busy-wait` requests are getting on with its job + * without resource initialization because + * main `busy-wait` request + * has already re-initialized it at here + */ + outgoing_requests.fetch_add(1); + pending_requests.fetch_sub(1); + return; + } + } + } + } + + // All non main requests became `outgoing` and look at on initialized resource + outgoing_requests++; + + // Each `busy-wait` request are not busy-wait now + pending_requests.fetch_sub(1); + } + return; + } + + template + void on_unlock(Args&& ...args) { + // Read unlock + /* + * Each released `outgoing` request checks out to doesn't use resource anymore. + * The last `outgoing` request becomes main `outgoing` request and + * must deinitialize resource if no `incoming` or `busy-wait` requests + * are waiting for it + */ + size_t thread_id = outgoing_requests.fetch_sub(1); + if (thread_id == 1) { + /* + * Make sure that no another `incoming` (including `busy-wait) + * exists. + * But beforehand its must make sure that no `incoming` or `pending` + * requests are exist. + * + * The main `outgoing` request is an one of `incoming` request + * (it is the oldest one in the current `incoming` bunch) and still + * holds resource in initialized state (thus we compare with 1). + * We must not deinitialize resource before decrease + * `incoming` requests counter because + * after it has got 0 value in `on_lock` another thread + * will start initialize resource procedure which will get conflict + * with current deinitialize procedure + * + * From this point, all `on_lock` request in another thread would + * become `busy-wait` without reaching main `incoming` state (CASE 2) + * */ + if (incoming_requests.load() == 1) { + /* + * The main `outgoing` request is ready to deinit shared resource + * in unconflicting manner. + * + * This is a critical section for single thread for main `outgoing` + * request + * + * CASE 2 only available in `on_lock` thread + * */ + get_self()->on_last_out(std::forward(args)...); + + /* + * Before main `outgoinq` request become released it must notify + * subsequent `busy-wait` requests in `on_lock` in another thread + * that main `busy-wait` must start reinit resource procedure + * */ + reinit.store(true); + + /* + * Deinitialize procedure is finished and main `outgoing` request + * (it is the oldest one in `incoming` request) must become released + * + * Right after when we decrease `incoming` counter + * the condition for equality + * `busy-wait` and `incoming` counter will become true (CASE 2 only) + * in `on_lock` in another threads. After that + * a main `busy-wait` request would check `reinit` condition + * */ + incoming_requests.fetch_sub(1); + return; + } + + /* + * At this point we have guarantee that new `incoming` requests + * had became increased in `on_lock` in another thread right before + * current thread deinitialize resource. + * + * So call off deinitialization procedure here + * */ + } + incoming_requests.fetch_sub(1); + } + + elastic_barrier(const elastic_barrier&) = delete; + elastic_barrier(elastic_barrier&&) = delete; + elastic_barrier& operator() (const elastic_barrier&) = delete; + elastic_barrier& operator() (elastic_barrier&&) = delete; +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // GAPI_STREAMING_ONEVPL_ACCELERATORS_UTILS_ELASTIC_BARRIER_HPP diff --git a/modules/gapi/src/streaming/onevpl/accelerators/utils/shared_lock.cpp b/modules/gapi/src/streaming/onevpl/accelerators/utils/shared_lock.cpp new file mode 100644 index 0000000000..ab3c48afd1 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/utils/shared_lock.cpp @@ -0,0 +1,95 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include +#include "streaming/onevpl/accelerators/utils/shared_lock.hpp" + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +SharedLock::SharedLock() { + exclusive_lock.store(false); + shared_counter.store(0); +} + +size_t SharedLock::shared_lock() { + size_t prev = 0; + bool in_progress = false; + bool pred_excl = exclusive_lock.load(); + do { + if (!pred_excl) { + // if no exclusive lock then start shared lock transaction + prev = shared_counter.fetch_add(1); + in_progress = true; // transaction is in progress + } else { + if (in_progress) { + in_progress = false; + shared_counter.fetch_sub(1); + } + std::this_thread::yield(); + } + + // test if exclusive lock happened before + pred_excl = exclusive_lock.load(); + } while (pred_excl || !in_progress); + + return prev; +} + +size_t SharedLock::unlock_shared() { + return shared_counter.fetch_sub(1); +} + +void SharedLock::lock() { + bool in_progress = false; + size_t prev_shared = shared_counter.load(); + do { + if (prev_shared == 0) { + bool expected = false; + while (!exclusive_lock.compare_exchange_strong(expected, true)) { + expected = false; + std::this_thread::yield(); + } + in_progress = true; + } else { + if (in_progress) { + in_progress = false; + exclusive_lock.store(false); + } + std::this_thread::yield(); + } + prev_shared = shared_counter.load(); + } while (prev_shared != 0 || !in_progress); +} + +bool SharedLock::try_lock() { + if (shared_counter.load() != 0) { + return false; + } + + bool expected = false; + if (exclusive_lock.compare_exchange_strong(expected, true)) { + if (shared_counter.load() == 0) { + return true; + } else { + exclusive_lock.store(false); + } + } + return false; +} + +void SharedLock::unlock() { + exclusive_lock.store(false); +} +bool SharedLock::owns() const { + return exclusive_lock.load(); +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/accelerators/utils/shared_lock.hpp b/modules/gapi/src/streaming/onevpl/accelerators/utils/shared_lock.hpp new file mode 100644 index 0000000000..704a474761 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/utils/shared_lock.hpp @@ -0,0 +1,47 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_SHARED_LOCK_HPP +#define GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_SHARED_LOCK_HPP + +#include +#include + +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +class GAPI_EXPORTS SharedLock { +public: + SharedLock(); + ~SharedLock() = default; + + size_t shared_lock(); + size_t unlock_shared(); + + void lock(); + bool try_lock(); + void unlock(); + + bool owns() const; +private: + SharedLock(const SharedLock&) = delete; + SharedLock& operator= (const SharedLock&) = delete; + SharedLock(SharedLock&&) = delete; + SharedLock& operator== (SharedLock&&) = delete; + + std::atomic exclusive_lock; + std::atomic shared_counter; +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_SHARED_LOCK_HPP diff --git a/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp b/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp index 64dc34329b..a4d85f2598 100644 --- a/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp +++ b/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp @@ -5,7 +5,7 @@ // Copyright (C) 2021 Intel Corporation #ifdef HAVE_ONEVPL -#include +#include "streaming/onevpl/onevpl_export.hpp" #include #include @@ -44,7 +44,7 @@ CfgParamDeviceSelector::CfgParamDeviceSelector(const CfgParams& cfg_params) : auto accel_mode_it = std::find_if(cfg_params.begin(), cfg_params.end(), [] (const CfgParam& value) { - return value.get_name() == "mfxImplDescription.AccelerationMode"; + return value.get_name() == CfgParam::acceleration_mode_name(); }); if (accel_mode_it == cfg_params.end()) { @@ -128,9 +128,11 @@ CfgParamDeviceSelector::CfgParamDeviceSelector(const CfgParams& cfg_params) : suggested_device = IDeviceSelector::create(hw_handle, "GPU", AccelType::DX11); suggested_context = IDeviceSelector::create(device_context, AccelType::DX11); #else - GAPI_LOG_WARNING(nullptr, "Unavailable \"mfxImplDescription.AccelerationMode: MFX_ACCEL_MODE_VIA_D3D11\"" + GAPI_LOG_WARNING(nullptr, "Unavailable \"" << CfgParam::acceleration_mode_name() << ": MFX_ACCEL_MODE_VIA_D3D11\"" "was chosen for current project configuration"); - throw std::logic_error("Unsupported \"mfxImplDescription.AccelerationMode: MFX_ACCEL_MODE_VIA_D3D11\""); + throw std::logic_error(std::string("Unsupported \"") + + CfgParam::acceleration_mode_name() + + ": MFX_ACCEL_MODE_VIA_D3D11\""); #endif // HAVE_DIRECTX #endif // HAVE_D3D11 break; @@ -140,7 +142,8 @@ CfgParamDeviceSelector::CfgParamDeviceSelector(const CfgParams& cfg_params) : break; } default: - throw std::logic_error("Unsupported \"mfxImplDescription.AccelerationMode\" requested: " + + throw std::logic_error(std::string("Unsupported \"") + + CfgParam::acceleration_mode_name() +"\" requested: " + std::to_string(accel_mode.Data.U32)); break; } @@ -154,14 +157,16 @@ CfgParamDeviceSelector::CfgParamDeviceSelector(Device::Ptr device_ptr, suggested_context(IDeviceSelector::create(nullptr, AccelType::HOST)) { auto accel_mode_it = std::find_if(cfg_params.begin(), cfg_params.end(), [] (const CfgParam& value) { - return value.get_name() == "mfxImplDescription.AccelerationMode"; + return value.get_name() == CfgParam::acceleration_mode_name(); }); if (accel_mode_it == cfg_params.end()) { GAPI_LOG_WARNING(nullptr, "Cannot deternime \"device_ptr\" type. " - "Make sure a param \"mfxImplDescription.AccelerationMode\" " + "Make sure a param \"" << CfgParam::acceleration_mode_name() << "\" " "presents in configurations and has correct value according to " "\"device_ptr\" type"); - throw std::logic_error("Missing \"mfxImplDescription.AccelerationMode\" param"); + throw std::logic_error(std::string("Missing \"") + + CfgParam::acceleration_mode_name() + + "\" param"); } GAPI_LOG_DEBUG(nullptr, "Turn on HW acceleration support for device: " << @@ -169,7 +174,7 @@ CfgParamDeviceSelector::CfgParamDeviceSelector(Device::Ptr device_ptr, ", context: " << ctx_ptr); if (!device_ptr) { GAPI_LOG_WARNING(nullptr, "Empty \"device_ptr\" is not allowed when " - "param \"mfxImplDescription.AccelerationMode\" existed"); + "param \"" << CfgParam::acceleration_mode_name() << "\" existed"); throw std::logic_error("Invalid param: \"device_ptr\""); } @@ -191,23 +196,36 @@ CfgParamDeviceSelector::CfgParamDeviceSelector(Device::Ptr device_ptr, suggested_context = IDeviceSelector::create(ctx_ptr, AccelType::DX11); ID3D11DeviceContext* dx_ctx_ptr = reinterpret_cast(suggested_context.get_ptr()); + + // oneVPL recommendation + { + ID3D11Multithread *pD11Multithread = nullptr; + dx_ctx_ptr->QueryInterface(IID_PPV_ARGS(&pD11Multithread)); + pD11Multithread->SetMultithreadProtected(true); + pD11Multithread->Release(); + } + dx_ctx_ptr->AddRef(); #else - GAPI_LOG_WARNING(nullptr, "Unavailable \"mfxImplDescription.AccelerationMode: MFX_ACCEL_MODE_VIA_D3D11\"" + GAPI_LOG_WARNING(nullptr, "Unavailable \"" << CfgParam::acceleration_mode_name() << + ": MFX_ACCEL_MODE_VIA_D3D11\"" "was chosen for current project configuration"); - throw std::logic_error("Unsupported \"mfxImplDescription.AccelerationMode: MFX_ACCEL_MODE_VIA_D3D11\""); + throw std::logic_error(std::string("Unsupported \"") + + CfgParam::acceleration_mode_name() + ": MFX_ACCEL_MODE_VIA_D3D11\""); #endif // HAVE_DIRECTX #endif // HAVE_D3D11 break; } case MFX_ACCEL_MODE_NA: { - GAPI_LOG_WARNING(nullptr, "Incompatible \"mfxImplDescription.AccelerationMode: MFX_ACCEL_MODE_NA\" with " + GAPI_LOG_WARNING(nullptr, "Incompatible \"" << CfgParam::acceleration_mode_name() << + ": MFX_ACCEL_MODE_NA\" with " "\"device_ptr\" and \"ctx_ptr\" arguments. " "You should not clarify these arguments with \"MFX_ACCEL_MODE_NA\" mode"); throw std::logic_error("Incompatible param: MFX_ACCEL_MODE_NA"); } default: - throw std::logic_error("Unsupported \"mfxImplDescription.AccelerationMode\" requested: " + + throw std::logic_error(std::string("Unsupported \"") + CfgParam::acceleration_mode_name() + + "\" requested: " + std::to_string(accel_mode.Data.U32)); break; } diff --git a/modules/gapi/src/streaming/onevpl/cfg_params.cpp b/modules/gapi/src/streaming/onevpl/cfg_params.cpp index 456725508f..97458faed3 100644 --- a/modules/gapi/src/streaming/onevpl/cfg_params.cpp +++ b/modules/gapi/src/streaming/onevpl/cfg_params.cpp @@ -86,6 +86,34 @@ CfgParam::CfgParam (const std::string& param_name, value_t&& param_value, bool i CfgParam::~CfgParam() = default; +CfgParam CfgParam::create_frames_pool_size(uint64_t value) { + return CfgParam::create(CfgParam::frames_pool_size_name(), value, false); +} + +CfgParam CfgParam::create_acceleration_mode(uint32_t value) { + return CfgParam::create(CfgParam::acceleration_mode_name(), value); +} + +CfgParam CfgParam::create_acceleration_mode(const char* value) { + return CfgParam::create(CfgParam::acceleration_mode_name(), std::string(value)); +} + +CfgParam CfgParam::create_decoder_id(uint32_t value) { + return CfgParam::create(CfgParam::decoder_id_name(), value); +} + +CfgParam CfgParam::create_decoder_id(const char* value) { + return CfgParam::create(CfgParam::decoder_id_name(), std::string(value)); +} + +CfgParam CfgParam::create_implementation(uint32_t value) { + return CfgParam::create(CfgParam::implementation_name(), value); +} + +CfgParam CfgParam::create_implementation(const char* value) { + return CfgParam::create(CfgParam::implementation_name(), std::string(value)); +} + CfgParam& CfgParam::operator=(const CfgParam& src) { if (this != &src) { m_priv = src.m_priv; diff --git a/modules/gapi/src/streaming/onevpl/cfg_params_parser.cpp b/modules/gapi/src/streaming/onevpl/cfg_params_parser.cpp index a683c7478e..07c639faa2 100644 --- a/modules/gapi/src/streaming/onevpl/cfg_params_parser.cpp +++ b/modules/gapi/src/streaming/onevpl/cfg_params_parser.cpp @@ -22,19 +22,16 @@ namespace onevpl { template <> struct ParamCreator { template - CfgParam create (const std::string& name, ValueType&& value) { + CfgParam create (const std::string& name, ValueType&& value, bool is_major_flag = false) { return CfgParam::create(name, std::forward(value), is_major_flag); } - bool is_major_flag = false; }; template <> struct ParamCreator { template - mfxVariant create (const std::string& name, ValueType&& value) { - static_assert(std::is_same::type, mfxU32>::value, - "ParamCreator supports mfxU32 at the moment. " - "Feel free to extend for more types"); + mfxVariant create (const std::string& name, ValueType&& value, bool is_major_flag = false) { + cv::util::suppress_unused_warning(is_major_flag); return create_impl(name, value); } private: @@ -44,6 +41,18 @@ private: ret.Data.U32 = value; return ret; } + mfxVariant create_impl(const std::string&, mfxI64 value) { + mfxVariant ret; + ret.Type = MFX_VARIANT_TYPE_I64; + ret.Data.I64 = value; + return ret; + } + mfxVariant create_impl(const std::string&, mfxU64 value) { + mfxVariant ret; + ret.Type = MFX_VARIANT_TYPE_U64; + ret.Data.U64 = value; + return ret; + } }; template @@ -67,14 +76,16 @@ std::vector get_params_from_string(const std::string& str) { std::string value = line.substr(name_endline_pos + 2); ParamCreator creator; - if (name == "mfxImplDescription.Impl") { + if (name == CfgParam::implementation_name()) { ret.push_back(creator.create(name, cstr_to_mfx_impl(value.c_str()))); - } else if (name == "mfxImplDescription.mfxDecoderDescription.decoder.CodecID") { + } else if (name == CfgParam::decoder_id_name()) { ret.push_back(creator.create(name, cstr_to_mfx_codec_id(value.c_str()))); - } else if (name == "mfxImplDescription.AccelerationMode") { + } else if (name == CfgParam::acceleration_mode_name()) { ret.push_back(creator.create(name, cstr_to_mfx_accel_mode(value.c_str()))); } else if (name == "mfxImplDescription.ApiVersion.Version") { ret.push_back(creator.create(name, cstr_to_mfx_version(value.c_str()))); + } else if (name == CfgParam::frames_pool_size_name()) { + ret.push_back(creator.create(name, strtoull_or_throw(value.c_str()), false)); } else { GAPI_LOG_DEBUG(nullptr, "Cannot parse configuration param, name: " << name << ", value: " << value); @@ -116,6 +127,32 @@ mfxVariant cfg_param_to_mfx_variant(const CfgParam& cfg_val) { }), cfg_val.get_value()); return ret; } + +size_t strtoull_or_throw(const char* str) { + char *end_ptr = nullptr; + errno = 0; + size_t ret = strtoull(str, &end_ptr, 10); + if ((end_ptr == str) || + ((ret == LONG_MAX || ret == LONG_MIN) && errno == ERANGE)) { + // nothing parsed from the string, handle errors or exit + GAPI_LOG_WARNING(nullptr, "strtoull failed for: " << str); + GAPI_Assert(false && "strtoull_or_throw"); + } + return ret; +} + +int64_t strtoll_or_throw(const char* str) { + char *end_ptr = nullptr; + errno = 0; + int64_t ret = strtoll(str, &end_ptr, 10); + if ((end_ptr == str) || + ((ret == LONG_MAX || ret == LONG_MIN) && errno == ERANGE)) { + // nothing parsed from the string, handle errors or exit + GAPI_LOG_WARNING(nullptr, "strtoll failed for: " << str); + GAPI_Assert(false && "strtoll_or_throw"); + } + return ret; +} } // namespace onevpl } // namespace wip } // namespace gapi diff --git a/modules/gapi/src/streaming/onevpl/cfg_params_parser.hpp b/modules/gapi/src/streaming/onevpl/cfg_params_parser.hpp index 6247eef916..c5e7685756 100644 --- a/modules/gapi/src/streaming/onevpl/cfg_params_parser.hpp +++ b/modules/gapi/src/streaming/onevpl/cfg_params_parser.hpp @@ -8,12 +8,7 @@ #define GAPI_STREAMING_ONEVPL_CFG_PARAM_PARSER_HPP #ifdef HAVE_ONEVPL -#if (MFX_VERSION >= 2000) -#include -#endif // MFX_VERSION - -#include -#include +#include "streaming/onevpl/onevpl_export.hpp" #include #include @@ -31,10 +26,14 @@ std::vector get_params_from_string(const std::string& str); template struct ParamCreator { template - ReturnType create(const std::string& name, ValueType&& value); + ReturnType create(const std::string& name, ValueType&& value, bool is_major = false); }; mfxVariant cfg_param_to_mfx_variant(const CfgParam& value); + +size_t strtoull_or_throw(const char* str); +int64_t strtoll_or_throw(const char* str); + } // namespace onevpl } // namespace wip } // namespace gapi diff --git a/modules/gapi/src/streaming/onevpl/data_provider_defines.hpp b/modules/gapi/src/streaming/onevpl/data_provider_defines.hpp index d31bece9fe..5a4c66fef2 100644 --- a/modules/gapi/src/streaming/onevpl/data_provider_defines.hpp +++ b/modules/gapi/src/streaming/onevpl/data_provider_defines.hpp @@ -2,8 +2,7 @@ #define GAPI_STREAMING_ONEVPL_DATA_PROVIDER_DEFINES_HPP #ifdef HAVE_ONEVPL -#include -#include +#include "streaming/onevpl/onevpl_export.hpp" #endif // HAVE_ONEVPL #include diff --git a/modules/gapi/src/streaming/onevpl/data_provider_dispatcher.cpp b/modules/gapi/src/streaming/onevpl/data_provider_dispatcher.cpp index bc712eb1a1..e900fad7c6 100644 --- a/modules/gapi/src/streaming/onevpl/data_provider_dispatcher.cpp +++ b/modules/gapi/src/streaming/onevpl/data_provider_dispatcher.cpp @@ -25,13 +25,14 @@ IDataProvider::Ptr DataProviderDispatcher::create(const std::string& file_path, // Look-up CodecId from input params // If set then raw data provider is preferred - GAPI_LOG_DEBUG(nullptr, "try find explicit cfg param\"mfxImplDescription.mfxDecoderDescription.decoder.CodecID\""); + GAPI_LOG_DEBUG(nullptr, "try find explicit cfg param \"" << + CfgParam::decoder_id_name() <<"\""); auto codec_it = std::find_if(cfg_params.begin(), cfg_params.end(), [] (const CfgParam& value) { - return value.get_name() == "mfxImplDescription.mfxDecoderDescription.decoder.CodecID"; + return value.get_name() == CfgParam::decoder_id_name(); }); if (codec_it != cfg_params.end()) { - GAPI_LOG_DEBUG(nullptr, "Dispatcher found \"mfxImplDescription.mfxDecoderDescription.decoder.CodecID\"" + GAPI_LOG_DEBUG(nullptr, "Dispatcher found \"" << CfgParam::decoder_id_name() << "\"" " so try on raw data provider at first"); try { diff --git a/modules/gapi/src/streaming/onevpl/data_provider_interface_exception.cpp b/modules/gapi/src/streaming/onevpl/data_provider_interface_exception.cpp index f30ab1cfad..18d51ebf2c 100644 --- a/modules/gapi/src/streaming/onevpl/data_provider_interface_exception.cpp +++ b/modules/gapi/src/streaming/onevpl/data_provider_interface_exception.cpp @@ -5,8 +5,7 @@ // Copyright (C) 2021 Intel Corporation #ifdef HAVE_ONEVPL -#include -#include +#include "streaming/onevpl/onevpl_export.hpp" #endif // HAVE_ONEVPL #include diff --git a/modules/gapi/src/streaming/onevpl/demux/async_mfp_demux_data_provider.hpp b/modules/gapi/src/streaming/onevpl/demux/async_mfp_demux_data_provider.hpp index 706d63dca7..f63593a46e 100644 --- a/modules/gapi/src/streaming/onevpl/demux/async_mfp_demux_data_provider.hpp +++ b/modules/gapi/src/streaming/onevpl/demux/async_mfp_demux_data_provider.hpp @@ -13,7 +13,7 @@ #include #ifdef HAVE_ONEVPL -#include +#include "streaming/onevpl/onevpl_export.hpp" #ifdef _WIN32 #define NOMINMAX diff --git a/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp b/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp index 0abca7f5a5..6707a401b1 100644 --- a/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp +++ b/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp @@ -16,6 +16,7 @@ #include "streaming/onevpl/engine/decode/decode_session.hpp" #include "streaming/onevpl/accelerators/accel_policy_interface.hpp" #include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "streaming/onevpl/cfg_params_parser.hpp" #include "streaming/onevpl/utils.hpp" #include "logger.hpp" @@ -24,95 +25,6 @@ namespace cv { namespace gapi { namespace wip { namespace onevpl { -/* UTILS */ -mfxU32 GetSurfaceSize(mfxU32 FourCC, mfxU32 width, mfxU32 height) { - mfxU32 nbytes = 0; - - mfxU32 half_width = width / 2; - mfxU32 half_height = height / 2; - switch (FourCC) { - case MFX_FOURCC_I420: - case MFX_FOURCC_NV12: - nbytes = width * height + 2 * half_width * half_height; - break; - case MFX_FOURCC_I010: - case MFX_FOURCC_P010: - nbytes = width * height + 2 * half_width * half_height; - nbytes *= 2; - break; - case MFX_FOURCC_RGB4: - nbytes = width * height * 4; - break; - default: - GAPI_LOG_WARNING(nullptr, "Unsupported FourCC requested: " << FourCC); - GAPI_Assert(false && "Unsupported FourCC requested"); - break; - } - return nbytes; -} - -surface_ptr_t create_surface_RGB4(mfxFrameInfo frameInfo, - std::shared_ptr out_buf_ptr, - size_t out_buf_ptr_offset, - size_t out_buf_size) -{ - mfxU8* buf = reinterpret_cast(out_buf_ptr.get()); - mfxU16 surfW = frameInfo.Width * 4; - mfxU16 surfH = frameInfo.Height; - (void)surfH; - - // TODO more intelligent check - if (out_buf_size <= out_buf_ptr_offset) { - throw std::runtime_error(std::string("Insufficient buffer size: ") + - std::to_string(out_buf_size) + ", buffer offset: " + - std::to_string(out_buf_ptr_offset) + - ", expected surface width: " + std::to_string(surfW) + - ", height: " + std::to_string(surfH)); - } - - std::unique_ptr handle(new mfxFrameSurface1); - memset(handle.get(), 0, sizeof(mfxFrameSurface1)); - - handle->Info = frameInfo; - handle->Data.B = buf + out_buf_ptr_offset; - handle->Data.G = handle->Data.B + 1; - handle->Data.R = handle->Data.B + 2; - handle->Data.A = handle->Data.B + 3; - handle->Data.Pitch = surfW; - - return Surface::create_surface(std::move(handle), out_buf_ptr); -} - -surface_ptr_t create_surface_other(mfxFrameInfo frameInfo, - std::shared_ptr out_buf_ptr, - size_t out_buf_ptr_offset, - size_t out_buf_size) -{ - mfxU8* buf = reinterpret_cast(out_buf_ptr.get()); - mfxU16 surfH = frameInfo.Height; - mfxU16 surfW = (frameInfo.FourCC == MFX_FOURCC_P010) ? frameInfo.Width * 2 : frameInfo.Width; - - // TODO more intelligent check - if (out_buf_size <= - out_buf_ptr_offset + (surfW * surfH) + ((surfW / 2) * (surfH / 2))) { - throw std::runtime_error(std::string("Insufficient buffer size: ") + - std::to_string(out_buf_size) + ", buffer offset: " + - std::to_string(out_buf_ptr_offset) + - ", expected surface width: " + std::to_string(surfW) + - ", height: " + std::to_string(surfH)); - } - - std::unique_ptr handle(new mfxFrameSurface1); - memset(handle.get(), 0, sizeof(mfxFrameSurface1)); - - handle->Info = frameInfo; - handle->Data.Y = buf + out_buf_ptr_offset; - handle->Data.U = buf + out_buf_ptr_offset + (surfW * surfH); - handle->Data.V = handle->Data.U + ((surfW / 2) * (surfH / 2)); - handle->Data.Pitch = surfW; - - return Surface::create_surface(std::move(handle), out_buf_ptr); -} VPLLegacyDecodeEngine::VPLLegacyDecodeEngine(std::unique_ptr&& accel) : ProcessingEngineBase(std::move(accel)) { @@ -146,8 +58,9 @@ VPLLegacyDecodeEngine::VPLLegacyDecodeEngine(std::unique_ptrDataLength)) ? my_sess.stream.get() + : nullptr, /* No more data to read, start decode draining mode*/ my_sess.procesing_surface_ptr.lock()->get_handle(), &sync_pair.second, @@ -164,33 +77,30 @@ VPLLegacyDecodeEngine::VPLLegacyDecodeEngine(std::unique_ptrget_handle(), &sync_pair.second, &sync_pair.first); } catch (const std::runtime_error& ex) { + // NB: not an error, yield CPU ticks to check + // surface availability at a next phase. + // But print WARNING to notify user about pipeline stuck GAPI_LOG_WARNING(nullptr, "[" << my_sess.session << "] has no surface, reason: " << ex.what()); - // TODO it is supposed to place `break;` here - // to simulate `yield`-like behavior. - // Further DX11 intergation logic claims more strict rules - // for enqueue surfaces. If no free surface - // is available it had better to wait free one by checking - // for async result than waste time in spinning. - // - // Put it as-is at now to not break - // current compatibility and avoid further merge-conflicts + break; } } if (my_sess.last_status == MFX_ERR_NONE) { my_sess.sync_queue.emplace(sync_pair); } else if (my_sess.last_status != MFX_ERR_MORE_DATA) /* suppress MFX_ERR_MORE_DATA warning */ { - GAPI_LOG_WARNING(nullptr, "pending ops count: " << my_sess.sync_queue.size() << + GAPI_LOG_WARNING(nullptr, "decode pending ops count: " << + my_sess.sync_queue.size() << ", sync id: " << sync_pair.first << - ", status: " << mfxstatus_to_string(my_sess.last_status)); + ", status: " << + mfxstatus_to_string(my_sess.last_status)); } return ExecutionStatus::Continue; }, @@ -198,20 +108,26 @@ VPLLegacyDecodeEngine::VPLLegacyDecodeEngine(std::unique_ptr ExecutionStatus { LegacyDecodeSession& my_sess = static_cast(sess); - if (!my_sess.sync_queue.empty()) // FIFO: check the oldest async operation complete - { - LegacyDecodeSession::op_handle_t& pending_op = my_sess.sync_queue.front(); - sess.last_status = MFXVideoCORE_SyncOperation(sess.session, pending_op.first, 0); + do { + if (!my_sess.sync_queue.empty()) { // FIFO: check the oldest async operation complete + LegacyDecodeSession::op_handle_t& pending_op = my_sess.sync_queue.front(); + sess.last_status = MFXVideoCORE_SyncOperation(sess.session, pending_op.first, 0); - GAPI_LOG_DEBUG(nullptr, "pending ops count: " << my_sess.sync_queue.size() << - ", sync id: " << pending_op.first << - ", status: " << mfxstatus_to_string(my_sess.last_status)); + GAPI_LOG_DEBUG(nullptr, "pending ops count: " << + my_sess.sync_queue.size() << + ", sync id: " << + pending_op.first << + ", surface: " << + pending_op.second << + ", status: " << + mfxstatus_to_string(my_sess.last_status)); - // put frames in ready queue on success - if (MFX_ERR_NONE == sess.last_status) { - on_frame_ready(my_sess, pending_op.second); + // put frames in ready queue on success + if (MFX_ERR_NONE == sess.last_status) { + on_frame_ready(my_sess, pending_op.second); + } } - } + } while (MFX_ERR_NONE == sess.last_status && !my_sess.sync_queue.empty()); return ExecutionStatus::Continue; }, // 4) Falls back on generic status procesing @@ -222,45 +138,138 @@ VPLLegacyDecodeEngine::VPLLegacyDecodeEngine(std::unique_ptr provider) -{ - mfxFrameAllocRequest decRequest = {}; +ProcessingEngineBase::session_ptr +VPLLegacyDecodeEngine::initialize_session(mfxSession mfx_session, + const std::vector& cfg_params, + std::shared_ptr provider) { + GAPI_DbgAssert(provider && "Cannot create decoder, data provider is nullptr"); + + // init session + acceleration_policy->init(mfx_session); + + // Get codec ID from data provider + IDataProvider::mfx_codec_id_type decoder_id_name = provider->get_mfx_codec_id(); + + // Prepare video param + mfxVideoParam mfxDecParams {}; + mfxDecParams.mfx.CodecId = decoder_id_name; + + // set memory stream direction accroding to accelearion policy device type + IDeviceSelector::DeviceScoreTable devices = acceleration_policy->get_device_selector()->select_devices(); + GAPI_Assert(devices.size() == 1 && "Multiple(or zero) acceleration devices case is unsupported"); + AccelType accel_type = devices.begin()->second.get_type(); + if (accel_type == AccelType::DX11) { + mfxDecParams.IOPattern = MFX_IOPATTERN_OUT_VIDEO_MEMORY; + } else if (accel_type == AccelType::HOST) { + mfxDecParams.IOPattern = MFX_IOPATTERN_OUT_SYSTEM_MEMORY; + } else { + GAPI_Assert(false && "unsupported AccelType from device selector"); + } + + // try fetch & decode input data + mfxStatus sts = MFX_ERR_NONE; + std::shared_ptr bitstream{}; + bool can_fetch_data = false; + do { + can_fetch_data = provider->fetch_bitstream_data(bitstream); + if (!can_fetch_data) { + // must fetch data always because EOF critical at this point + GAPI_LOG_WARNING(nullptr, "cannot decode header from provider: " << provider.get() << + ". Unexpected EOF"); + throw std::runtime_error("Error reading bitstream: EOF"); + } + + sts = MFXVideoDECODE_DecodeHeader(mfx_session, bitstream.get(), &mfxDecParams); + if(MFX_ERR_NONE != sts && MFX_ERR_MORE_DATA != sts) { + throw std::runtime_error("Error decoding header, error: " + + mfxstatus_to_string(sts)); + } + } while (sts == MFX_ERR_MORE_DATA && !provider->empty()); + + if (MFX_ERR_NONE != sts) { + GAPI_LOG_WARNING(nullptr, "cannot decode header from provider: " << provider.get() + << ". Make sure data source is valid and/or " + "\"" << CfgParam::decoder_id_name() << "\"" + " has correct value in case of demultiplexed raw input"); + throw std::runtime_error("Error decode header, error: " + + mfxstatus_to_string(sts)); + } + mfxFrameAllocRequest decRequest {}; + // Query number required surfaces for decoder - MFXVideoDECODE_QueryIOSurf(mfx_session, &decoder_param.param, &decRequest); + MFXVideoDECODE_QueryIOSurf(mfx_session, &mfxDecParams, &decRequest); // External (application) allocation of decode surfaces GAPI_LOG_DEBUG(nullptr, "Query IOSurf for session: " << mfx_session << + ", mfxFrameAllocRequest.NumFrameMin: " << decRequest.NumFrameMin << ", mfxFrameAllocRequest.NumFrameSuggested: " << decRequest.NumFrameSuggested << ", mfxFrameAllocRequest.Type: " << decRequest.Type); - mfxU32 singleSurfaceSize = GetSurfaceSize(decoder_param.param.mfx.FrameInfo.FourCC, - decoder_param.param.mfx.FrameInfo.Width, - decoder_param.param.mfx.FrameInfo.Height); - if (!singleSurfaceSize) { - throw std::runtime_error("Cannot determine surface size for: fourCC" + - std::to_string(decoder_param.param.mfx.FrameInfo.FourCC) + - ", width: " + std::to_string(decoder_param.param.mfx.FrameInfo.Width) + - ", height: " + std::to_string(decoder_param.param.mfx.FrameInfo.Height)); + // NB: override NumFrameSuggested preallocation size (how many frames we can hold) + size_t preallocated_frames_count = decRequest.NumFrameSuggested; + // NB: if you see bunch of WARNING about "cannot get free surface from pool" + // and have abundant RAM size then increase `preallocated_frames_count` + // to keep more free surfaces in a round. Otherwise VPL decode pipeline will be waiting + // till application is freeing unusable surface on its side. + // + auto queue_capacity_it = std::find_if(cfg_params.begin(), cfg_params.end(), [] (const CfgParam& value) { + return value.get_name() == CfgParam::frames_pool_size_name(); + }); + if (queue_capacity_it != cfg_params.end()) { + cv::util::visit(cv::util::overload_lambdas( + [&preallocated_frames_count](uint8_t value) { preallocated_frames_count = static_cast(value); }, + [&preallocated_frames_count](int8_t value) { preallocated_frames_count = static_cast(value); }, + [&preallocated_frames_count](uint16_t value) { preallocated_frames_count = static_cast(value); }, + [&preallocated_frames_count](int16_t value) { preallocated_frames_count = static_cast(value); }, + [&preallocated_frames_count](uint32_t value) { preallocated_frames_count = static_cast(value); }, + [&preallocated_frames_count](int32_t value) { preallocated_frames_count = static_cast(value); }, + [&preallocated_frames_count](uint64_t value) { preallocated_frames_count = static_cast(value); }, + [&preallocated_frames_count](int64_t value) { preallocated_frames_count = static_cast(value); }, + [&preallocated_frames_count](float_t value) { preallocated_frames_count = static_cast(value); }, + [&preallocated_frames_count](double_t value) { preallocated_frames_count = static_cast(value); }, + [&preallocated_frames_count](void*) { GAPI_Assert(false && "`void*` is unsupported type"); }, + [&preallocated_frames_count](const std::string& value) { + preallocated_frames_count = strtoull_or_throw(value.c_str()); + }), + queue_capacity_it->get_value()); + + GAPI_LOG_INFO(nullptr, "Try to use CfgParam \"" << CfgParam::frames_pool_size_name() << "\": " << + preallocated_frames_count << ", for session: " << mfx_session); + + } + if (preallocated_frames_count < decRequest.NumFrameMin) { + GAPI_LOG_WARNING(nullptr, "Cannot proceed with CfgParam \"" << CfgParam::frames_pool_size_name() << "\": " << + preallocated_frames_count << ". It must be equal or greater than " + "mfxFrameAllocRequest.NumFrameMin: " << decRequest.NumFrameMin); + throw std::runtime_error(std::string("Invalid value of param: ") + + CfgParam::frames_pool_size_name() + ", underflow"); + } else { + if (static_cast(std::numeric_limits::max()) < preallocated_frames_count) { + GAPI_LOG_WARNING(nullptr, "Cannot proceed with CfgParam \"" << CfgParam::frames_pool_size_name() << "\": " << + preallocated_frames_count << ". It must not be equal than " << + std::numeric_limits::max()); + throw std::runtime_error(std::string("Invalid value of param: ") + + CfgParam::frames_pool_size_name() + ", overflow"); + } + decRequest.NumFrameSuggested = static_cast(preallocated_frames_count); + GAPI_LOG_DEBUG(nullptr, "mfxFrameAllocRequest overriden by user input for session: " << mfx_session << + ", mfxFrameAllocRequest.NumFrameMin: " << decRequest.NumFrameMin << + ", mfxFrameAllocRequest.NumFrameSuggested: " << decRequest.NumFrameSuggested << + ", mfxFrameAllocRequest.Type: " << decRequest.Type); } - const auto &frameInfo = decoder_param.param.mfx.FrameInfo; - auto surface_creator = - [&frameInfo] (std::shared_ptr out_buf_ptr, size_t out_buf_ptr_offset, - size_t out_buf_size) -> surface_ptr_t { - return (frameInfo.FourCC == MFX_FOURCC_RGB4) ? - create_surface_RGB4(frameInfo, out_buf_ptr, out_buf_ptr_offset, - out_buf_size) : - create_surface_other(frameInfo, out_buf_ptr, out_buf_ptr_offset, - out_buf_size);}; - - //TODO Configure preallocation size (how many frames we can hold) - const size_t preallocated_frames_count = 30; VPLAccelerationPolicy::pool_key_t decode_pool_key = - acceleration_policy->create_surface_pool(decRequest.NumFrameSuggested * preallocated_frames_count, - singleSurfaceSize, - surface_creator); + acceleration_policy->create_surface_pool(decRequest, mfxDecParams); + + // Input parameters finished, now initialize decode + // create decoder for session accoring to header recovered from source file + sts = MFXVideoDECODE_Init(mfx_session, &mfxDecParams); + if (MFX_ERR_NONE != sts) { + throw std::runtime_error("Error initializing Decode, error: " + + mfxstatus_to_string(sts)); + } + + DecoderParams decoder_param {bitstream, mfxDecParams}; // create session std::shared_ptr sess_ptr = @@ -271,6 +280,7 @@ void VPLLegacyDecodeEngine::initialize_session(mfxSession mfx_session, sess_ptr->init_surface_pool(decode_pool_key); // prepare working decode surface sess_ptr->swap_surface(*this); + return sess_ptr; } ProcessingEngineBase::ExecutionStatus VPLLegacyDecodeEngine::execute_op(operation_t& op, EngineSession& sess) { @@ -303,14 +313,13 @@ ProcessingEngineBase::ExecutionStatus VPLLegacyDecodeEngine::process_error(mfxSt sess.swap_surface(*this); return ExecutionStatus::Continue; } catch (const std::runtime_error& ex) { - GAPI_LOG_WARNING(nullptr, "[" << sess.session << "] error: " << ex.what() << - "Abort"); - // TODO it is supposed to be `break;` here in future PR + GAPI_LOG_WARNING(nullptr, "[" << sess.session << "] error: " << ex.what()); + return ExecutionStatus::Continue; // read more data } } case MFX_ERR_MORE_DATA: // The function requires more bitstream at input before decoding can proceed - if (!sess.data_provider || sess.data_provider->empty()) { - // No more data to drain from decoder, start encode draining mode + if (!(sess.data_provider || (sess.stream && sess.stream->DataLength))) { + // No more data to drain from decoder return ExecutionStatus::Processed; } else @@ -326,7 +335,7 @@ ProcessingEngineBase::ExecutionStatus VPLLegacyDecodeEngine::process_error(mfxSt return ExecutionStatus::Continue; } catch (const std::runtime_error& ex) { GAPI_LOG_WARNING(nullptr, "[" << sess.session << "] error: " << ex.what()); - // TODO it is supposed to be `break;` here in future PR + return ExecutionStatus::Continue; // read more data } break; } @@ -371,9 +380,8 @@ ProcessingEngineBase::ExecutionStatus VPLLegacyDecodeEngine::process_error(mfxSt sess.swap_surface(*this); return ExecutionStatus::Continue; } catch (const std::runtime_error& ex) { - GAPI_LOG_WARNING(nullptr, "[" << sess.session << "] error: " << ex.what() << - "Abort"); - // TODO it is supposed to be `break;` here in future PR + GAPI_LOG_WARNING(nullptr, "[" << sess.session << "] error: " << ex.what()); + return ExecutionStatus::Continue; } default: GAPI_LOG_WARNING(nullptr, "Unknown status code: " << mfxstatus_to_string(status) << diff --git a/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.hpp b/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.hpp index 3499d7f6df..f6a02db3db 100644 --- a/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.hpp +++ b/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.hpp @@ -12,10 +12,7 @@ #include "streaming/onevpl/engine/processing_engine_base.hpp" #ifdef HAVE_ONEVPL -#if (MFX_VERSION >= 2000) - #include -#endif -#include +#include "streaming/onevpl/onevpl_export.hpp" namespace cv { namespace gapi { @@ -31,8 +28,9 @@ class VPLLegacyDecodeEngine : public ProcessingEngineBase { public: VPLLegacyDecodeEngine(std::unique_ptr&& accel); - void initialize_session(mfxSession mfx_session, DecoderParams&& decoder_param, - std::shared_ptr provider) override; + session_ptr initialize_session(mfxSession mfx_session, + const std::vector& cfg_params, + std::shared_ptr provider) override; private: ExecutionStatus execute_op(operation_t& op, EngineSession& sess) override; diff --git a/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.cpp b/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.cpp index 871f588ffa..bbb1378767 100644 --- a/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.cpp +++ b/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.cpp @@ -52,8 +52,10 @@ void LegacyDecodeSession::swap_surface(VPLLegacyDecodeEngine& engine) { procesing_surface_ptr = cand; } catch (const std::runtime_error& ex) { - GAPI_LOG_WARNING(nullptr, "[" << session << "] error: " << ex.what() << - "Abort"); + GAPI_LOG_WARNING(nullptr, "[" << session << "] error: " << ex.what()); + + // Delegate exception processing on caller + throw; } } @@ -72,6 +74,10 @@ Data::Meta LegacyDecodeSession::generate_frame_meta() { }; return meta; } + +const mfxVideoParam& LegacyDecodeSession::get_video_param() const { + return mfx_decoder_param; +} } // namespace onevpl } // namespace wip } // namespace gapi diff --git a/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.hpp b/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.hpp index 58be858fdb..476a575172 100644 --- a/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.hpp +++ b/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.hpp @@ -15,10 +15,7 @@ #include "streaming/onevpl/engine/engine_session.hpp" #include "streaming/onevpl/accelerators/accel_policy_interface.hpp" #ifdef HAVE_ONEVPL -#if (MFX_VERSION >= 2000) - #include -#endif -#include +#include "streaming/onevpl/onevpl_export.hpp" namespace cv { namespace gapi { @@ -40,11 +37,11 @@ public: void swap_surface(VPLLegacyDecodeEngine& engine); void init_surface_pool(VPLAccelerationPolicy::pool_key_t key); + Data::Meta generate_frame_meta(); + const mfxVideoParam& get_video_param() const override; +private: mfxVideoParam mfx_decoder_param; std::shared_ptr data_provider; - - Data::Meta generate_frame_meta(); -private: VPLAccelerationPolicy::pool_key_t decoder_pool_id; mfxFrameAllocRequest request; diff --git a/modules/gapi/src/streaming/onevpl/engine/engine_session.hpp b/modules/gapi/src/streaming/onevpl/engine/engine_session.hpp index e0c6d01f8b..67018d0fd7 100644 --- a/modules/gapi/src/streaming/onevpl/engine/engine_session.hpp +++ b/modules/gapi/src/streaming/onevpl/engine/engine_session.hpp @@ -18,7 +18,7 @@ #include #ifdef HAVE_ONEVPL -#include +#include "streaming/onevpl/onevpl_export.hpp" namespace cv { namespace gapi { @@ -40,6 +40,8 @@ struct GAPI_EXPORTS EngineSession { EngineSession(mfxSession sess, std::shared_ptr&& str); std::string error_code_to_str() const; virtual ~EngineSession(); + + virtual const mfxVideoParam& get_video_param() const = 0; }; } // namespace onevpl } // namespace wip diff --git a/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp index 9a4c4a7fb0..72f2f62fc4 100644 --- a/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp +++ b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp @@ -23,7 +23,8 @@ ProcessingEngineBase::ProcessingEngineBase(std::unique_ptrerror_code_to_str() << ", " << ProcessingEngineBase::status_to_string(status) << @@ -57,6 +59,7 @@ ProcessingEngineBase::ExecutionStatus ProcessingEngineBase::process(mfxSession s } if (status == ExecutionStatus::Processed) { + GAPI_LOG_INFO(nullptr, "Processed [" << session << "]"); sessions.erase(sess_it); execution_table.erase(session); } @@ -90,6 +93,7 @@ void ProcessingEngineBase::get_frame(Data &data) { data = ready_frames.front(); ready_frames.pop(); + GAPI_LOG_DEBUG(nullptr, " elapsed ready frames count: " << ready_frames.size()); } const VPLAccelerationPolicy* ProcessingEngineBase::get_accel() const { diff --git a/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp index 4b34721d67..059ef963de 100644 --- a/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp +++ b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp @@ -8,6 +8,7 @@ #define GAPI_STREAMING_ONEVPL_ENGINE_PROCESSING_ENGINE_BASE_HPP #include +#include #include "streaming/onevpl/engine/engine_session.hpp" #include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS @@ -49,9 +50,9 @@ public: ProcessingEngineBase(std::unique_ptr&& accel); virtual ~ProcessingEngineBase(); - virtual void initialize_session(mfxSession mfx_session, - DecoderParams&& decoder_param, - std::shared_ptr provider) = 0; + virtual session_ptr initialize_session(mfxSession mfx_session, + const std::vector& cfg_params, + std::shared_ptr provider) = 0; ExecutionStatus process(mfxSession session); size_t get_ready_frames_count() const; diff --git a/modules/gapi/src/streaming/onevpl/file_data_provider.cpp b/modules/gapi/src/streaming/onevpl/file_data_provider.cpp index 020d471b55..a86d541904 100644 --- a/modules/gapi/src/streaming/onevpl/file_data_provider.cpp +++ b/modules/gapi/src/streaming/onevpl/file_data_provider.cpp @@ -28,14 +28,14 @@ FileDataProvider::FileDataProvider(const std::string& file_path, codec_params.size()); auto codec_it = std::find_if(codec_params.begin(), codec_params.end(), [] (const CfgParam& value) { - return value.get_name() == "mfxImplDescription.mfxDecoderDescription.decoder.CodecID"; + return value.get_name() == CfgParam::decoder_id_name(); }); if (codec_it == codec_params.end()) { GAPI_LOG_WARNING(nullptr, "[" << this << "] " << - "\"mfxImplDescription.mfxDecoderDescription.decoder.CodecID\" " + "\"" << CfgParam::decoder_id_name() << "\" " "is absent, total param count" << codec_params.size()); - throw DataProviderUnsupportedException("\"mfxImplDescription.mfxDecoderDescription.decoder.CodecID\" " + throw DataProviderUnsupportedException(std::string("\"") + CfgParam::decoder_id_name() + "\" " "is required for FileDataProvider"); } diff --git a/modules/gapi/src/streaming/onevpl/onevpl_export.hpp b/modules/gapi/src/streaming/onevpl/onevpl_export.hpp new file mode 100644 index 0000000000..44970ee7be --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/onevpl_export.hpp @@ -0,0 +1,25 @@ +#ifndef GAPI_STREAMING_ONEVPL_EXPORT_HPP +#define GAPI_STREAMING_ONEVPL_EXPORT_HPP + +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4201) +#pragma warning(disable : 4302) +#pragma warning(disable : 4311) +#pragma warning(disable : 4312) +#endif // defined(_MSC_VER) + +#ifdef HAVE_ONEVPL +#if (MFX_VERSION >= 2000) +#include +#endif // MFX_VERSION + +#include +#include +#endif // HAVE_ONEVPL + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif // defined(_MSC_VER) + +#endif // GAPI_STREAMING_ONEVPL_EXPORT_HPP diff --git a/modules/gapi/src/streaming/onevpl/source_priv.cpp b/modules/gapi/src/streaming/onevpl/source_priv.cpp index c5de2a6998..fd2a401957 100644 --- a/modules/gapi/src/streaming/onevpl/source_priv.cpp +++ b/modules/gapi/src/streaming/onevpl/source_priv.cpp @@ -53,14 +53,15 @@ GSource::Priv::Priv() : mfx_session(), description(), description_is_valid(false), - engine() + engine(), + consumed_frames_count() { GAPI_LOG_INFO(nullptr, "Initialized MFX handle: " << mfx_handle); } GSource::Priv::Priv(std::shared_ptr provider, const std::vector& params, - std::shared_ptr) : + std::shared_ptr device_selector) : GSource::Priv() { // Enable Config @@ -193,13 +194,9 @@ GSource::Priv::Priv(std::shared_ptr provider, GAPI_LOG_INFO(nullptr, "Initialized MFX session: " << mfx_session); - // initialize decoder - // Find codec ID from config - IDataProvider::mfx_codec_id_type decoder_id = provider->get_mfx_codec_id(); - // create session driving engine if required if (!engine) { - std::unique_ptr acceleration = initializeHWAccel(); + std::unique_ptr acceleration = initializeHWAccel(device_selector); // TODO Add factory static method in ProcessingEngineBase if (mfx_impl_description->ApiVersion.Major >= VPL_NEW_API_MAJOR_VERSION) { @@ -211,106 +208,54 @@ GSource::Priv::Priv(std::shared_ptr provider, } } - //create decoder for session accoring to header recovered from source file - DecoderParams decoder_param = create_decoder_from_file(decoder_id, provider); - // create engine session for processing mfx session pipeline - engine->initialize_session(mfx_session, std::move(decoder_param), - provider); + auto engine_session_ptr = engine->initialize_session(mfx_session, cfg_params, + provider); + + const mfxVideoParam& video_param = engine_session_ptr->get_video_param(); + + // set valid description + description.size = cv::Size { + video_param.mfx.FrameInfo.Width, + video_param.mfx.FrameInfo.Height}; + switch(video_param.mfx.FrameInfo.FourCC) { + case MFX_FOURCC_I420: + throw std::runtime_error("Cannot parse GMetaArg description: MediaFrame doesn't support I420 type"); + case MFX_FOURCC_NV12: + description.fmt = cv::MediaFormat::NV12; + break; + default: + throw std::runtime_error("Cannot parse GMetaArg description: MediaFrame unknown 'fmt' type: " + + std::to_string(video_param.mfx.FrameInfo.FourCC)); + } + description_is_valid = true; //prepare session for processing engine->process(mfx_session); } -GSource::Priv::~Priv() -{ +GSource::Priv::~Priv() { + engine.reset(); + + GAPI_LOG_INFO(nullptr, "consumed frames count: " << consumed_frames_count); GAPI_LOG_INFO(nullptr, "Unload MFX implementation description: " << mfx_impl_description); MFXDispReleaseImplDescription(mfx_handle, mfx_impl_description); GAPI_LOG_INFO(nullptr, "Unload MFX handle: " << mfx_handle); MFXUnload(mfx_handle); } -DecoderParams GSource::Priv::create_decoder_from_file(uint32_t decoder_id, - std::shared_ptr provider) -{ - GAPI_DbgAssert(provider && "Cannot create decoder, data provider is nullptr"); - - std::shared_ptr bitstream{}; - - // Retrieve the frame information from input stream - mfxVideoParam mfxDecParams {}; - mfxDecParams.mfx.CodecId = decoder_id; - mfxDecParams.IOPattern = MFX_IOPATTERN_OUT_SYSTEM_MEMORY;//MFX_IOPATTERN_OUT_VIDEO_MEMORY; - mfxStatus sts = MFX_ERR_NONE; - bool can_fetch_data = false; - do { - can_fetch_data = provider->fetch_bitstream_data(bitstream); - if (!can_fetch_data) { - // must fetch data always because EOF critical at this point - GAPI_LOG_WARNING(nullptr, "cannot decode header from provider: " << provider.get() << - ". Unexpected EOF"); - throw std::runtime_error("Error reading bitstream: EOF"); - } - - sts = MFXVideoDECODE_DecodeHeader(mfx_session, bitstream.get(), &mfxDecParams); - if(MFX_ERR_NONE != sts && MFX_ERR_MORE_DATA != sts) { - throw std::runtime_error("Error decoding header, error: " + - mfxstatus_to_string(sts)); - } - } while (sts == MFX_ERR_MORE_DATA && !provider->empty()); - - if (MFX_ERR_NONE != sts) { - GAPI_LOG_WARNING(nullptr, "cannot decode header from provider: " << provider.get() - << ". Make sure data source is valid and/or " - "\"mfxImplDescription.mfxDecoderDescription.decoder.CodecID\"" - " has correct value in case of demultiplexed raw input"); - throw std::runtime_error("Error decode header, error: " + - mfxstatus_to_string(sts)); - } - - // Input parameters finished, now initialize decode - sts = MFXVideoDECODE_Init(mfx_session, &mfxDecParams); - if (MFX_ERR_NONE != sts) { - throw std::runtime_error("Error initializing Decode, error: " + - mfxstatus_to_string(sts)); - } - - // set valid description - description.size = cv::Size { - mfxDecParams.mfx.FrameInfo.Width, - mfxDecParams.mfx.FrameInfo.Height}; - switch(mfxDecParams.mfx.FrameInfo.FourCC) { - case MFX_FOURCC_I420: - GAPI_Assert(false && "Cannot create GMetaArg description: " - "MediaFrame doesn't support I420 type"); - case MFX_FOURCC_NV12: - description.fmt = cv::MediaFormat::NV12; - break; - default: - { - GAPI_LOG_WARNING(nullptr, "Cannot create GMetaArg description: " - "MediaFrame unknown 'fmt' type: " << - std::to_string(mfxDecParams.mfx.FrameInfo.FourCC)); - GAPI_Assert(false && "Cannot create GMetaArg description: invalid value"); - } - } - description_is_valid = true; - - return {bitstream, mfxDecParams}; -} - -std::unique_ptr GSource::Priv::initializeHWAccel() +std::unique_ptr GSource::Priv::initializeHWAccel(std::shared_ptr selector) { std::unique_ptr ret; auto accel_mode_it = std::find_if(cfg_params.begin(), cfg_params.end(), [] (const CfgParam& value) { - return value.get_name() == "mfxImplDescription.AccelerationMode"; + return value.get_name() == CfgParam::acceleration_mode_name(); }); if (accel_mode_it == cfg_params.end()) { GAPI_LOG_DEBUG(nullptr, "No HW Accel requested. Use CPU"); - ret.reset(new VPLCPUAccelerationPolicy); + ret.reset(new VPLCPUAccelerationPolicy(selector)); return ret; } @@ -320,13 +265,13 @@ std::unique_ptr GSource::Priv::initializeHWAccel() switch(accel_mode.Data.U32) { case MFX_ACCEL_MODE_VIA_D3D11: { - std::unique_ptr cand(new VPLDX11AccelerationPolicy); + std::unique_ptr cand(new VPLDX11AccelerationPolicy(selector)); ret = std::move(cand); break; } case MFX_ACCEL_MODE_NA: { - std::unique_ptr cand(new VPLCPUAccelerationPolicy); + std::unique_ptr cand(new VPLCPUAccelerationPolicy(selector)); ret = std::move(cand); break; } @@ -367,6 +312,7 @@ bool GSource::Priv::pull(cv::gapi::wip::Data& data) if (engine->get_ready_frames_count()) { engine->get_frame(data); + consumed_frames_count++; return true; } else { return false; diff --git a/modules/gapi/src/streaming/onevpl/source_priv.hpp b/modules/gapi/src/streaming/onevpl/source_priv.hpp index 07e1139191..b835850d35 100644 --- a/modules/gapi/src/streaming/onevpl/source_priv.hpp +++ b/modules/gapi/src/streaming/onevpl/source_priv.hpp @@ -17,14 +17,7 @@ #include #ifdef HAVE_ONEVPL -#if (MFX_VERSION >= 2000) -#include -#endif // MFX_VERSION - -#include - -#include - +#include "streaming/onevpl/onevpl_export.hpp" #include "streaming/onevpl/engine/processing_engine_base.hpp" namespace cv { @@ -49,9 +42,7 @@ struct GSource::Priv GMetaArg descr_of() const; private: Priv(); - DecoderParams create_decoder_from_file(uint32_t decoder_id, - std::shared_ptr provider); - std::unique_ptr initializeHWAccel(); + std::unique_ptr initializeHWAccel(std::shared_ptr selector); mfxLoader mfx_handle; mfxImplDescription *mfx_impl_description; @@ -64,6 +55,8 @@ private: bool description_is_valid; std::unique_ptr engine; + + size_t consumed_frames_count; }; } // namespace onevpl } // namespace wip diff --git a/modules/gapi/src/streaming/onevpl/utils.cpp b/modules/gapi/src/streaming/onevpl/utils.cpp index 6cbe0e7ea1..3ec0dea8ae 100644 --- a/modules/gapi/src/streaming/onevpl/utils.cpp +++ b/modules/gapi/src/streaming/onevpl/utils.cpp @@ -14,6 +14,18 @@ #include "streaming/onevpl/utils.hpp" #include "logger.hpp" +#define ONEVPL_STRINGIFY_CASE(value) \ + case value: return #value; + +#define ONEVPL_STRINGIFY_IF(value) \ + if (!strcmp(cstr, #value)) { \ + return value; \ + } + +#define APPEND_STRINGIFY_MASK_N_ERASE(value, pref, mask) \ + if (value & mask) { ss << pref << #mask; value ^= mask; } + + namespace cv { namespace gapi { namespace wip { @@ -21,153 +33,96 @@ namespace onevpl { const char* mfx_impl_to_cstr(const mfxIMPL impl) { switch (impl) { - case MFX_IMPL_TYPE_SOFTWARE: - return "MFX_IMPL_TYPE_SOFTWARE"; - case MFX_IMPL_TYPE_HARDWARE: - return "MFX_IMPL_TYPE_HARDWARE"; - default: - return "unknown mfxIMPL"; + ONEVPL_STRINGIFY_CASE(MFX_IMPL_TYPE_SOFTWARE); + ONEVPL_STRINGIFY_CASE(MFX_IMPL_TYPE_HARDWARE); + default: return "unknown mfxIMPL"; } } mfxIMPL cstr_to_mfx_impl(const char* cstr) { - if (!strcmp(cstr, "MFX_IMPL_TYPE_SOFTWARE")) { - return MFX_IMPL_TYPE_SOFTWARE; - } else if (!strcmp(cstr, "MFX_IMPL_TYPE_HARDWARE")) { - return MFX_IMPL_TYPE_HARDWARE; - } - - throw std::logic_error(std::string("Invalid \"mfxImplDescription.Impl\":") + cstr); + ONEVPL_STRINGIFY_IF(MFX_IMPL_TYPE_SOFTWARE) + ONEVPL_STRINGIFY_IF(MFX_IMPL_TYPE_HARDWARE) + throw std::logic_error(std::string("Invalid \"") + CfgParam::implementation_name() + + "\":" + cstr); } const char* mfx_accel_mode_to_cstr (const mfxAccelerationMode mode) { switch (mode) { - case MFX_ACCEL_MODE_NA: - return "MFX_ACCEL_MODE_NA"; - case MFX_ACCEL_MODE_VIA_D3D9: - return "MFX_ACCEL_MODE_VIA_D3D9"; - case MFX_ACCEL_MODE_VIA_D3D11: - return "MFX_ACCEL_MODE_VIA_D3D11"; - case MFX_ACCEL_MODE_VIA_VAAPI: - return "MFX_ACCEL_MODE_VIA_VAAPI"; - case MFX_ACCEL_MODE_VIA_VAAPI_DRM_MODESET: - return "MFX_ACCEL_MODE_VIA_VAAPI_DRM_MODESET"; - case MFX_ACCEL_MODE_VIA_VAAPI_GLX: - return "MFX_ACCEL_MODE_VIA_VAAPI_GLX"; - case MFX_ACCEL_MODE_VIA_VAAPI_X11: - return "MFX_ACCEL_MODE_VIA_VAAPI_X11"; - case MFX_ACCEL_MODE_VIA_VAAPI_WAYLAND: - return "MFX_ACCEL_MODE_VIA_VAAPI_WAYLAND"; - case MFX_ACCEL_MODE_VIA_HDDLUNITE: - return "MFX_ACCEL_MODE_VIA_HDDLUNITE"; - default: - return "unknown mfxAccelerationMode"; + ONEVPL_STRINGIFY_CASE(MFX_ACCEL_MODE_NA) + ONEVPL_STRINGIFY_CASE(MFX_ACCEL_MODE_VIA_D3D9) + ONEVPL_STRINGIFY_CASE(MFX_ACCEL_MODE_VIA_D3D11) + ONEVPL_STRINGIFY_CASE(MFX_ACCEL_MODE_VIA_VAAPI) + ONEVPL_STRINGIFY_CASE(MFX_ACCEL_MODE_VIA_VAAPI_DRM_MODESET) + ONEVPL_STRINGIFY_CASE(MFX_ACCEL_MODE_VIA_VAAPI_GLX) + ONEVPL_STRINGIFY_CASE(MFX_ACCEL_MODE_VIA_VAAPI_X11) + ONEVPL_STRINGIFY_CASE(MFX_ACCEL_MODE_VIA_VAAPI_WAYLAND) + ONEVPL_STRINGIFY_CASE(MFX_ACCEL_MODE_VIA_HDDLUNITE) + default: return "unknown mfxAccelerationMode"; } - return "unknown mfxAccelerationMode"; } mfxAccelerationMode cstr_to_mfx_accel_mode(const char* cstr) { - if (!strcmp(cstr, "MFX_ACCEL_MODE_NA")) { - return MFX_ACCEL_MODE_NA; - } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_D3D9")) { - return MFX_ACCEL_MODE_VIA_D3D9; - } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_D3D11")) { - return MFX_ACCEL_MODE_VIA_D3D11; - } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_VAAPI")) { - return MFX_ACCEL_MODE_VIA_VAAPI; - } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_VAAPI_DRM_MODESET")) { - return MFX_ACCEL_MODE_VIA_VAAPI_DRM_MODESET; - } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_VAAPI_GLX")) { - return MFX_ACCEL_MODE_VIA_VAAPI_GLX; - } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_VAAPI_X11")) { - return MFX_ACCEL_MODE_VIA_VAAPI_X11; - } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_VAAPI_WAYLAND")) { - return MFX_ACCEL_MODE_VIA_VAAPI_WAYLAND; - } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_HDDLUNITE")) { - return MFX_ACCEL_MODE_VIA_HDDLUNITE; - } - throw std::logic_error(std::string("Invalid \"mfxImplDescription.AccelerationMode\":") + cstr); + ONEVPL_STRINGIFY_IF(MFX_ACCEL_MODE_NA) + ONEVPL_STRINGIFY_IF(MFX_ACCEL_MODE_VIA_D3D9) + ONEVPL_STRINGIFY_IF(MFX_ACCEL_MODE_VIA_D3D11) + ONEVPL_STRINGIFY_IF(MFX_ACCEL_MODE_VIA_VAAPI) + ONEVPL_STRINGIFY_IF(MFX_ACCEL_MODE_VIA_VAAPI_DRM_MODESET) + ONEVPL_STRINGIFY_IF(MFX_ACCEL_MODE_VIA_VAAPI_GLX) + ONEVPL_STRINGIFY_IF(MFX_ACCEL_MODE_VIA_VAAPI_X11) + ONEVPL_STRINGIFY_IF(MFX_ACCEL_MODE_VIA_VAAPI_WAYLAND) + ONEVPL_STRINGIFY_IF(MFX_ACCEL_MODE_VIA_HDDLUNITE) + throw std::logic_error(std::string("Invalid \"") + + CfgParam::acceleration_mode_name() + + "\":" + cstr); } const char* mfx_resource_type_to_cstr (const mfxResourceType type) { switch (type) { - case MFX_RESOURCE_SYSTEM_SURFACE: - return "MFX_RESOURCE_SYSTEM_SURFACE"; - case MFX_RESOURCE_VA_SURFACE: - return "MFX_RESOURCE_VA_SURFACE"; - case MFX_RESOURCE_VA_BUFFER: - return "MFX_RESOURCE_VA_BUFFER"; - case MFX_RESOURCE_DX9_SURFACE: - return "MFX_RESOURCE_DX9_SURFACE"; - case MFX_RESOURCE_DX11_TEXTURE: - return "MFX_RESOURCE_DX11_TEXTURE"; - case MFX_RESOURCE_DX12_RESOURCE: - return "MFX_RESOURCE_DX12_RESOURCE"; - case MFX_RESOURCE_DMA_RESOURCE: - return "MFX_RESOURCE_DMA_RESOURCE"; - case MFX_RESOURCE_HDDLUNITE_REMOTE_MEMORY: - return "MFX_RESOURCE_HDDLUNITE_REMOTE_MEMORY"; - default: - return "unknown mfxResourceType"; + ONEVPL_STRINGIFY_CASE(MFX_RESOURCE_SYSTEM_SURFACE) + ONEVPL_STRINGIFY_CASE(MFX_RESOURCE_VA_SURFACE) + ONEVPL_STRINGIFY_CASE(MFX_RESOURCE_VA_BUFFER) + ONEVPL_STRINGIFY_CASE(MFX_RESOURCE_DX9_SURFACE) + ONEVPL_STRINGIFY_CASE(MFX_RESOURCE_DX11_TEXTURE) + ONEVPL_STRINGIFY_CASE(MFX_RESOURCE_DX12_RESOURCE) + ONEVPL_STRINGIFY_CASE(MFX_RESOURCE_DMA_RESOURCE) + ONEVPL_STRINGIFY_CASE(MFX_RESOURCE_HDDLUNITE_REMOTE_MEMORY) + default: return "unknown mfxResourceType"; } } mfxResourceType cstr_to_mfx_resource_type(const char* cstr) { - if (!strcmp(cstr, "MFX_RESOURCE_SYSTEM_SURFACE")) { - return MFX_RESOURCE_SYSTEM_SURFACE; - } else if (!strcmp(cstr, "MFX_RESOURCE_VA_SURFACE")) { - return MFX_RESOURCE_VA_SURFACE; - } else if (!strcmp(cstr, "MFX_RESOURCE_VA_BUFFER")) { - return MFX_RESOURCE_VA_BUFFER; - } else if (!strcmp(cstr, "MFX_RESOURCE_DX9_SURFACE")) { - return MFX_RESOURCE_DX9_SURFACE; - } else if (!strcmp(cstr, "MFX_RESOURCE_DX11_TEXTURE")) { - return MFX_RESOURCE_DX11_TEXTURE; - } else if (!strcmp(cstr, "MFX_RESOURCE_DX12_RESOURCE")) { - return MFX_RESOURCE_DX12_RESOURCE; - } else if (!strcmp(cstr, "MFX_RESOURCE_DMA_RESOURCE")) { - return MFX_RESOURCE_DMA_RESOURCE; - } else if (!strcmp(cstr, "MFX_RESOURCE_HDDLUNITE_REMOTE_MEMORY")) { - return MFX_RESOURCE_HDDLUNITE_REMOTE_MEMORY; - } + ONEVPL_STRINGIFY_IF(MFX_RESOURCE_SYSTEM_SURFACE) + ONEVPL_STRINGIFY_IF(MFX_RESOURCE_VA_SURFACE) + ONEVPL_STRINGIFY_IF(MFX_RESOURCE_VA_BUFFER) + ONEVPL_STRINGIFY_IF(MFX_RESOURCE_DX9_SURFACE) + ONEVPL_STRINGIFY_IF(MFX_RESOURCE_DX11_TEXTURE) + ONEVPL_STRINGIFY_IF(MFX_RESOURCE_DX12_RESOURCE) + ONEVPL_STRINGIFY_IF(MFX_RESOURCE_DMA_RESOURCE) + ONEVPL_STRINGIFY_IF(MFX_RESOURCE_HDDLUNITE_REMOTE_MEMORY) throw std::logic_error(std::string("Invalid \"decoder.Profiles.MemDesc.MemHandleType\":") + cstr); } mfxU32 cstr_to_mfx_codec_id(const char* cstr) { - if (!strcmp(cstr, "MFX_CODEC_AVC")) { - return MFX_CODEC_AVC; - } else if (!strcmp(cstr, "MFX_CODEC_HEVC")) { - return MFX_CODEC_HEVC; - } else if (!strcmp(cstr, "MFX_CODEC_MPEG2")) { - return MFX_CODEC_MPEG2; - } else if (!strcmp(cstr, "MFX_CODEC_VC1")) { - return MFX_CODEC_VC1; - } else if (!strcmp(cstr, "MFX_CODEC_CAPTURE")) { - return MFX_CODEC_CAPTURE; - } else if (!strcmp(cstr, "MFX_CODEC_VP9")) { - return MFX_CODEC_VP9; - } else if (!strcmp(cstr, "MFX_CODEC_AV1")) { - return MFX_CODEC_AV1; - } - throw std::logic_error(std::string("Cannot parse \"mfxImplDescription.mfxDecoderDescription.decoder.CodecID\" value: ") + cstr); + ONEVPL_STRINGIFY_IF(MFX_CODEC_AVC) + ONEVPL_STRINGIFY_IF(MFX_CODEC_HEVC) + ONEVPL_STRINGIFY_IF(MFX_CODEC_MPEG2) + ONEVPL_STRINGIFY_IF(MFX_CODEC_VC1) + ONEVPL_STRINGIFY_IF(MFX_CODEC_CAPTURE) + ONEVPL_STRINGIFY_IF(MFX_CODEC_VP9) + ONEVPL_STRINGIFY_IF(MFX_CODEC_AV1) + throw std::logic_error(std::string("Cannot parse \"") + CfgParam::decoder_id_name() + + "\" value: " + cstr); } const char* mfx_codec_id_to_cstr(mfxU32 mfx_id) { switch(mfx_id) { - case MFX_CODEC_AVC: - return "MFX_CODEC_AVC"; - case MFX_CODEC_HEVC: - return "MFX_CODEC_HEVC"; - case MFX_CODEC_MPEG2: - return "MFX_CODEC_MPEG2"; - case MFX_CODEC_VC1: - return "MFX_CODEC_VC1"; - case MFX_CODEC_VP9: - return "MFX_CODEC_VP9"; - case MFX_CODEC_AV1: - return "MFX_CODEC_AV1"; - case MFX_CODEC_JPEG: - return "MFX_CODEC_JPEG"; + ONEVPL_STRINGIFY_CASE(MFX_CODEC_AVC) + ONEVPL_STRINGIFY_CASE(MFX_CODEC_HEVC) + ONEVPL_STRINGIFY_CASE(MFX_CODEC_MPEG2) + ONEVPL_STRINGIFY_CASE(MFX_CODEC_VC1) + ONEVPL_STRINGIFY_CASE(MFX_CODEC_VP9) + ONEVPL_STRINGIFY_CASE(MFX_CODEC_AV1) + ONEVPL_STRINGIFY_CASE(MFX_CODEC_JPEG) default: return ""; } @@ -189,145 +144,92 @@ const char* mfx_codec_type_to_cstr(const mfxU32 fourcc, const mfxU32 type) { switch (fourcc) { case MFX_CODEC_JPEG: { switch (type) { - case MFX_PROFILE_UNKNOWN: - return "MFX_PROFILE_UNKNOWN"; - case MFX_PROFILE_JPEG_BASELINE: - return "MFX_PROFILE_JPEG_BASELINE"; - default: - return "(idesc.Version.Major) << "." << static_cast(idesc.Version.Minor) << std::endl; - out << "mfxImplDescription.Impl: " << mfx_impl_to_cstr(idesc.Impl) << std::endl; - out << "(*)mfxImplDescription.AccelerationMode: " << mfx_accel_mode_to_cstr(idesc.AccelerationMode) << std::endl; + out << "(*)" << CfgParam::implementation_name() << ": " << mfx_impl_to_cstr(idesc.Impl) << std::endl; + out << "(*)" << CfgParam::acceleration_mode_name() << ": " << mfx_accel_mode_to_cstr(idesc.AccelerationMode) << std::endl; out << "mfxImplDescription.ApiVersion: " << idesc.ApiVersion.Major << "." << idesc.ApiVersion.Minor << std::endl; out << "(*)mfxImplDescription.ApiVersion.Version: " << idesc.ApiVersion.Version << std::endl; out << "mfxImplDescription.ImplName: " << idesc.ImplName << std::endl; @@ -386,7 +288,7 @@ std::ostream& operator<< (std::ostream& out, const mfxImplDescription& idesc) << "." << static_cast(dec.Version.Minor) << std::endl; for (int codec = 0; codec < dec.NumCodecs; codec++) { auto cid = dec.Codecs[codec].CodecID; - out << "(*)mfxImplDescription.mfxDecoderDescription.decoder.CodecID: " << cid;//(cid & 0xff) << "." << (cid >> 8 & 0xff) << "." << (cid >> 16 & 0xff) << "." << (cid >> 24 & 0xff) << std::endl; + out << "(*)" << CfgParam::decoder_id_name() << ": " << cid;//(cid & 0xff) << "." << (cid >> 8 & 0xff) << "." << (cid >> 16 & 0xff) << "." << (cid >> 24 & 0xff) << std::endl; out << "mfxImplDescription.mfxDecoderDescription.decoder.MaxcodecLevel: " << dec.Codecs[codec].MaxcodecLevel << std::endl; for (int profile = 0; profile < dec.Codecs[codec].NumProfiles; profile++) { out << "mfxImplDescription.mfxDecoderDescription.decoder.Profiles: " @@ -422,69 +324,69 @@ std::string mfxstatus_to_string(int64_t err) { } std::string mfxstatus_to_string(mfxStatus err) { - switch(err) - { - case MFX_ERR_NONE: - return "MFX_ERR_NONE"; - case MFX_ERR_UNKNOWN: - return "MFX_ERR_UNKNOWN"; - case MFX_ERR_NULL_PTR: - return "MFX_ERR_NULL_PTR"; - case MFX_ERR_UNSUPPORTED: - return "MFX_ERR_UNSUPPORTED"; - case MFX_ERR_MEMORY_ALLOC: - return "MFX_ERR_MEMORY_ALLOC"; - case MFX_ERR_NOT_ENOUGH_BUFFER: - return "MFX_ERR_NOT_ENOUGH_BUFFER"; - case MFX_ERR_INVALID_HANDLE: - return "MFX_ERR_INVALID_HANDLE"; - case MFX_ERR_LOCK_MEMORY: - return "MFX_ERR_LOCK_MEMORY"; - case MFX_ERR_NOT_INITIALIZED: - return "MFX_ERR_NOT_INITIALIZED"; - case MFX_ERR_NOT_FOUND: - return "MFX_ERR_NOT_FOUND"; - case MFX_ERR_MORE_DATA: - return "MFX_ERR_MORE_DATA"; - case MFX_ERR_MORE_SURFACE: - return "MFX_ERR_MORE_SURFACE"; - case MFX_ERR_ABORTED: - return "MFX_ERR_ABORTED"; - case MFX_ERR_DEVICE_LOST: - return "MFX_ERR_DEVICE_LOST"; - case MFX_ERR_INCOMPATIBLE_VIDEO_PARAM: - return "MFX_ERR_INCOMPATIBLE_VIDEO_PARAM"; - case MFX_ERR_INVALID_VIDEO_PARAM: - return "MFX_ERR_INVALID_VIDEO_PARAM"; - case MFX_ERR_UNDEFINED_BEHAVIOR: - return "MFX_ERR_UNDEFINED_BEHAVIOR"; - case MFX_ERR_DEVICE_FAILED: - return "MFX_ERR_DEVICE_FAILED"; - case MFX_ERR_MORE_BITSTREAM: - return "MFX_ERR_MORE_BITSTREAM"; - case MFX_ERR_GPU_HANG: - return "MFX_ERR_GPU_HANG"; - case MFX_ERR_REALLOC_SURFACE: - return "MFX_ERR_REALLOC_SURFACE"; - case MFX_ERR_RESOURCE_MAPPED: - return "MFX_ERR_RESOURCE_MAPPED"; - case MFX_ERR_NOT_IMPLEMENTED: - return "MFX_ERR_NOT_IMPLEMENTED"; - case MFX_WRN_DEVICE_BUSY: - return "MFX_WRN_DEVICE_BUSY"; - case MFX_WRN_VIDEO_PARAM_CHANGED: - return "MFX_WRN_VIDEO_PARAM_CHANGED"; - case MFX_WRN_IN_EXECUTION: - return "MFX_WRN_IN_EXECUTION"; - - default: - break; + switch(err) { + ONEVPL_STRINGIFY_CASE(MFX_ERR_NONE) + ONEVPL_STRINGIFY_CASE(MFX_ERR_UNKNOWN) + ONEVPL_STRINGIFY_CASE(MFX_ERR_NULL_PTR) + ONEVPL_STRINGIFY_CASE(MFX_ERR_UNSUPPORTED) + ONEVPL_STRINGIFY_CASE(MFX_ERR_MEMORY_ALLOC) + ONEVPL_STRINGIFY_CASE(MFX_ERR_NOT_ENOUGH_BUFFER) + ONEVPL_STRINGIFY_CASE(MFX_ERR_INVALID_HANDLE) + ONEVPL_STRINGIFY_CASE(MFX_ERR_LOCK_MEMORY) + ONEVPL_STRINGIFY_CASE(MFX_ERR_NOT_INITIALIZED) + ONEVPL_STRINGIFY_CASE(MFX_ERR_NOT_FOUND) + ONEVPL_STRINGIFY_CASE(MFX_ERR_MORE_DATA) + ONEVPL_STRINGIFY_CASE(MFX_ERR_MORE_SURFACE) + ONEVPL_STRINGIFY_CASE(MFX_ERR_ABORTED) + ONEVPL_STRINGIFY_CASE(MFX_ERR_DEVICE_LOST) + ONEVPL_STRINGIFY_CASE(MFX_ERR_INCOMPATIBLE_VIDEO_PARAM) + ONEVPL_STRINGIFY_CASE(MFX_ERR_INVALID_VIDEO_PARAM) + ONEVPL_STRINGIFY_CASE(MFX_ERR_UNDEFINED_BEHAVIOR) + ONEVPL_STRINGIFY_CASE(MFX_ERR_DEVICE_FAILED) + ONEVPL_STRINGIFY_CASE(MFX_ERR_MORE_BITSTREAM) + ONEVPL_STRINGIFY_CASE(MFX_ERR_GPU_HANG) + ONEVPL_STRINGIFY_CASE(MFX_ERR_REALLOC_SURFACE) + ONEVPL_STRINGIFY_CASE(MFX_ERR_RESOURCE_MAPPED) + ONEVPL_STRINGIFY_CASE(MFX_ERR_NOT_IMPLEMENTED) + ONEVPL_STRINGIFY_CASE(MFX_WRN_DEVICE_BUSY) + ONEVPL_STRINGIFY_CASE(MFX_WRN_VIDEO_PARAM_CHANGED) + ONEVPL_STRINGIFY_CASE(MFX_WRN_IN_EXECUTION) + default: break; } std::string ret(""; return ret; } + +std::string ext_mem_frame_type_to_cstr(int type) { + std::stringstream ss; + APPEND_STRINGIFY_MASK_N_ERASE(type, "|", MFX_MEMTYPE_DXVA2_DECODER_TARGET); + APPEND_STRINGIFY_MASK_N_ERASE(type, "|", MFX_MEMTYPE_DXVA2_PROCESSOR_TARGET); + // NB: accoring to VPL source the commented MFX_* constane below are belong to the + // same actual integral value as condition abobe. So it is impossible + // to distinct them in condition branch. Just put this comment and possible + // constans here... + //APPEND_STRINGIFY_MASK_N_ERASE(type, "|", MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET); + //APPEND_STRINGIFY_MASK_N_ERASE(type, "|", MFX_MEMTYPE_VIDEO_MEMORY_PROCESSOR_TARGET); + APPEND_STRINGIFY_MASK_N_ERASE(type, "|", MFX_MEMTYPE_SYSTEM_MEMORY); + APPEND_STRINGIFY_MASK_N_ERASE(type, "|", MFX_MEMTYPE_RESERVED1); + APPEND_STRINGIFY_MASK_N_ERASE(type, "|", MFX_MEMTYPE_FROM_ENCODE); + APPEND_STRINGIFY_MASK_N_ERASE(type, "|", MFX_MEMTYPE_FROM_DECODE); + APPEND_STRINGIFY_MASK_N_ERASE(type, "|", MFX_MEMTYPE_FROM_VPPIN); + APPEND_STRINGIFY_MASK_N_ERASE(type, "|", MFX_MEMTYPE_FROM_VPPOUT); + APPEND_STRINGIFY_MASK_N_ERASE(type, "|", MFX_MEMTYPE_FROM_ENC); + APPEND_STRINGIFY_MASK_N_ERASE(type, "|", MFX_MEMTYPE_INTERNAL_FRAME); + APPEND_STRINGIFY_MASK_N_ERASE(type, "|", MFX_MEMTYPE_EXTERNAL_FRAME); + APPEND_STRINGIFY_MASK_N_ERASE(type, "|", MFX_MEMTYPE_EXPORT_FRAME); + //APPEND_STRINGIFY_MASK_N_ERASE(type, "|", MFX_MEMTYPE_SHARED_RESOURCE); + APPEND_STRINGIFY_MASK_N_ERASE(type, "|", MFX_MEMTYPE_VIDEO_MEMORY_ENCODER_TARGET); + + if (type != 0) { + ss << "(rest: " << std::to_string(type) << ")"; + } + return ss.str(); +} } // namespace onevpl } // namespace wip } // namespace gapi diff --git a/modules/gapi/src/streaming/onevpl/utils.hpp b/modules/gapi/src/streaming/onevpl/utils.hpp index 723ba16a98..36711bf9a0 100644 --- a/modules/gapi/src/streaming/onevpl/utils.hpp +++ b/modules/gapi/src/streaming/onevpl/utils.hpp @@ -8,12 +8,7 @@ #define GAPI_STREAMING_ONEVPL_ONEVPL_UTILS_HPP #ifdef HAVE_ONEVPL -#if (MFX_VERSION >= 2000) -#include -#endif // MFX_VERSION - -#include -#include +#include "streaming/onevpl/onevpl_export.hpp" #include #include @@ -42,11 +37,19 @@ void release(COMNonManageableType *ptr) { template using ComPtrGuard = std::unique_ptr)>; +template +using ComSharedPtrGuard = std::shared_ptr; + template ComPtrGuard createCOMPtrGuard(COMNonManageableType *ptr = nullptr) { return ComPtrGuard {ptr, &release}; } +template +ComSharedPtrGuard createCOMSharedPtrGuard(ComPtrGuard&& unique_guard) { + return ComSharedPtrGuard(std::move(unique_guard)); +} + const char* mfx_impl_to_cstr(const mfxIMPL impl); @@ -75,6 +78,7 @@ std::string mfxstatus_to_string(mfxStatus err); std::ostream& operator<< (std::ostream& out, const mfxImplDescription& idesc); +std::string ext_mem_frame_type_to_cstr(int type); } // namespace onevpl } // namespace wip } // namespace gapi diff --git a/modules/gapi/test/common/gapi_streaming_tests_common.hpp b/modules/gapi/test/common/gapi_streaming_tests_common.hpp new file mode 100644 index 0000000000..500371727e --- /dev/null +++ b/modules/gapi/test/common/gapi_streaming_tests_common.hpp @@ -0,0 +1,86 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef OPENCV_GAPI_STREAMING_TESTS_COMMON_HPP +#define OPENCV_GAPI_STREAMING_TESTS_COMMON_HPP + +#include "gapi_tests_common.hpp" +#include +#include +#include "streaming/onevpl/data_provider_defines.hpp" + +#ifdef HAVE_ONEVPL +#include "streaming/onevpl/onevpl_export.hpp" + +namespace opencv_test { +namespace streaming { +namespace onevpl { + +struct StreamDataProvider : public cv::gapi::wip::onevpl::IDataProvider { + + StreamDataProvider(std::istream& in) : data_stream (in) { + EXPECT_TRUE(in); + } + +mfx_codec_id_type get_mfx_codec_id() const override { + return MFX_CODEC_HEVC; + } + + bool fetch_bitstream_data(std::shared_ptr &out_bitstream) override { + if (empty()) { + return false; + } + + if (!out_bitstream) { + out_bitstream = std::make_shared(); + out_bitstream->MaxLength = 2000000; + out_bitstream->Data = (mfxU8 *)calloc(out_bitstream->MaxLength, sizeof(mfxU8)); + if(!out_bitstream->Data) { + throw std::runtime_error("Cannot allocate bitstream.Data bytes: " + + std::to_string(out_bitstream->MaxLength * sizeof(mfxU8))); + } + out_bitstream->CodecId = get_mfx_codec_id(); + } + + mfxU8 *p0 = out_bitstream->Data; + mfxU8 *p1 = out_bitstream->Data + out_bitstream->DataOffset; + EXPECT_FALSE(out_bitstream->DataOffset > out_bitstream->MaxLength - 1); + EXPECT_FALSE(out_bitstream->DataLength + out_bitstream->DataOffset > out_bitstream->MaxLength); + + std::copy_n(p1, out_bitstream->DataLength, p0); + + out_bitstream->DataOffset = 0; + out_bitstream->DataLength += static_cast(fetch_data(out_bitstream->MaxLength - out_bitstream->DataLength, + out_bitstream->Data + out_bitstream->DataLength)); + return out_bitstream->DataLength != 0; + } + + size_t fetch_data(size_t out_data_size, void* out_data_buf) { + data_stream.read(reinterpret_cast(out_data_buf), out_data_size); + return data_stream.gcount(); + } + bool empty() const override { + return data_stream.eof() || data_stream.bad(); + } +private: + std::istream& data_stream; +}; + +static const unsigned char hevc_header[] = { + 0x00, 0x00, 0x00, 0x01, 0x40, 0x01, 0x0C, 0x06, 0xFF, 0xFF, 0x01, 0x40, 0x00, + 0x00, 0x03, 0x00, 0x80, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, 0x00, + 0x00, 0x04, 0x02, 0x10, 0x30, 0x00, 0x00, 0x03, 0x00, 0x10, 0x00, 0x00, 0x03, + 0x01, 0xE5, 0x00, 0x00, 0x00, 0x01, 0x42, 0x01, 0x06, 0x01, 0x40, 0x00, 0x00, + 0x03, 0x00, 0x80, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, 0x00, 0x00, + 0xA0, 0x10, 0x20, 0x61, 0x63, 0x41, 0x00, 0x86, 0x49, 0x1B, 0x2B, 0x20, 0x00, + 0x00, 0x00, 0x01, 0x44, 0x01, 0xC0, 0x71, 0xC0, 0xD9, 0x20, 0x00, 0x00, 0x00, + 0x01, 0x26, 0x01, 0xAF, 0x0C +}; +} // namespace onevpl +} // namespace streaming +} // namespace opencv_test +#endif // HAVE_ONEVPL +#endif // OPENCV_GAPI_STREAMING_TESTS_HPP diff --git a/modules/gapi/test/streaming/gapi_streaming_tests.cpp b/modules/gapi/test/streaming/gapi_streaming_tests.cpp index fefd3e07e1..8cef807cd2 100644 --- a/modules/gapi/test/streaming/gapi_streaming_tests.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_tests.cpp @@ -7,7 +7,7 @@ #include "../test_precomp.hpp" -#include "../common/gapi_tests_common.hpp" +#include "../common/gapi_streaming_tests_common.hpp" #include // sleep_for (Delay) @@ -24,18 +24,8 @@ #include #include #include +#include -#include -#include "streaming/onevpl/data_provider_defines.hpp" - -#ifdef HAVE_ONEVPL - -#if (MFX_VERSION >= 2000) -#include -#endif - -#include -#endif // HAVE_ONEVPL namespace opencv_test { @@ -116,7 +106,7 @@ struct GAPI_Streaming: public ::testing::TestWithParam &out_bitstream) override { - if (empty()) { - return false; - } - - if (!out_bitstream) { - out_bitstream = std::make_shared(); - out_bitstream->MaxLength = 2000000; - out_bitstream->Data = (mfxU8 *)calloc(out_bitstream->MaxLength, sizeof(mfxU8)); - if(!out_bitstream->Data) { - throw std::runtime_error("Cannot allocate bitstream.Data bytes: " + - std::to_string(out_bitstream->MaxLength * sizeof(mfxU8))); - } - out_bitstream->CodecId = get_mfx_codec_id(); - } - - mfxU8 *p0 = out_bitstream->Data; - mfxU8 *p1 = out_bitstream->Data + out_bitstream->DataOffset; - EXPECT_FALSE(out_bitstream->DataOffset > out_bitstream->MaxLength - 1); - EXPECT_FALSE(out_bitstream->DataLength + out_bitstream->DataOffset > out_bitstream->MaxLength); - - std::copy_n(p1, out_bitstream->DataLength, p0); - - out_bitstream->DataOffset = 0; - out_bitstream->DataLength += static_cast(fetch_data(out_bitstream->MaxLength - out_bitstream->DataLength, - out_bitstream->Data + out_bitstream->DataLength)); - return out_bitstream->DataLength != 0; - } - - size_t fetch_data(size_t out_data_size, void* out_data_buf) { - data_stream.read(reinterpret_cast(out_data_buf), out_data_size); - return (size_t)data_stream.gcount(); - } - bool empty() const override { - return data_stream.eof() || data_stream.bad(); - } -private: - std::istream& data_stream; -}; -#endif // HAVE_ONEVPL } // anonymous namespace TEST_P(GAPI_Streaming, SmokeTest_ConstInput_GMat) @@ -2244,31 +2183,21 @@ TEST(GAPI_Streaming, TestPythonAPI) } #ifdef HAVE_ONEVPL -const unsigned char hevc_header[] = { - 0x00, 0x00, 0x00, 0x01, 0x40, 0x01, 0x0C, 0x06, 0xFF, 0xFF, 0x01, 0x40, 0x00, - 0x00, 0x03, 0x00, 0x80, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, 0x00, - 0x00, 0x04, 0x02, 0x10, 0x30, 0x00, 0x00, 0x03, 0x00, 0x10, 0x00, 0x00, 0x03, - 0x01, 0xE5, 0x00, 0x00, 0x00, 0x01, 0x42, 0x01, 0x06, 0x01, 0x40, 0x00, 0x00, - 0x03, 0x00, 0x80, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, 0x00, 0x00, - 0xA0, 0x10, 0x20, 0x61, 0x63, 0x41, 0x00, 0x86, 0x49, 0x1B, 0x2B, 0x20, 0x00, - 0x00, 0x00, 0x01, 0x44, 0x01, 0xC0, 0x71, 0xC0, 0xD9, 0x20, 0x00, 0x00, 0x00, - 0x01, 0x26, 0x01, 0xAF, 0x0C -}; + TEST(OneVPL_Source, Init) { using CfgParam = cv::gapi::wip::onevpl::CfgParam; std::vector src_params; - src_params.push_back(CfgParam::create("mfxImplDescription.Impl", - MFX_IMPL_TYPE_HARDWARE)); - src_params.push_back(CfgParam::create("mfxImplDescription.AccelerationMode", - MFX_ACCEL_MODE_VIA_D3D11, false)); - src_params.push_back(CfgParam::create("mfxImplDescription.mfxDecoderDescription.decoder.CodecID", - MFX_CODEC_HEVC)); + src_params.push_back(CfgParam::create_implementation(MFX_IMPL_TYPE_HARDWARE)); + src_params.push_back(CfgParam::create_acceleration_mode(MFX_ACCEL_MODE_VIA_D3D11)); + src_params.push_back(CfgParam::create_decoder_id(MFX_CODEC_HEVC)); std::stringstream stream(std::ios_base::in | std::ios_base::out | std::ios_base::binary); - EXPECT_TRUE(stream.write(reinterpret_cast(const_cast(hevc_header)), - sizeof(hevc_header))); - auto stream_data_provider = std::make_shared(stream); + + EXPECT_TRUE(stream.write(reinterpret_cast(const_cast(streaming::onevpl::hevc_header)), + sizeof(streaming::onevpl::hevc_header))); + std::shared_ptr stream_data_provider = + std::make_shared(stream); cv::Ptr cap; bool cap_created = false; @@ -2285,7 +2214,7 @@ TEST(OneVPL_Source, Init) } EXPECT_TRUE(stream_data_provider->empty()); } -#endif +#endif // HAVE_ONEVPL TEST(GAPI_Streaming, TestDesyncRMat) { cv::GMat in; diff --git a/modules/gapi/test/streaming/gapi_streaming_utils_test.cpp b/modules/gapi/test/streaming/gapi_streaming_utils_test.cpp new file mode 100644 index 0000000000..5599b8826f --- /dev/null +++ b/modules/gapi/test/streaming/gapi_streaming_utils_test.cpp @@ -0,0 +1,348 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + + +#include "../test_precomp.hpp" + +#include "../common/gapi_streaming_tests_common.hpp" + +#include +#include + +#define private public +#include "streaming/onevpl/accelerators/utils/shared_lock.hpp" +#undef private + +#include "streaming/onevpl/accelerators/utils/elastic_barrier.hpp" + +namespace opencv_test +{ +namespace +{ +using cv::gapi::wip::onevpl::SharedLock; + +struct TestBarrier : public cv::gapi::wip::onevpl::elastic_barrier { + void on_first_in_impl(size_t visitor_id) { + + static std::atomic thread_counter{}; + thread_counter++; + EXPECT_EQ(thread_counter.load(), 1); + + visitors_in.insert(visitor_id); + last_visitor_id = visitor_id; + + thread_counter--; + EXPECT_EQ(thread_counter.load(), 0); + } + + void on_last_out_impl(size_t visitor_id) { + + static std::atomic thread_counter{}; + thread_counter++; + EXPECT_EQ(thread_counter.load(), 1); + + visitors_out.insert(visitor_id); + last_visitor_id = visitor_id; + + thread_counter--; + EXPECT_EQ(thread_counter.load(), 0); + } + + size_t last_visitor_id = 0; + std::set visitors_in; + std::set visitors_out; +}; + +TEST(OneVPL_SharedLock, Create) { + SharedLock lock; + EXPECT_EQ(lock.shared_counter.load(), size_t{0}); +} + +TEST(OneVPL_SharedLock, Read_SingleThread) +{ + SharedLock lock; + + const size_t single_thread_read_count = 100; + for(size_t i = 0; i < single_thread_read_count; i++) { + lock.shared_lock(); + EXPECT_FALSE(lock.owns()); + } + EXPECT_EQ(lock.shared_counter.load(), single_thread_read_count); + + for(size_t i = 0; i < single_thread_read_count; i++) { + lock.unlock_shared(); + EXPECT_FALSE(lock.owns()); + } + + EXPECT_EQ(lock.shared_counter.load(), size_t{0}); +} + +TEST(OneVPL_SharedLock, TryLock_SingleThread) +{ + SharedLock lock; + + EXPECT_TRUE(lock.try_lock()); + EXPECT_TRUE(lock.owns()); + + lock.unlock(); + EXPECT_FALSE(lock.owns()); + EXPECT_EQ(lock.shared_counter.load(), size_t{0}); +} + +TEST(OneVPL_SharedLock, Write_SingleThread) +{ + SharedLock lock; + + lock.lock(); + EXPECT_TRUE(lock.owns()); + + lock.unlock(); + EXPECT_FALSE(lock.owns()); + EXPECT_EQ(lock.shared_counter.load(), size_t{0}); +} + +TEST(OneVPL_SharedLock, TryLockTryLock_SingleThread) +{ + SharedLock lock; + + lock.try_lock(); + EXPECT_FALSE(lock.try_lock()); + lock.unlock(); + + EXPECT_FALSE(lock.owns()); +} + +TEST(OneVPL_SharedLock, ReadTryLock_SingleThread) +{ + SharedLock lock; + + lock.shared_lock(); + EXPECT_FALSE(lock.owns()); + EXPECT_FALSE(lock.try_lock()); + lock.unlock_shared(); + + EXPECT_TRUE(lock.try_lock()); + EXPECT_TRUE(lock.owns()); + lock.unlock(); +} + +TEST(OneVPL_SharedLock, WriteTryLock_SingleThread) +{ + SharedLock lock; + + lock.lock(); + EXPECT_TRUE(lock.owns()); + EXPECT_FALSE(lock.try_lock()); + lock.unlock(); + + EXPECT_TRUE(lock.try_lock()); + EXPECT_TRUE(lock.owns()); + lock.unlock(); +} + + +TEST(OneVPL_SharedLock, Write_MultiThread) +{ + SharedLock lock; + + std::promise barrier; + std::shared_future sync = barrier.get_future(); + + static const size_t inc_count = 10000000; + size_t shared_value = 0; + auto work = [&lock, &shared_value](size_t count) { + for (size_t i = 0; i < count; i ++) { + lock.lock(); + shared_value ++; + lock.unlock(); + } + }; + + std::thread worker_thread([&barrier, sync, work] () { + + std::thread sub_worker([&barrier, work] () { + barrier.set_value(); + work(inc_count); + }); + + sync.wait(); + work(inc_count); + sub_worker.join(); + }); + sync.wait(); + + work(inc_count); + worker_thread.join(); + + EXPECT_EQ(shared_value, inc_count * 3); +} + +TEST(OneVPL_SharedLock, ReadWrite_MultiThread) +{ + SharedLock lock; + + std::promise barrier; + std::future sync = barrier.get_future(); + + static const size_t inc_count = 10000000; + size_t shared_value = 0; + auto write_work = [&lock, &shared_value](size_t count) { + for (size_t i = 0; i < count; i ++) { + lock.lock(); + shared_value ++; + lock.unlock(); + } + }; + + auto read_work = [&lock, &shared_value](size_t count) { + + auto old_shared_value = shared_value; + for (size_t i = 0; i < count; i ++) { + lock.shared_lock(); + EXPECT_TRUE(shared_value >= old_shared_value); + old_shared_value = shared_value; + lock.unlock_shared(); + } + }; + + std::thread writer_thread([&barrier, write_work] () { + barrier.set_value(); + write_work(inc_count); + }); + sync.wait(); + + read_work(inc_count); + writer_thread.join(); + + EXPECT_EQ(shared_value, inc_count); +} + + +TEST(OneVPL_ElasticBarrier, single_thread_visit) +{ + TestBarrier barrier; + + const size_t max_visit_count = 10000; + size_t visit_id = 0; + for (visit_id = 0; visit_id < max_visit_count; visit_id++) { + barrier.visit_in(visit_id); + EXPECT_EQ(barrier.visitors_in.size(), size_t{1}); + } + EXPECT_EQ(barrier.last_visitor_id, size_t{0}); + EXPECT_EQ(barrier.visitors_out.size(), size_t{0}); + + for (visit_id = 0; visit_id < max_visit_count; visit_id++) { + barrier.visit_out(visit_id); + EXPECT_EQ(barrier.visitors_in.size(), size_t{1}); + } + EXPECT_EQ(barrier.last_visitor_id, visit_id - 1); + EXPECT_EQ(barrier.visitors_out.size(), size_t{1}); +} + + +TEST(OneVPL_ElasticBarrier, multi_thread_visit) +{ + TestBarrier tested_barrier; + + static const size_t max_visit_count = 10000000; + std::atomic visit_in_wait_counter{}; + std::promise start_sync_barrier; + std::shared_future start_sync = start_sync_barrier.get_future(); + std::promise phase_sync_barrier; + std::shared_future phase_sync = phase_sync_barrier.get_future(); + + auto visit_worker_job = [&tested_barrier, + &visit_in_wait_counter, + start_sync, + phase_sync] (size_t worker_id) { + + start_sync.wait(); + + // first phase + const size_t begin_range = worker_id * max_visit_count; + const size_t end_range = (worker_id + 1) * max_visit_count; + for (size_t visit_id = begin_range; visit_id < end_range; visit_id++) { + tested_barrier.visit_in(visit_id); + } + + // notify all worker first phase ready + visit_in_wait_counter.fetch_add(1); + + // wait main second phase + phase_sync.wait(); + + // second phase + for (size_t visit_id = begin_range; visit_id < end_range; visit_id++) { + tested_barrier.visit_out(visit_id); + } + }; + + auto visit_main_job = [&tested_barrier, + &visit_in_wait_counter, + &phase_sync_barrier] (size_t total_workers_count, + size_t worker_id) { + + const size_t begin_range = worker_id * max_visit_count; + const size_t end_range = (worker_id + 1) * max_visit_count; + for (size_t visit_id = begin_range; visit_id < end_range; visit_id++) { + tested_barrier.visit_in(visit_id); + } + + // wait all workers first phase done + visit_in_wait_counter.fetch_add(1); + while (visit_in_wait_counter.load() != total_workers_count) { + std::this_thread::yield(); + }; + + // TEST invariant: last_visitor_id MUST be one from any FIRST worker visitor_id + bool one_of_available_ids_matched = false; + for (size_t id = 0; id < total_workers_count; id ++) { + size_t expected_last_visitor_for_id = id * max_visit_count; + one_of_available_ids_matched |= + (tested_barrier.last_visitor_id == expected_last_visitor_for_id) ; + } + EXPECT_TRUE(one_of_available_ids_matched); + + // unblock all workers to work out second phase + phase_sync_barrier.set_value(); + + // continue second phase + for (size_t visit_id = begin_range; visit_id < end_range; visit_id++) { + tested_barrier.visit_out(visit_id); + } + }; + + size_t max_worker_count = std::thread::hardware_concurrency(); + if (max_worker_count < 2) { + max_worker_count = 2; // logical 2 threads required at least + } + std::vector workers; + workers.reserve(max_worker_count); + for (size_t worker_id = 1; worker_id < max_worker_count; worker_id++) { + workers.emplace_back(visit_worker_job, worker_id); + } + + // let's go for first phase + start_sync_barrier.set_value(); + + // utilize main thread as well + visit_main_job(max_worker_count, 0); + + // join all threads second phase + for (auto& w : workers) { + w.join(); + } + + // TEST invariant: last_visitor_id MUST be one from any LATTER worker visitor_id + bool one_of_available_ids_matched = false; + for (size_t id = 0; id < max_worker_count; id ++) { + one_of_available_ids_matched |= + (tested_barrier.last_visitor_id == ((id + 1) * max_visit_count - 1)) ; + } + EXPECT_TRUE(one_of_available_ids_matched); +} +} +} // opencv_test diff --git a/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp b/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp index a84f92fafc..c62f58eecf 100644 --- a/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp @@ -7,7 +7,7 @@ #include "../test_precomp.hpp" -#include "../common/gapi_tests_common.hpp" +#include "../common/gapi_streaming_tests_common.hpp" #include #include @@ -27,14 +27,16 @@ #include #include -#include - #ifdef HAVE_ONEVPL #include +#include "streaming/onevpl/cfg_param_device_selector.hpp" #include "streaming/onevpl/accelerators/surface/surface.hpp" #include "streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp" #include "streaming/onevpl/accelerators/accel_policy_cpu.hpp" +#include "streaming/onevpl/accelerators/accel_policy_dx11.hpp" +#include "streaming/onevpl/accelerators/dx11_alloc_resource.hpp" +#include "streaming/onevpl/accelerators/utils/shared_lock.hpp" #include "streaming/onevpl/engine/processing_engine_base.hpp" #include "streaming/onevpl/engine/engine_session.hpp" @@ -60,6 +62,11 @@ struct TestProcessingSession : public cv::gapi::wip::onevpl::EngineSession { TestProcessingSession(mfxSession mfx_session) : EngineSession(mfx_session, {}) { } + + const mfxVideoParam& get_video_param() const override { + static mfxVideoParam empty; + return empty; + } }; struct TestProcessingEngine: public cv::gapi::wip::onevpl::ProcessingEngineBase { @@ -98,14 +105,66 @@ struct TestProcessingEngine: public cv::gapi::wip::onevpl::ProcessingEngineBase ); } - void initialize_session(mfxSession mfx_session, - cv::gapi::wip::onevpl::DecoderParams&&, - std::shared_ptr) override { + std::shared_ptr + initialize_session(mfxSession mfx_session, + const std::vector&, + std::shared_ptr) override { - register_session(mfx_session); + return register_session(mfx_session); } }; +template +class TestLockableAllocator { +public : + using self_t = TestLockableAllocator; + mfxFrameAllocator get() { + return m_allocator; + } +private: + TestLockableAllocator(mfxFrameAllocator allocator) : + m_allocator(allocator) { + } + + static mfxStatus MFX_CDECL lock_cb(mfxHDL, mfxMemId mid, mfxFrameData *ptr) { + auto it = lock_processor_table.find(mid); + EXPECT_TRUE(it != lock_processor_table.end()); + return it->second(mid, ptr); + } + static mfxStatus MFX_CDECL unlock_cb(mfxHDL, mfxMemId mid, mfxFrameData *ptr) { + auto it = unlock_processor_table.find(mid); + EXPECT_TRUE(it != unlock_processor_table.end()); + return it->second(mid, ptr); + } + + template + friend TestLockableAllocator create_test_allocator(mfxMemId, L, U); + + static std::map lock_processor_table; + static std::map unlock_processor_table; + + mfxFrameAllocator m_allocator; +}; +template +std::map TestLockableAllocator::lock_processor_table {}; + +template +std::map TestLockableAllocator::unlock_processor_table {}; + +template +TestLockableAllocator +create_test_allocator(mfxMemId mid, LockProcessor lock_p, UnlockProcessor unlock_p) { + mfxFrameAllocator allocator {}; + + TestLockableAllocator::lock_processor_table[mid] = lock_p; + allocator.Lock = &TestLockableAllocator::lock_cb; + + TestLockableAllocator::unlock_processor_table[mid] = unlock_p; + allocator.Unlock = &TestLockableAllocator::unlock_cb; + + return TestLockableAllocator {allocator}; +} + cv::gapi::wip::onevpl::surface_ptr_t create_test_surface(std::shared_ptr out_buf_ptr, size_t, size_t) { std::unique_ptr handle(new mfxFrameSurface1{}); @@ -262,8 +321,10 @@ TEST(OneVPL_Source_CPU_Accelerator, InitDestroy) { using cv::gapi::wip::onevpl::VPLCPUAccelerationPolicy; using cv::gapi::wip::onevpl::VPLAccelerationPolicy; + using cv::gapi::wip::onevpl::CfgParamDeviceSelector; - auto acceleration_policy = std::make_shared(); + auto acceleration_policy = + std::make_shared(std::make_shared()); size_t surface_count = 10; size_t surface_size_bytes = 1024; @@ -292,9 +353,11 @@ TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConsume) { using cv::gapi::wip::onevpl::VPLCPUAccelerationPolicy; using cv::gapi::wip::onevpl::VPLAccelerationPolicy; + using cv::gapi::wip::onevpl::CfgParamDeviceSelector; using cv::gapi::wip::onevpl::Surface; - auto acceleration_policy = std::make_shared(); + auto acceleration_policy = + std::make_shared(std::make_shared()); size_t surface_count = 10; size_t surface_size_bytes = 1024; @@ -348,9 +411,11 @@ TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConcurrentConsume) { using cv::gapi::wip::onevpl::VPLCPUAccelerationPolicy; using cv::gapi::wip::onevpl::VPLAccelerationPolicy; + using cv::gapi::wip::onevpl::CfgParamDeviceSelector; using cv::gapi::wip::onevpl::Surface; - auto acceleration_policy = std::make_shared(); + auto acceleration_policy = + std::make_shared(std::make_shared()); size_t surface_count = 10; size_t surface_size_bytes = 1024; @@ -416,7 +481,7 @@ TEST(OneVPL_Source_ProcessingEngine, Init) TestProcessingEngine engine(std::move(accel)); mfxSession mfx_session{}; - engine.initialize_session(mfx_session, DecoderParams{}, std::shared_ptr{}); + engine.initialize_session(mfx_session, {}, std::shared_ptr{}); EXPECT_EQ(0, engine.get_ready_frames_count()); ProcessingEngineBase::ExecutionStatus ret = engine.process(mfx_session); @@ -444,6 +509,181 @@ TEST(OneVPL_Source_ProcessingEngine, Init) cv::gapi::wip::Data frame; engine.get_frame(frame); } + +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 +TEST(OneVPL_Source_DX11_Accel, Init) +{ + using namespace cv::gapi::wip::onevpl; + + std::vector cfg_params_w_dx11; + cfg_params_w_dx11.push_back(CfgParam::create_acceleration_mode(MFX_ACCEL_MODE_VIA_D3D11)); + VPLDX11AccelerationPolicy accel(std::make_shared(cfg_params_w_dx11)); + + mfxLoader mfx_handle = MFXLoad(); + + mfxConfig cfg_inst_0 = MFXCreateConfig(mfx_handle); + EXPECT_TRUE(cfg_inst_0); + mfxVariant mfx_param_0; + mfx_param_0.Type = MFX_VARIANT_TYPE_U32; + mfx_param_0.Data.U32 = MFX_IMPL_TYPE_HARDWARE; + EXPECT_EQ(MFXSetConfigFilterProperty(cfg_inst_0,(mfxU8 *)CfgParam::implementation_name(), + mfx_param_0), MFX_ERR_NONE); + + mfxConfig cfg_inst_1 = MFXCreateConfig(mfx_handle); + EXPECT_TRUE(cfg_inst_1); + mfxVariant mfx_param_1; + mfx_param_1.Type = MFX_VARIANT_TYPE_U32; + mfx_param_1.Data.U32 = MFX_ACCEL_MODE_VIA_D3D11; + EXPECT_EQ(MFXSetConfigFilterProperty(cfg_inst_1,(mfxU8 *)CfgParam::acceleration_mode_name(), + mfx_param_1), MFX_ERR_NONE); + + mfxConfig cfg_inst_2 = MFXCreateConfig(mfx_handle); + EXPECT_TRUE(cfg_inst_2); + mfxVariant mfx_param_2; + mfx_param_2.Type = MFX_VARIANT_TYPE_U32; + mfx_param_2.Data.U32 = MFX_CODEC_HEVC; + EXPECT_EQ(MFXSetConfigFilterProperty(cfg_inst_2,(mfxU8 *)CfgParam::decoder_id_name(), + mfx_param_2), MFX_ERR_NONE); + + // create session + mfxSession mfx_session{}; + mfxStatus sts = MFXCreateSession(mfx_handle, 0, &mfx_session); + EXPECT_EQ(MFX_ERR_NONE, sts); + + // assign acceleration + EXPECT_NO_THROW(accel.init(mfx_session)); + + // create proper bitstream + mfxBitstream bitstream{}; + const int BITSTREAM_BUFFER_SIZE = 2000000; + bitstream.MaxLength = BITSTREAM_BUFFER_SIZE; + bitstream.Data = (mfxU8 *)calloc(bitstream.MaxLength, sizeof(mfxU8)); + EXPECT_TRUE(bitstream.Data); + + // simulate read stream + bitstream.DataOffset = 0; + bitstream.DataLength = sizeof(streaming::onevpl::hevc_header) * sizeof(streaming::onevpl::hevc_header[0]); + memcpy(bitstream.Data, streaming::onevpl::hevc_header, bitstream.DataLength); + bitstream.CodecId = MFX_CODEC_HEVC; + + // prepare dec params + mfxVideoParam mfxDecParams {}; + mfxDecParams.mfx.CodecId = bitstream.CodecId; + mfxDecParams.IOPattern = MFX_IOPATTERN_OUT_VIDEO_MEMORY; + sts = MFXVideoDECODE_DecodeHeader(mfx_session, &bitstream, &mfxDecParams); + EXPECT_EQ(MFX_ERR_NONE, sts); + + mfxFrameAllocRequest request{}; + memset(&request, 0, sizeof(request)); + sts = MFXVideoDECODE_QueryIOSurf(mfx_session, &mfxDecParams, &request); + EXPECT_EQ(MFX_ERR_NONE, sts); + + // Allocate surfaces for decoder + VPLAccelerationPolicy::pool_key_t key = accel.create_surface_pool(request, + mfxDecParams); + auto cand_surface = accel.get_free_surface(key).lock(); + + sts = MFXVideoDECODE_Init(mfx_session, &mfxDecParams); + EXPECT_EQ(MFX_ERR_NONE, sts); + + MFXVideoDECODE_Close(mfx_session); + EXPECT_EQ(MFX_ERR_NONE, sts); + + EXPECT_NO_THROW(accel.deinit(mfx_session)); + MFXClose(mfx_session); + MFXUnload(mfx_handle); +} +#endif // HAVE_DIRECTX +#endif // HAVE_D3D11 + +TEST(OneVPL_Source_DX11_FrameLockable, LockUnlock_without_Adaptee) +{ + using namespace cv::gapi::wip::onevpl; + mfxMemId mid = 0; + int lock_counter = 0; + int unlock_counter = 0; + + std::function lock = + [&lock_counter] (mfxMemId, mfxFrameData *) { + lock_counter ++; + return MFX_ERR_NONE; + }; + std::function unlock = + [&unlock_counter] (mfxMemId, mfxFrameData *) { + unlock_counter++; + return MFX_ERR_NONE; + }; + + auto test_allocator = create_test_allocator(mid, lock, unlock); + LockAdapter adapter(test_allocator.get()); + + mfxFrameData data; + const int exec_count = 123; + for (int i = 0; i < exec_count; i ++) { + EXPECT_EQ(adapter.read_lock(mid, data), 0); + adapter.write_lock(mid, data); + EXPECT_EQ(adapter.unlock_read(mid, data), 0); + adapter.unlock_write(mid, data); + } + + EXPECT_EQ(lock_counter, exec_count * 2); + EXPECT_EQ(unlock_counter, exec_count * 2); +} + +TEST(OneVPL_Source_DX11_FrameLockable, LockUnlock_with_Adaptee) +{ + using namespace cv::gapi::wip::onevpl; + mfxMemId mid = 0; + int r_lock_counter = 0; + int r_unlock_counter = 0; + int w_lock_counter = 0; + int w_unlock_counter = 0; + + SharedLock adaptee; + std::function lock = + [&r_lock_counter, &w_lock_counter, &adaptee] (mfxMemId, mfxFrameData *) { + if (adaptee.owns()) { + w_lock_counter ++; + } else { + r_lock_counter ++; + } + return MFX_ERR_NONE; + }; + std::function unlock = + [&r_unlock_counter, &w_unlock_counter, &adaptee] (mfxMemId, mfxFrameData *) { + if (adaptee.owns()) { + w_unlock_counter ++; + } else { + r_unlock_counter ++; + } + return MFX_ERR_NONE; + }; + + auto test_allocator = create_test_allocator(mid, lock, unlock); + LockAdapter adapter(test_allocator.get()); + + adapter.set_adaptee(&adaptee); + + mfxFrameData data; + const int exec_count = 123; + for (int i = 0; i < exec_count; i ++) { + EXPECT_EQ(adapter.read_lock(mid, data), 0); + EXPECT_FALSE(adaptee.try_lock()); + + EXPECT_EQ(adapter.unlock_read(mid, data), 1); + EXPECT_TRUE(adaptee.try_lock()); + adaptee.unlock(); + + adapter.write_lock(mid, data); + adapter.unlock_write(mid, data); + } + + EXPECT_EQ(r_lock_counter, exec_count); + EXPECT_EQ(r_unlock_counter, exec_count); + EXPECT_EQ(w_lock_counter, exec_count); + EXPECT_EQ(w_unlock_counter, exec_count); +} } } // namespace opencv_test #endif // HAVE_ONEVPL diff --git a/modules/gapi/test/streaming/gapi_streaming_vpl_data_provider.cpp b/modules/gapi/test/streaming/gapi_streaming_vpl_data_provider.cpp index 4e797ae9ef..c8c27fa6a4 100644 --- a/modules/gapi/test/streaming/gapi_streaming_vpl_data_provider.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_vpl_data_provider.cpp @@ -39,8 +39,6 @@ array_element_t files[] = { true, true, true}, array_element_t {"highgui/video/sample_322x242_15frames.yuv420p.libvpx-vp9.mp4", true, true, true}, - array_element_t {"highgui/video/sample_322x242_15frames.yuv420p.libx264.avi", - true, true, true}, array_element_t {"highgui/video/sample_322x242_15frames.yuv420p.libx264.mp4", true, true, true}, array_element_t {"highgui/video/sample_322x242_15frames.yuv420p.libx265.mp4", @@ -82,7 +80,7 @@ TEST_P(OneVPL_Source_MFPAsyncDispatcherTest, open_and_decode_file) mfxVariant mfx_param_0; mfx_param_0.Type = MFX_VARIANT_TYPE_U32; mfx_param_0.Data.U32 = provider_ptr->get_mfx_codec_id(); - EXPECT_EQ(MFXSetConfigFilterProperty(cfg_inst_0,(mfxU8 *)"mfxImplDescription.mfxDecoderDescription.decoder.CodecID", + EXPECT_EQ(MFXSetConfigFilterProperty(cfg_inst_0,(mfxU8 *)CfgParam::decoder_id_name(), mfx_param_0), MFX_ERR_NONE); // create MFX session @@ -135,7 +133,7 @@ TEST_P(OneVPL_Source_MFPAsyncDispatcherTest, choose_dmux_provider) EXPECT_FALSE(dd_result); provider_ptr = DataProviderDispatcher::create(path, { CfgParam::create( - "mfxImplDescription.mfxDecoderDescription.decoder.CodecID", + CfgParam::decoder_id_name(), "MFX_CODEC_HEVC") /* Doesn't matter what codec for RAW here*/}); EXPECT_TRUE(std::dynamic_pointer_cast(provider_ptr)); GTEST_SUCCEED(); diff --git a/modules/gapi/test/streaming/gapi_streaming_vpl_device_selector.cpp b/modules/gapi/test/streaming/gapi_streaming_vpl_device_selector.cpp index 2f42742b88..d484dcec75 100644 --- a/modules/gapi/test/streaming/gapi_streaming_vpl_device_selector.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_vpl_device_selector.cpp @@ -30,7 +30,7 @@ #endif // HAVE_DIRECTX #ifdef HAVE_ONEVPL -#include +#include "streaming/onevpl/onevpl_export.hpp" #include "streaming/onevpl/cfg_param_device_selector.hpp" namespace opencv_test @@ -94,8 +94,7 @@ TEST(OneVPL_Source_Device_Selector_CfgParam, DefaultDeviceWithAccelNACfgParam) { using namespace cv::gapi::wip::onevpl; std::vector cfg_params_w_no_accel; - cfg_params_w_no_accel.push_back(CfgParam::create("mfxImplDescription.AccelerationMode", - MFX_ACCEL_MODE_NA)); + cfg_params_w_no_accel.push_back(CfgParam::create_acceleration_mode(MFX_ACCEL_MODE_NA)); CfgParamDeviceSelector selector(cfg_params_w_no_accel); IDeviceSelector::DeviceScoreTable devs = selector.select_devices(); EXPECT_EQ(devs.size(), 1); @@ -126,8 +125,7 @@ TEST(OneVPL_Source_Device_Selector_CfgParam, DefaultDeviceWithDX11AccelCfgParam_ { using namespace cv::gapi::wip::onevpl; std::vector cfg_params_w_dx11; - cfg_params_w_dx11.push_back(CfgParam::create("mfxImplDescription.AccelerationMode", - MFX_ACCEL_MODE_VIA_D3D11)); + cfg_params_w_dx11.push_back(CfgParam::create_acceleration_mode(MFX_ACCEL_MODE_VIA_D3D11)); std::unique_ptr selector_ptr; EXPECT_NO_THROW(selector_ptr.reset(new CfgParamDeviceSelector(cfg_params_w_dx11))); IDeviceSelector::DeviceScoreTable devs = selector_ptr->select_devices(); @@ -146,8 +144,7 @@ TEST(OneVPL_Source_Device_Selector_CfgParam, NULLDeviceWithDX11AccelCfgParam_DX1 { using namespace cv::gapi::wip::onevpl; std::vector cfg_params_w_dx11; - cfg_params_w_dx11.push_back(CfgParam::create("mfxImplDescription.AccelerationMode", - MFX_ACCEL_MODE_VIA_D3D11)); + cfg_params_w_dx11.push_back(CfgParam::create_acceleration_mode(MFX_ACCEL_MODE_VIA_D3D11)); Device::Ptr empty_device_ptr = nullptr; Context::Ptr empty_ctx_ptr = nullptr; EXPECT_THROW(CfgParamDeviceSelector sel(empty_device_ptr, "GPU", @@ -179,8 +176,7 @@ TEST(OneVPL_Source_Device_Selector_CfgParam, ExternalDeviceWithDX11AccelCfgParam std::unique_ptr selector_ptr; std::vector cfg_params_w_dx11; - cfg_params_w_dx11.push_back(CfgParam::create("mfxImplDescription.AccelerationMode", - MFX_ACCEL_MODE_VIA_D3D11)); + cfg_params_w_dx11.push_back(CfgParam::create_acceleration_mode(MFX_ACCEL_MODE_VIA_D3D11)); EXPECT_NO_THROW(selector_ptr.reset(new CfgParamDeviceSelector(device, "GPU", device_context, cfg_params_w_dx11))); @@ -205,8 +201,7 @@ TEST(OneVPL_Source_Device_Selector_CfgParam, DX11DeviceFromCfgParamWithDX11Disab { using namespace cv::gapi::wip::onevpl; std::vector cfg_params_w_non_existed_dx11; - cfg_params_w_not_existed_dx11.push_back(CfgParam::create("mfxImplDescription.AccelerationMode", - MFX_ACCEL_MODE_VIA_D3D11)); + cfg_params_w_not_existed_dx11.push_back(CfgParam::create_acceleration_mode(MFX_ACCEL_MODE_VIA_D3D11)); EXPECT_THROW(CfgParamDeviceSelector{cfg_params_w_non_existed_dx11}, std::logic_error); } From 02ac6ec81c9b31b4783a2763a7565aaeafdd3abe Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 8 Dec 2021 10:45:02 +0000 Subject: [PATCH 144/226] cmake: option to disable GStreamer in G-API - OPENCV_GAPI_GSTREAMER --- modules/gapi/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index cc83606694..a98cfce6e7 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -300,7 +300,8 @@ if(HAVE_GAPI_ONEVPL) endif() endif() -if(HAVE_GSTREAMER) +ocv_option(OPENCV_GAPI_GSTREAMER "Build G-API with GStreamer support" HAVE_GSTREAMER) +if(HAVE_GSTREAMER AND OPENCV_GAPI_GSTREAMER) if(TARGET opencv_test_gapi) ocv_target_compile_definitions(opencv_test_gapi PRIVATE -DHAVE_GSTREAMER) ocv_target_link_libraries(opencv_test_gapi PRIVATE ocv.3rdparty.gstreamer) From d6891c705e269465222049510ef0920a47507ca9 Mon Sep 17 00:00:00 2001 From: Andrey Senyaev <76472231+asenyaev@users.noreply.github.com> Date: Wed, 8 Dec 2021 21:51:34 +0300 Subject: [PATCH 145/226] Merge pull request #21219 from asenyaev:asen/remove_distutils * Replaced distutils module to sysconfig * Fixed getting a path to python lib --- cmake/OpenCVDetectPython.cmake | 2 +- modules/python/common.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/OpenCVDetectPython.cmake b/cmake/OpenCVDetectPython.cmake index 4ff02a77d3..6e7bb18c1b 100644 --- a/cmake/OpenCVDetectPython.cmake +++ b/cmake/OpenCVDetectPython.cmake @@ -177,7 +177,7 @@ if(NOT ${found}) if(NOT ANDROID AND NOT IOS) if(CMAKE_HOST_UNIX) - execute_process(COMMAND ${_executable} -c "from distutils.sysconfig import *; print(get_python_lib())" + execute_process(COMMAND ${_executable} -c "from sysconfig import *; print(get_path('purelib'))" RESULT_VARIABLE _cvpy_process OUTPUT_VARIABLE _std_packages_path OUTPUT_STRIP_TRAILING_WHITESPACE) diff --git a/modules/python/common.cmake b/modules/python/common.cmake index ebbb2e2f65..301cda20b9 100644 --- a/modules/python/common.cmake +++ b/modules/python/common.cmake @@ -63,7 +63,7 @@ else() if("${${PYTHON}_VERSION_MAJOR}" STREQUAL "2") set(__python_ext_suffix_var "SO") endif() - execute_process(COMMAND ${${PYTHON}_EXECUTABLE} -c "import distutils.sysconfig; print(distutils.sysconfig.get_config_var('${__python_ext_suffix_var}'))" + execute_process(COMMAND ${${PYTHON}_EXECUTABLE} -c "import sysconfig; print(sysconfig.get_config_var('${__python_ext_suffix_var}'))" RESULT_VARIABLE PYTHON_CVPY_PROCESS OUTPUT_VARIABLE CVPY_SUFFIX OUTPUT_STRIP_TRAILING_WHITESPACE) From 6b4ea68bc76379de8b1ebcf3a2ed7574e1758738 Mon Sep 17 00:00:00 2001 From: Vincent Rabaud Date: Wed, 17 Nov 2021 17:38:21 +0100 Subject: [PATCH 146/226] Solve Rect overflow issues. Fow now, it is possible to define valid rectangle for which some functions overflow (e.g. br(), ares() ...). This patch fixes the intersection operator so that it works with any rectangle. --- modules/core/include/opencv2/core/types.hpp | 34 ++++++++++++++++----- modules/core/test/test_misc.cpp | 32 +++++++++++++++++++ 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/modules/core/include/opencv2/core/types.hpp b/modules/core/include/opencv2/core/types.hpp index 0b1d948156..82dedcb850 100644 --- a/modules/core/include/opencv2/core/types.hpp +++ b/modules/core/include/opencv2/core/types.hpp @@ -1895,13 +1895,33 @@ Rect_<_Tp>& operator -= ( Rect_<_Tp>& a, const Size_<_Tp>& b ) template static inline Rect_<_Tp>& operator &= ( Rect_<_Tp>& a, const Rect_<_Tp>& b ) { - _Tp x1 = std::max(a.x, b.x); - _Tp y1 = std::max(a.y, b.y); - a.width = std::min(a.x + a.width, b.x + b.width) - x1; - a.height = std::min(a.y + a.height, b.y + b.height) - y1; - a.x = x1; - a.y = y1; - if( a.width <= 0 || a.height <= 0 ) + if (a.empty() || b.empty()) { + a = Rect(); + return a; + } + const Rect_<_Tp>& Rx_min = (a.x < b.x) ? a : b; + const Rect_<_Tp>& Rx_max = (a.x < b.x) ? b : a; + const Rect_<_Tp>& Ry_min = (a.y < b.y) ? a : b; + const Rect_<_Tp>& Ry_max = (a.y < b.y) ? b : a; + // Looking at the formula below, we will compute Rx_min.width - (Rx_max.x - Rx_min.x) + // but we want to avoid overflows. Rx_min.width >= 0 and (Rx_max.x - Rx_min.x) >= 0 + // by definition so the difference does not overflow. The only thing that can overflow + // is (Rx_max.x - Rx_min.x). And it can only overflow if Rx_min.x < 0. + // Let us first deal with the following case. + if ((Rx_min.x < 0 && Rx_min.x + Rx_min.width < Rx_max.x) || + (Ry_min.y < 0 && Ry_min.y + Ry_min.height < Ry_max.y)) { + a = Rect(); + return a; + } + // We now know that either Rx_min.x >= 0, or + // Rx_min.x < 0 && Rx_min.x + Rx_min.width >= Rx_max.x and therefore + // Rx_min.width >= (Rx_max.x - Rx_min.x) which means (Rx_max.x - Rx_min.x) + // is inferior to a valid int and therefore does not overflow. + a.width = std::min(Rx_min.width - (Rx_max.x - Rx_min.x), Rx_max.width); + a.height = std::min(Ry_min.height - (Ry_max.y - Ry_min.y), Ry_max.height); + a.x = Rx_max.x; + a.y = Ry_max.y; + if (a.empty()) a = Rect(); return a; } diff --git a/modules/core/test/test_misc.cpp b/modules/core/test/test_misc.cpp index 372aab7eb0..d9e9119230 100644 --- a/modules/core/test/test_misc.cpp +++ b/modules/core/test/test_misc.cpp @@ -784,4 +784,36 @@ TEST(Core_Check, testSize_1) } +template class Rect_Test : public testing::Test {}; + +TYPED_TEST_CASE_P(Rect_Test); + +// Reimplement C++11 std::numeric_limits<>::lowest. +template T cv_numeric_limits_lowest(); +template<> int cv_numeric_limits_lowest() { return INT_MIN; } +template<> float cv_numeric_limits_lowest() { return -FLT_MAX; } +template<> double cv_numeric_limits_lowest() { return -DBL_MAX; } + +TYPED_TEST_P(Rect_Test, Overflows) { + typedef Rect_ R; + TypeParam num_max = std::numeric_limits::max(); + TypeParam num_lowest = cv_numeric_limits_lowest(); + EXPECT_EQ(R(0, 0, 10, 10), R(0, 0, 10, 10) & R(0, 0, 10, 10)); + EXPECT_EQ(R(5, 6, 4, 3), R(0, 0, 10, 10) & R(5, 6, 4, 3)); + EXPECT_EQ(R(5, 6, 3, 2), R(0, 0, 8, 8) & R(5, 6, 4, 3)); + // Test with overflowing dimenions. + EXPECT_EQ(R(5, 0, 5, 10), R(0, 0, 10, 10) & R(5, 0, num_max, num_max)); + // Test with overflowing dimensions for floats/doubles. + EXPECT_EQ(R(num_max, 0, num_max / 4, 10), R(num_max, 0, num_max / 2, 10) & R(num_max, 0, num_max / 4, 10)); + // Test with overflowing coordinates. + EXPECT_EQ(R(), R(20, 0, 10, 10) & R(num_lowest, 0, 10, 10)); + EXPECT_EQ(R(), R(20, 0, 10, 10) & R(0, num_lowest, 10, 10)); + EXPECT_EQ(R(), R(num_lowest, 0, 10, 10) & R(0, num_lowest, 10, 10)); +} +REGISTER_TYPED_TEST_CASE_P(Rect_Test, Overflows); + +typedef ::testing::Types RectTypes; +INSTANTIATE_TYPED_TEST_CASE_P(Negative_Test, Rect_Test, RectTypes); + + }} // namespace From 62a010a25d40b6f3d81960eed0f4279f33bc1827 Mon Sep 17 00:00:00 2001 From: UncleLLD Date: Fri, 10 Dec 2021 03:11:05 +0800 Subject: [PATCH 147/226] Merge pull request #21224 from UncleLLD:fix-cvtColor-error fix cvtColor-error * fix gray image channel error * fix gray image channel error * fix cvtColor error after the video end * fix cvtColor error after the video end and change next variable * fix cvtColor error after the video end * reset next variable * fix cvtColor error after the video end * fix cvtColor error after the video end --- .../video/optical_flow/optical_flow.py | 28 +++++++++++-------- .../video/optical_flow/optical_flow_dense.py | 28 +++++++++++-------- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/samples/python/tutorial_code/video/optical_flow/optical_flow.py b/samples/python/tutorial_code/video/optical_flow/optical_flow.py index 93bb2c421e..0e298e773a 100644 --- a/samples/python/tutorial_code/video/optical_flow/optical_flow.py +++ b/samples/python/tutorial_code/video/optical_flow/optical_flow.py @@ -17,12 +17,12 @@ feature_params = dict( maxCorners = 100, blockSize = 7 ) # Parameters for lucas kanade optical flow -lk_params = dict( winSize = (15,15), +lk_params = dict( winSize = (15, 15), maxLevel = 2, criteria = (cv.TERM_CRITERIA_EPS | cv.TERM_CRITERIA_COUNT, 10, 0.03)) # Create some random colors -color = np.random.randint(0,255,(100,3)) +color = np.random.randint(0, 255, (100, 3)) # Take first frame and find corners in it ret, old_frame = cap.read() @@ -33,7 +33,11 @@ p0 = cv.goodFeaturesToTrack(old_gray, mask = None, **feature_params) mask = np.zeros_like(old_frame) while(1): - ret,frame = cap.read() + ret, frame = cap.read() + if not ret: + print('No frames grabbed!') + break + frame_gray = cv.cvtColor(frame, cv.COLOR_BGR2GRAY) # calculate optical flow @@ -45,18 +49,20 @@ while(1): good_old = p0[st==1] # draw the tracks - for i,(new,old) in enumerate(zip(good_new, good_old)): - a,b = new.ravel() - c,d = old.ravel() - mask = cv.line(mask, (int(a),int(b)),(int(c),int(d)), color[i].tolist(), 2) - frame = cv.circle(frame,(int(a),int(b)),5,color[i].tolist(),-1) - img = cv.add(frame,mask) + for i, (new, old) in enumerate(zip(good_new, good_old)): + a, b = new.ravel() + c, d = old.ravel() + mask = cv.line(mask, (int(a), int(b)), (int(c), int(d)), color[i].tolist(), 2) + frame = cv.circle(frame, (int(a), int(b)), 5, color[i].tolist(), -1) + img = cv.add(frame, mask) - cv.imshow('frame',img) + cv.imshow('frame', img) k = cv.waitKey(30) & 0xff if k == 27: break # Now update the previous frame and previous points old_gray = frame_gray.copy() - p0 = good_new.reshape(-1,1,2) + p0 = good_new.reshape(-1, 1, 2) + +cv.destroyAllWindows() diff --git a/samples/python/tutorial_code/video/optical_flow/optical_flow_dense.py b/samples/python/tutorial_code/video/optical_flow/optical_flow_dense.py index b937b24ea7..8980c151c5 100644 --- a/samples/python/tutorial_code/video/optical_flow/optical_flow_dense.py +++ b/samples/python/tutorial_code/video/optical_flow/optical_flow_dense.py @@ -2,22 +2,28 @@ import numpy as np import cv2 as cv cap = cv.VideoCapture(cv.samples.findFile("vtest.avi")) ret, frame1 = cap.read() -prvs = cv.cvtColor(frame1,cv.COLOR_BGR2GRAY) +prvs = cv.cvtColor(frame1, cv.COLOR_BGR2GRAY) hsv = np.zeros_like(frame1) -hsv[...,1] = 255 +hsv[..., 1] = 255 while(1): ret, frame2 = cap.read() - next = cv.cvtColor(frame2,cv.COLOR_BGR2GRAY) - flow = cv.calcOpticalFlowFarneback(prvs,next, None, 0.5, 3, 15, 3, 5, 1.2, 0) - mag, ang = cv.cartToPolar(flow[...,0], flow[...,1]) - hsv[...,0] = ang*180/np.pi/2 - hsv[...,2] = cv.normalize(mag,None,0,255,cv.NORM_MINMAX) - bgr = cv.cvtColor(hsv,cv.COLOR_HSV2BGR) - cv.imshow('frame2',bgr) + if not ret: + print('No frames grabbed!') + break + + next = cv.cvtColor(frame2, cv.COLOR_BGR2GRAY) + flow = cv.calcOpticalFlowFarneback(prvs, next, None, 0.5, 3, 15, 3, 5, 1.2, 0) + mag, ang = cv.cartToPolar(flow[..., 0], flow[..., 1]) + hsv[..., 0] = ang*180/np.pi/2 + hsv[..., 2] = cv.normalize(mag, None, 0, 255, cv.NORM_MINMAX) + bgr = cv.cvtColor(hsv, cv.COLOR_HSV2BGR) + cv.imshow('frame2', bgr) k = cv.waitKey(30) & 0xff if k == 27: break elif k == ord('s'): - cv.imwrite('opticalfb.png',frame2) - cv.imwrite('opticalhsv.png',bgr) + cv.imwrite('opticalfb.png', frame2) + cv.imwrite('opticalhsv.png', bgr) prvs = next + +cv.destroyAllWindows() From 659cf7249e78453b7bb2cc4da3df2a5d19176cd5 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 9 Dec 2021 20:23:35 +0000 Subject: [PATCH 148/226] imgproc(ocl): fix resizeLN, avoid integer overflow --- modules/imgproc/src/opencl/resize.cl | 8 ++++---- modules/imgproc/src/resize.cpp | 3 ++- modules/imgproc/test/ocl/test_warp.cpp | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/modules/imgproc/src/opencl/resize.cl b/modules/imgproc/src/opencl/resize.cl index 67603e4f17..a28c59296e 100644 --- a/modules/imgproc/src/opencl/resize.cl +++ b/modules/imgproc/src/opencl/resize.cl @@ -51,8 +51,6 @@ #endif #endif -#define INTER_RESIZE_COEF_SCALE (1 << INTER_RESIZE_COEF_BITS) -#define CAST_BITS (INTER_RESIZE_COEF_BITS << 1) #define INC(x,l) min(x+1,l-1) #define noconvert @@ -188,7 +186,9 @@ __kernel void resizeLN(__global const uchar * srcptr, int src_step, int src_offs int y_ = INC(y, src_rows); int x_ = INC(x, src_cols); -#if depth <= 4 +#if depth <= 1 // 8U/8S only, 16U+ cause integer overflows +#define INTER_RESIZE_COEF_SCALE (1 << INTER_RESIZE_COEF_BITS) +#define CAST_BITS (INTER_RESIZE_COEF_BITS << 1) u = u * INTER_RESIZE_COEF_SCALE; v = v * INTER_RESIZE_COEF_SCALE; @@ -214,7 +214,7 @@ __kernel void resizeLN(__global const uchar * srcptr, int src_step, int src_offs WT data2 = convertToWT(loadpix(srcptr + mad24(y_, src_step, mad24(x, TSIZE, src_offset)))); WT data3 = convertToWT(loadpix(srcptr + mad24(y_, src_step, mad24(x_, TSIZE, src_offset)))); - T uval = u1 * v1 * data0 + u * v1 * data1 + u1 * v *data2 + u * v *data3; + T uval = convertToDT((u1 * v1) * data0 + (u * v1) * data1 + (u1 * v) * data2 + (u * v) * data3); #endif storepix(uval, dstptr + mad24(dy, dst_step, mad24(dx, TSIZE, dst_offset))); } diff --git a/modules/imgproc/src/resize.cpp b/modules/imgproc/src/resize.cpp index 4f1a4576ce..90a05085e3 100644 --- a/modules/imgproc/src/resize.cpp +++ b/modules/imgproc/src/resize.cpp @@ -3376,7 +3376,8 @@ static bool ocl_resize( InputArray _src, OutputArray _dst, Size dsize, } else { - int wdepth = std::max(depth, CV_32S), wtype = CV_MAKETYPE(wdepth, cn); + int wdepth = depth <= CV_8S ? CV_32S : std::max(depth, CV_32F); + int wtype = CV_MAKETYPE(wdepth, cn); k.create("resizeLN", ocl::imgproc::resize_oclsrc, format("-D INTER_LINEAR -D depth=%d -D T=%s -D T1=%s " "-D WT=%s -D convertToWT=%s -D convertToDT=%s -D cn=%d " diff --git a/modules/imgproc/test/ocl/test_warp.cpp b/modules/imgproc/test/ocl/test_warp.cpp index 15e024a140..b43c9b6732 100644 --- a/modules/imgproc/test/ocl/test_warp.cpp +++ b/modules/imgproc/test/ocl/test_warp.cpp @@ -327,6 +327,20 @@ OCL_TEST_P(Resize, Mat) } } +OCL_TEST(Resize, overflow_21198) +{ + Mat src(Size(600, 600), CV_16UC3, Scalar::all(32768)); + UMat src_u; + src.copyTo(src_u); + + Mat dst; + cv::resize(src, dst, Size(1024, 1024), 0, 0, INTER_LINEAR); + UMat dst_u; + cv::resize(src_u, dst_u, Size(1024, 1024), 0, 0, INTER_LINEAR); + EXPECT_LE(cv::norm(dst_u, dst, NORM_INF), 1.0f); +} + + ///////////////////////////////////////////////////////////////////////////////////////////////// // remap From b2005ccaef83532582c1a57f19a60052047963a7 Mon Sep 17 00:00:00 2001 From: Patrick Whalen Date: Thu, 2 Dec 2021 11:10:30 -0800 Subject: [PATCH 149/226] Fix broken build for Qt6 with options: WITH_QT=ON and WITH_OPENGL=ON - QGLWidget changed to QOpenGLWidget in window_QT.h for Qt6 using typedef OpenCVQtWidgetBase for handling Qt version - Implement Qt6/OpenGL functionality in window_QT.cpp - Swap QGLWidget:: function calls for OpenCVQtWidgetBase:: function calls - QGLWidget::updateGL deprecated, swap to QOpenGLWidget::update for Qt6 - Add preprocessor definition to detect Qt6 -- HAVE_QT6 - Add OpenGLWidgets to qdeps list in highgui CMakeLists.txt - find_package CMake command added for locating Qt module OpenGLWidgets - Added check that Qt6::OpenGLWidgets component is found. Shut off Qt-openGL functionality if not found. --- cmake/OpenCVFindLibsGUI.cmake | 7 +++++++ modules/highgui/CMakeLists.txt | 5 +++++ modules/highgui/src/window_QT.cpp | 23 ++++++++++++++--------- modules/highgui/src/window_QT.h | 18 ++++++++++++++++-- 4 files changed, 42 insertions(+), 11 deletions(-) diff --git a/cmake/OpenCVFindLibsGUI.cmake b/cmake/OpenCVFindLibsGUI.cmake index b33929e539..7224bddf90 100644 --- a/cmake/OpenCVFindLibsGUI.cmake +++ b/cmake/OpenCVFindLibsGUI.cmake @@ -47,6 +47,13 @@ if(WITH_QT) find_package(Qt${QT_VERSION_MAJOR} COMPONENTS OpenGL QUIET) if(Qt${QT_VERSION_MAJOR}OpenGL_FOUND) set(QT_QTOPENGL_FOUND ON) # HAVE_QT_OPENGL is defined below + if(QT_VERSION_MAJOR GREATER 5) # QGL -> QOpenGL + find_package(Qt${QT_VERSION_MAJOR} COMPONENTS OpenGLWidgets QUIET) + if(NOT Qt${QT_VERSION_MAJOR}OpenGLWidgets_FOUND) + message(STATUS "Qt OpenGLWidgets component not found: turning off Qt OpenGL functionality") + set(QT_QTOPENGL_FOUND FALSE) + endif() + endif() endif() endif() endif() diff --git a/modules/highgui/CMakeLists.txt b/modules/highgui/CMakeLists.txt index 9177c1ba46..27cfbb3672 100644 --- a/modules/highgui/CMakeLists.txt +++ b/modules/highgui/CMakeLists.txt @@ -60,6 +60,7 @@ if(HAVE_QT) set(CMAKE_INCLUDE_CURRENT_DIR ON) if(QT_VERSION_MAJOR EQUAL 6) + add_definitions(-DHAVE_QT6) # QGLWidget deprecated for QT6, use this preprocessor to adjust window_QT.[h,cpp] QT6_ADD_RESOURCES(_RCC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.qrc) QT6_WRAP_CPP(_MOC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.h) elseif(QT_VERSION_MAJOR EQUAL 5) @@ -78,6 +79,10 @@ if(HAVE_QT) set(qt_deps Core Gui Widgets Test Concurrent) if(HAVE_QT_OPENGL) add_definitions(-DHAVE_QT_OPENGL) + # QOpenGLWidget requires Qt6 package component OpenGLWidgets + if(QT_VERSION_MAJOR GREATER 5) + list(APPEND qt_deps OpenGLWidgets) + endif() list(APPEND qt_deps OpenGL) endif() diff --git a/modules/highgui/src/window_QT.cpp b/modules/highgui/src/window_QT.cpp index e3c831ab50..f6ba44b425 100644 --- a/modules/highgui/src/window_QT.cpp +++ b/modules/highgui/src/window_QT.cpp @@ -3227,7 +3227,9 @@ void DefaultViewPort::setSize(QSize /*size_*/) #ifdef HAVE_QT_OPENGL -OpenGlViewPort::OpenGlViewPort(QWidget* _parent) : QGLWidget(_parent), OCVViewPort(), size(-1, -1) + +// QOpenGLWidget vs QGLWidget info: https://www.qt.io/blog/2014/09/10/qt-weekly-19-qopenglwidget +OpenGlViewPort::OpenGlViewPort(QWidget* _parent) : OpenCVQtWidgetBase(_parent), OCVViewPort(), size(-1, -1) { glDrawCallback = 0; glDrawData = 0; @@ -3281,7 +3283,11 @@ void OpenGlViewPort::makeCurrentOpenGlContext() void OpenGlViewPort::updateGl() { + #ifdef HAVE_QT6 + QOpenGLWidget::update(); + #else QGLWidget::updateGL(); + #endif } void OpenGlViewPort::initializeGL() @@ -3308,31 +3314,31 @@ void OpenGlViewPort::paintGL() void OpenGlViewPort::wheelEvent(QWheelEvent* evnt) { icvmouseEvent((QMouseEvent *)evnt, mouse_wheel); - QGLWidget::wheelEvent(evnt); + OpenCVQtWidgetBase::wheelEvent(evnt); } void OpenGlViewPort::mousePressEvent(QMouseEvent* evnt) { icvmouseEvent(evnt, mouse_down); - QGLWidget::mousePressEvent(evnt); + OpenCVQtWidgetBase::mousePressEvent(evnt); } void OpenGlViewPort::mouseReleaseEvent(QMouseEvent* evnt) { icvmouseEvent(evnt, mouse_up); - QGLWidget::mouseReleaseEvent(evnt); + OpenCVQtWidgetBase::mouseReleaseEvent(evnt); } void OpenGlViewPort::mouseDoubleClickEvent(QMouseEvent* evnt) { icvmouseEvent(evnt, mouse_dbclick); - QGLWidget::mouseDoubleClickEvent(evnt); + OpenCVQtWidgetBase::mouseDoubleClickEvent(evnt); } void OpenGlViewPort::mouseMoveEvent(QMouseEvent* evnt) { icvmouseEvent(evnt, mouse_move); - QGLWidget::mouseMoveEvent(evnt); + OpenCVQtWidgetBase::mouseMoveEvent(evnt); } @@ -3340,8 +3346,7 @@ QSize OpenGlViewPort::sizeHint() const { if (size.width() > 0 && size.height() > 0) return size; - - return QGLWidget::sizeHint(); + return OpenCVQtWidgetBase::sizeHint(); } void OpenGlViewPort::setSize(QSize size_) @@ -3350,6 +3355,6 @@ void OpenGlViewPort::setSize(QSize size_) updateGeometry(); } -#endif +#endif //HAVE_QT_OPENGL #endif // HAVE_QT diff --git a/modules/highgui/src/window_QT.h b/modules/highgui/src/window_QT.h index 398f3869f8..b93b9ba597 100644 --- a/modules/highgui/src/window_QT.h +++ b/modules/highgui/src/window_QT.h @@ -48,7 +48,14 @@ #if defined( HAVE_QT_OPENGL ) #include -#include + + // QGLWidget deprecated and no longer functions with Qt6, use QOpenGLWidget instead + #ifdef HAVE_QT6 + #include + #else + #include + #endif + #endif #include @@ -431,7 +438,14 @@ protected: #ifdef HAVE_QT_OPENGL -class OpenGlViewPort : public QGLWidget, public OCVViewPort +// Use QOpenGLWidget for Qt6 (QGLWidget is deprecated) +#ifdef HAVE_QT6 +typedef QOpenGLWidget OpenCVQtWidgetBase; +#else +typedef QGLWidget OpenCVQtWidgetBase; +#endif + +class OpenGlViewPort : public OpenCVQtWidgetBase, public OCVViewPort { public: explicit OpenGlViewPort(QWidget* parent); From 4d3cf77ad50de8692e14103f0959d569e2d10b86 Mon Sep 17 00:00:00 2001 From: Andrey Senyaev Date: Fri, 10 Dec 2021 11:46:27 +0300 Subject: [PATCH 150/226] Replaced distutils to shutil when copying files in a tree --- modules/objc/generator/gen_objc.py | 8 +++++++- platforms/ios/build_framework.py | 7 ++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/modules/objc/generator/gen_objc.py b/modules/objc/generator/gen_objc.py index 2c41833f18..3e3ff8a2b0 100755 --- a/modules/objc/generator/gen_objc.py +++ b/modules/objc/generator/gen_objc.py @@ -9,7 +9,13 @@ import io from shutil import copyfile from pprint import pformat from string import Template -from distutils.dir_util import copy_tree + +if sys.version_info >= (3, 8): # Python 3.8+ + from shutil import copytree + def copy_tree(src, dst): + copytree(src, dst, dirs_exist_ok=True) +else: + from distutils.dir_util import copy_tree try: from io import StringIO # Python 3 diff --git a/platforms/ios/build_framework.py b/platforms/ios/build_framework.py index 0ce56321bf..a2b541b864 100755 --- a/platforms/ios/build_framework.py +++ b/platforms/ios/build_framework.py @@ -34,7 +34,12 @@ Adding --dynamic parameter will build {framework_name}.framework as App Store dy from __future__ import print_function, unicode_literals import glob, os, os.path, shutil, string, sys, argparse, traceback, multiprocessing, codecs, io from subprocess import check_call, check_output, CalledProcessError -from distutils.dir_util import copy_tree + +if sys.version_info >= (3, 8): # Python 3.8+ + def copy_tree(src, dst): + shutil.copytree(src, dst, dirs_exist_ok=True) +else: + from distutils.dir_util import copy_tree sys.path.insert(0, os.path.abspath(os.path.abspath(os.path.dirname(__file__))+'/../apple')) from cv_build_utils import execute, print_error, get_xcode_major, get_xcode_setting, get_xcode_version, get_cmake_version From b70370f3fdb39b08c8be3c22aafbb77a7063bdba Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Fri, 10 Dec 2021 11:58:59 +0300 Subject: [PATCH 151/226] Merge pull request #21230 from sivanov-work:gapi_win32_vpl_fix G-API: Fix Win32 build: uint64_t ->size_t * Fix Win32 build: uint64_t ->size_t * Fix for MAC --- .../include/opencv2/gapi/streaming/onevpl/cfg_params.hpp | 2 +- modules/gapi/samples/onevpl_infer_single_roi.cpp | 4 ++-- modules/gapi/src/streaming/onevpl/cfg_params.cpp | 8 ++++++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/cfg_params.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/cfg_params.hpp index 0f35200f49..bfd922496a 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/onevpl/cfg_params.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/cfg_params.hpp @@ -61,7 +61,7 @@ struct GAPI_EXPORTS CfgParam { * */ static constexpr const char *frames_pool_size_name() { return "frames_pool_size"; } - static CfgParam create_frames_pool_size(uint64_t value); + static CfgParam create_frames_pool_size(size_t value); /** * @brief acceleration_mode_name diff --git a/modules/gapi/samples/onevpl_infer_single_roi.cpp b/modules/gapi/samples/onevpl_infer_single_roi.cpp index 1ab06de739..06950bcabe 100644 --- a/modules/gapi/samples/onevpl_infer_single_roi.cpp +++ b/modules/gapi/samples/onevpl_infer_single_roi.cpp @@ -196,8 +196,8 @@ int main(int argc, char *argv[]) { std::string file_path = cmd.get("input"); const std::string output = cmd.get("output"); const auto face_model_path = cmd.get("facem"); - const auto streaming_queue_capacity = cmd.get("streaming_queue_capacity"); - const auto source_queue_capacity = cmd.get("frames_pool_size"); + const auto streaming_queue_capacity = cmd.get("streaming_queue_capacity"); + const auto source_queue_capacity = cmd.get("frames_pool_size"); // check ouput file extension if (!output.empty()) { diff --git a/modules/gapi/src/streaming/onevpl/cfg_params.cpp b/modules/gapi/src/streaming/onevpl/cfg_params.cpp index 97458faed3..599f751358 100644 --- a/modules/gapi/src/streaming/onevpl/cfg_params.cpp +++ b/modules/gapi/src/streaming/onevpl/cfg_params.cpp @@ -86,8 +86,12 @@ CfgParam::CfgParam (const std::string& param_name, value_t&& param_value, bool i CfgParam::~CfgParam() = default; -CfgParam CfgParam::create_frames_pool_size(uint64_t value) { - return CfgParam::create(CfgParam::frames_pool_size_name(), value, false); +CfgParam CfgParam::create_frames_pool_size(size_t value) { + // NB: cast to uint64_t because CfgParam inner variant works over + // uint64_t instead of size_t and mirrored VPL types variety + // but size_t looks more friendly for C++ high-level development + return CfgParam::create(CfgParam::frames_pool_size_name(), + static_cast(value), false); } CfgParam CfgParam::create_acceleration_mode(uint32_t value) { From c08954c18b54897d2cea2903d0365a2336b1b7f5 Mon Sep 17 00:00:00 2001 From: Simon Wilson Date: Fri, 10 Dec 2021 22:36:14 +1100 Subject: [PATCH 152/226] Merge pull request #21227 from sbwilson:fix_framework_unicode_headers * fix unicode errors for framework headers This would crash if the header file included non-ASCII characters. This change ensures that headers are read and written as UTF-8 encoded files instead of ascii. * Adds spaces after commas --- platforms/ios/build_framework.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platforms/ios/build_framework.py b/platforms/ios/build_framework.py index 0ce56321bf..09ac0197d8 100755 --- a/platforms/ios/build_framework.py +++ b/platforms/ios/build_framework.py @@ -406,11 +406,11 @@ class Builder: for dirname, dirs, files in os.walk(os.path.join(dstdir, "Headers")): for filename in files: filepath = os.path.join(dirname, filename) - with open(filepath) as file: + with codecs.open(filepath, "r", "utf-8") as file: body = file.read() body = body.replace("include \"opencv2/", "include \"" + name + "/") body = body.replace("include Date: Fri, 10 Dec 2021 19:44:27 +0800 Subject: [PATCH 153/226] Merge pull request #21154 from pccvlab:MatMul_with_two_inputs Add BatchMatMul layer support for tf_importer * two inputs * support batch_matmul * refactor: remove useless code * refactor: decrease nesting --- modules/dnn/src/tensorflow/tf_importer.cpp | 20 +++++++++++++++++++- modules/dnn/test/test_tf_importer.cpp | 8 ++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index 9fb8f60b41..5fafa2b9d5 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -646,7 +646,7 @@ const TFImporter::DispatchMap TFImporter::buildDispatchMap() dispatch["Conv2D"] = dispatch["SpaceToBatchND"] = dispatch["DepthwiseConv2dNative"] = dispatch["Pad"] = dispatch["MirrorPad"] = dispatch["Conv3D"] = &TFImporter::parseConvolution; dispatch["BiasAdd"] = dispatch["Add"] = dispatch["AddV2"] = dispatch["Sub"] = dispatch["AddN"] = &TFImporter::parseBias; - dispatch["MatMul"] = &TFImporter::parseMatMul; + dispatch["MatMul"] = dispatch["BatchMatMul"] = &TFImporter::parseMatMul; dispatch["Reshape"] = &TFImporter::parseReshape; dispatch["Flatten"] = dispatch["Squeeze"] = &TFImporter::parseFlatten; dispatch["Transpose"] = &TFImporter::parseTranspose; @@ -983,6 +983,24 @@ void TFImporter::parseMatMul(tensorflow::GraphDef& net, const tensorflow::NodeDe layerParams.set("bias_term", false); layerParams.blobs.resize(1); + bool hasConstBlob = false; + for(int i = 0; i < layer.input_size(); i++) { + if (value_id.find(layer.input(i)) != value_id.end()) + hasConstBlob = true; + } + if (!hasConstBlob) + { + layerParams.blobs.clear(); + int id = dstNet.addLayer(name, "InnerProduct", layerParams); + layer_id[name] = id; + + // two inputs + for(int ii=0; ii Date: Mon, 6 Dec 2021 09:54:48 +0300 Subject: [PATCH 154/226] videoio(MSMF): add queue for async ReadSample() --- modules/videoio/src/cap_msmf.cpp | 89 +++++++++++++++++++--------- modules/videoio/test/test_camera.cpp | 15 ++++- 2 files changed, 74 insertions(+), 30 deletions(-) diff --git a/modules/videoio/src/cap_msmf.cpp b/modules/videoio/src/cap_msmf.cpp index 68171ea7af..d78236913b 100644 --- a/modules/videoio/src/cap_msmf.cpp +++ b/modules/videoio/src/cap_msmf.cpp @@ -32,6 +32,7 @@ #endif #include #include +#include #include #include #include @@ -395,8 +396,10 @@ private: class SourceReaderCB : public IMFSourceReaderCallback { public: + static const size_t MSMF_READER_MAX_QUEUE_SIZE = 3; + SourceReaderCB() : - m_nRefCount(0), m_hEvent(CreateEvent(NULL, FALSE, FALSE, NULL)), m_bEOS(FALSE), m_hrStatus(S_OK), m_reader(NULL), m_dwStreamIndex(0), m_lastSampleTimestamp(0) + m_nRefCount(0), m_hEvent(CreateEvent(NULL, FALSE, FALSE, NULL)), m_bEOS(FALSE), m_hrStatus(S_OK), m_reader(NULL), m_dwStreamIndex(0) { } @@ -441,12 +444,19 @@ public: if (pSample) { CV_LOG_DEBUG(NULL, "videoio(MSMF): got frame at " << llTimestamp); - if (m_lastSample.Get()) + if (m_capturedFrames.size() >= MSMF_READER_MAX_QUEUE_SIZE) { - CV_LOG_DEBUG(NULL, "videoio(MSMF): drop frame (not processed)"); +#if 0 + CV_LOG_DEBUG(NULL, "videoio(MSMF): drop frame (not processed). Timestamp=" << m_capturedFrames.front().timestamp); + m_capturedFrames.pop(); +#else + // this branch reduces latency if we drop frames due to slow processing. + // avoid fetching of already outdated frames from the queue's front. + CV_LOG_DEBUG(NULL, "videoio(MSMF): drop previous frames (not processed): " << m_capturedFrames.size()); + std::queue().swap(m_capturedFrames); // similar to missing m_capturedFrames.clean(); +#endif } - m_lastSampleTimestamp = llTimestamp; - m_lastSample = pSample; + m_capturedFrames.emplace(CapturedFrameInfo{ llTimestamp, _ComPtr(pSample), hrStatus }); } } else @@ -483,30 +493,43 @@ public: return S_OK; } - HRESULT Wait(DWORD dwMilliseconds, _ComPtr& mediaSample, BOOL& pbEOS) + HRESULT Wait(DWORD dwMilliseconds, _ComPtr& mediaSample, LONGLONG& sampleTimestamp, BOOL& pbEOS) { pbEOS = FALSE; - DWORD dwResult = WaitForSingleObject(m_hEvent, dwMilliseconds); - if (dwResult == WAIT_TIMEOUT) + for (;;) { - return E_PENDING; - } - else if (dwResult != WAIT_OBJECT_0) - { - return HRESULT_FROM_WIN32(GetLastError()); - } + { + cv::AutoLock lock(m_mutex); - pbEOS = m_bEOS; - if (!pbEOS) - { - cv::AutoLock lock(m_mutex); - mediaSample = m_lastSample; - CV_Assert(mediaSample); - m_lastSample.Release(); - ResetEvent(m_hEvent); // event is auto-reset, but we need this forced reset due time gap between wait() and mutex hold. + pbEOS = m_bEOS && m_capturedFrames.empty(); + if (pbEOS) + return m_hrStatus; + + if (!m_capturedFrames.empty()) + { + CV_Assert(!m_capturedFrames.empty()); + CapturedFrameInfo frameInfo = m_capturedFrames.front(); m_capturedFrames.pop(); + CV_LOG_DEBUG(NULL, "videoio(MSMF): handle frame at " << frameInfo.timestamp); + mediaSample = frameInfo.sample; + CV_Assert(mediaSample); + sampleTimestamp = frameInfo.timestamp; + ResetEvent(m_hEvent); // event is auto-reset, but we need this forced reset due time gap between wait() and mutex hold. + return frameInfo.hrStatus; + } + } + + CV_LOG_DEBUG(NULL, "videoio(MSMF): waiting for frame... "); + DWORD dwResult = WaitForSingleObject(m_hEvent, dwMilliseconds); + if (dwResult == WAIT_TIMEOUT) + { + return E_PENDING; + } + else if (dwResult != WAIT_OBJECT_0) + { + return HRESULT_FROM_WIN32(GetLastError()); + } } - return m_hrStatus; } private: @@ -525,8 +548,14 @@ public: IMFSourceReader *m_reader; DWORD m_dwStreamIndex; - LONGLONG m_lastSampleTimestamp; - _ComPtr m_lastSample; + + struct CapturedFrameInfo { + LONGLONG timestamp; + _ComPtr sample; + HRESULT hrStatus; + }; + + std::queue m_capturedFrames; }; //================================================================================================== @@ -1569,7 +1598,6 @@ bool CvCapture_MSMF::configureAudioFrame() } audioDataInUse.clear(); audioDataInUse.shrink_to_fit(); - return true; } else @@ -1690,6 +1718,7 @@ bool CvCapture_MSMF::grabFrame() if (grabIsDone) { grabIsDone = false; + CV_LOG_DEBUG(NULL, "videoio(MSMF): return pre-grabbed frame " << usedVideoSampleTime); return true; } @@ -1717,7 +1746,8 @@ bool CvCapture_MSMF::grabFrame() } } BOOL bEOS = false; - if (FAILED(hr = reader->Wait( videoStream == -1 ? INFINITE : 10000, (videoStream != -1) ? usedVideoSample : audioSamples[0], bEOS))) // 10 sec + LONGLONG timestamp = 0; + if (FAILED(hr = reader->Wait( videoStream == -1 ? INFINITE : 10000, (videoStream != -1) ? usedVideoSample : audioSamples[0], timestamp, bEOS))) // 10 sec { CV_LOG_WARNING(NULL, "videoio(MSMF): can't grab frame. Error: " << hr); return false; @@ -1728,10 +1758,11 @@ bool CvCapture_MSMF::grabFrame() return false; } if (videoStream != -1) - usedVideoSampleTime = reader->m_lastSampleTimestamp; + usedVideoSampleTime = timestamp; if (audioStream != -1) return configureAudioFrame(); + CV_LOG_DEBUG(NULL, "videoio(MSMF): grabbed frame " << usedVideoSampleTime); return true; } else if (isOpen) @@ -1765,6 +1796,7 @@ bool CvCapture_MSMF::grabFrame() bool CvCapture_MSMF::retrieveVideoFrame(cv::OutputArray frame) { CV_TRACE_FUNCTION(); + CV_LOG_DEBUG(NULL, "videoio(MSMF): retrieve video frame start..."); do { if (!usedVideoSample) @@ -1855,6 +1887,7 @@ bool CvCapture_MSMF::retrieveVideoFrame(cv::OutputArray frame) buffer2d->Unlock2D(); else buf->Unlock(); + CV_LOG_DEBUG(NULL, "videoio(MSMF): retrieve video frame done!"); return !frame.empty(); } while (0); diff --git a/modules/videoio/test/test_camera.cpp b/modules/videoio/test/test_camera.cpp index c8a5598dcc..fc269959c3 100644 --- a/modules/videoio/test/test_camera.cpp +++ b/modules/videoio/test/test_camera.cpp @@ -26,15 +26,26 @@ static void test_readFrames(/*const*/ VideoCapture& capture, const int N = 100, const bool validTickAndFps = cvTickFreq != 0 && fps != 0.; testTimestamps &= validTickAndFps; + double frame0ts = 0; + for (int i = 0; i < N; i++) { SCOPED_TRACE(cv::format("frame=%d", i)); capture >> frame; - const int64 sysTimeCurr = cv::getTickCount(); - const double camTimeCurr = capture.get(cv::CAP_PROP_POS_MSEC); ASSERT_FALSE(frame.empty()); + const int64 sysTimeCurr = cv::getTickCount(); + double camTimeCurr = capture.get(cv::CAP_PROP_POS_MSEC); + if (i == 0) + frame0ts = camTimeCurr; + camTimeCurr -= frame0ts; // normalized timestamp based on the first frame + + if (cvtest::debugLevel > 0) + { + std::cout << i << ": " << camTimeCurr << std::endl; + } + // Do we have a previous frame? if (i > 0 && testTimestamps) { From 1599f9f0c06424d8b851437c2b85d46085df3da6 Mon Sep 17 00:00:00 2001 From: HAN Liutong Date: Sat, 11 Dec 2021 00:03:22 +0800 Subject: [PATCH 155/226] Merge pull request #21086 from hanliutong:rvv-dnn Further optimize DNN for RISC-V Vector. * Optimize DNN on RVV by using vsetvl. * Rename vl. * Update fastConv by using setvl instead of mask. * Fix fastDepthwiseConv --- modules/dnn/src/layers/layers_common.simd.hpp | 394 ++++++++---------- 1 file changed, 172 insertions(+), 222 deletions(-) diff --git a/modules/dnn/src/layers/layers_common.simd.hpp b/modules/dnn/src/layers/layers_common.simd.hpp index 66cfe3e0ff..67a4b3c065 100644 --- a/modules/dnn/src/layers/layers_common.simd.hpp +++ b/modules/dnn/src/layers/layers_common.simd.hpp @@ -782,15 +782,10 @@ void fastGEMM( const float* aptr, size_t astep, const float* bptr, size_t bstep, float* cptr, size_t cstep, int ma, int na, int nb ) { - int n = 0; - int vl = vsetvlmax_e32m4(); - int mvl = vl; - for( ; n < nb; n += vl ) + int avl = nb, vl; + for(int n = 0; n < nb; n += vl, avl -= vl) { - if ( n + vl > nb) { - mvl = nb - n; - } - + vl = vsetvl_e32m4(avl); for( int m = 0; m < ma; m += 7 ) { const float* aptr0 = aptr + astep*m; @@ -827,22 +822,22 @@ void fastGEMM( const float* aptr, size_t astep, const float* bptr, float a5 = aptr5[k]; float a6 = aptr6[k]; - vfloat32m4_t b = vle32_v_f32m4(bptr + k*bstep + n, mvl); - d0 = vfmacc_vf_f32m4(d0, a0, b, mvl); - d1 = vfmacc_vf_f32m4(d1, a1, b, mvl); - d2 = vfmacc_vf_f32m4(d2, a2, b, mvl); - d3 = vfmacc_vf_f32m4(d3, a3, b, mvl); - d4 = vfmacc_vf_f32m4(d4, a4, b, mvl); - d5 = vfmacc_vf_f32m4(d5, a5, b, mvl); - d6 = vfmacc_vf_f32m4(d6, a6, b, mvl); + vfloat32m4_t b = vle32_v_f32m4(bptr + k*bstep + n, vl); + d0 = vfmacc_vf_f32m4(d0, a0, b, vl); + d1 = vfmacc_vf_f32m4(d1, a1, b, vl); + d2 = vfmacc_vf_f32m4(d2, a2, b, vl); + d3 = vfmacc_vf_f32m4(d3, a3, b, vl); + d4 = vfmacc_vf_f32m4(d4, a4, b, vl); + d5 = vfmacc_vf_f32m4(d5, a5, b, vl); + d6 = vfmacc_vf_f32m4(d6, a6, b, vl); } - vse32_v_f32m4(cptr0 + n, d0, mvl); - vse32_v_f32m4(cptr1 + n, d1, mvl); - vse32_v_f32m4(cptr2 + n, d2, mvl); - vse32_v_f32m4(cptr3 + n, d3, mvl); - vse32_v_f32m4(cptr4 + n, d4, mvl); - vse32_v_f32m4(cptr5 + n, d5, mvl); - vse32_v_f32m4(cptr6 + n, d6, mvl); + vse32_v_f32m4(cptr0 + n, d0, vl); + vse32_v_f32m4(cptr1 + n, d1, vl); + vse32_v_f32m4(cptr2 + n, d2, vl); + vse32_v_f32m4(cptr3 + n, d3, vl); + vse32_v_f32m4(cptr4 + n, d4, vl); + vse32_v_f32m4(cptr5 + n, d5, vl); + vse32_v_f32m4(cptr6 + n, d6, vl); } } } @@ -851,7 +846,7 @@ void fastGEMM1T( const float* vec, const float* weights, size_t wstep, const float* bias, float* dst, int nvecs, int vecsize ) { - int vlm2 = vsetvlmax_e32m2(); + const int vlm2 = vsetvlmax_e32m2(); int i = 0; for( ; i <= nvecs - 15; i += 15 ) { @@ -862,45 +857,26 @@ void fastGEMM1T( const float* vec, const float* weights, vs6 = vfmv_v_f_f32m2(0, vlm2), vs7 = vfmv_v_f_f32m2(0, vlm2), vs8 = vfmv_v_f_f32m2(0, vlm2), vs9 = vfmv_v_f_f32m2(0, vlm2), vs10 = vfmv_v_f_f32m2(0, vlm2), vs11 = vfmv_v_f_f32m2(0, vlm2), vs12 = vfmv_v_f_f32m2(0, vlm2), vs13 = vfmv_v_f_f32m2(0, vlm2), vs14 = vfmv_v_f_f32m2(0, vlm2); - int k = 0; - for( ; k < vecsize - vlm2; k += vlm2, wptr += vlm2 ) + int avl = vecsize, vl; + for(int k = 0 ; k < vecsize; k += vl, wptr += vl, avl -= vl) { - vfloat32m2_t v = vle32_v_f32m2(vec + k, vlm2); - - vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, vlm2), v, vlm2); - vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep, vlm2), v, vlm2); - vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*2, vlm2), v, vlm2); - vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*3, vlm2), v, vlm2); - vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*4, vlm2), v, vlm2); - vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*5, vlm2), v, vlm2); - vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*6, vlm2), v, vlm2); - vs7 = vfmacc_vv_f32m2(vs7, vle32_v_f32m2(wptr + wstep*7, vlm2), v, vlm2); - vs8 = vfmacc_vv_f32m2(vs8, vle32_v_f32m2(wptr + wstep*8, vlm2), v, vlm2); - vs9 = vfmacc_vv_f32m2(vs9, vle32_v_f32m2(wptr + wstep*9, vlm2), v, vlm2); - vs10 = vfmacc_vv_f32m2(vs10, vle32_v_f32m2(wptr + wstep*10, vlm2), v, vlm2); - vs11 = vfmacc_vv_f32m2(vs11, vle32_v_f32m2(wptr + wstep*11, vlm2), v, vlm2); - vs12 = vfmacc_vv_f32m2(vs12, vle32_v_f32m2(wptr + wstep*12, vlm2), v, vlm2); - vs13 = vfmacc_vv_f32m2(vs13, vle32_v_f32m2(wptr + wstep*13, vlm2), v, vlm2); - vs14 = vfmacc_vv_f32m2(vs14, vle32_v_f32m2(wptr + wstep*14, vlm2), v, vlm2); - } - int kvl = vecsize - k; - if (kvl > 0) { - vfloat32m2_t v = vle32_v_f32m2(vec + k, kvl); - vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, kvl), v, kvl); - vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep*1, kvl), v, kvl); - vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*2, kvl), v, kvl); - vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*3, kvl), v, kvl); - vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*4, kvl), v, kvl); - vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*5, kvl), v, kvl); - vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*6, kvl), v, kvl); - vs7 = vfmacc_vv_f32m2(vs7, vle32_v_f32m2(wptr + wstep*7, kvl), v, kvl); - vs8 = vfmacc_vv_f32m2(vs8, vle32_v_f32m2(wptr + wstep*8, kvl), v, kvl); - vs9 = vfmacc_vv_f32m2(vs9, vle32_v_f32m2(wptr + wstep*9, kvl), v, kvl); - vs10 = vfmacc_vv_f32m2(vs10, vle32_v_f32m2(wptr + wstep*10, kvl), v, kvl); - vs11 = vfmacc_vv_f32m2(vs11, vle32_v_f32m2(wptr + wstep*11, kvl), v, kvl); - vs12 = vfmacc_vv_f32m2(vs12, vle32_v_f32m2(wptr + wstep*12, kvl), v, kvl); - vs13 = vfmacc_vv_f32m2(vs13, vle32_v_f32m2(wptr + wstep*13, kvl), v, kvl); - vs14 = vfmacc_vv_f32m2(vs14, vle32_v_f32m2(wptr + wstep*14, kvl), v, kvl); + vl = vsetvl_e32m2(avl); + vfloat32m2_t v = vle32_v_f32m2(vec + k, vl); + vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, vl), v, vl); + vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep, vl), v, vl); + vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*2, vl), v, vl); + vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*3, vl), v, vl); + vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*4, vl), v, vl); + vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*5, vl), v, vl); + vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*6, vl), v, vl); + vs7 = vfmacc_vv_f32m2(vs7, vle32_v_f32m2(wptr + wstep*7, vl), v, vl); + vs8 = vfmacc_vv_f32m2(vs8, vle32_v_f32m2(wptr + wstep*8, vl), v, vl); + vs9 = vfmacc_vv_f32m2(vs9, vle32_v_f32m2(wptr + wstep*9, vl), v, vl); + vs10 = vfmacc_vv_f32m2(vs10, vle32_v_f32m2(wptr + wstep*10, vl), v, vl); + vs11 = vfmacc_vv_f32m2(vs11, vle32_v_f32m2(wptr + wstep*11, vl), v, vl); + vs12 = vfmacc_vv_f32m2(vs12, vle32_v_f32m2(wptr + wstep*12, vl), v, vl); + vs13 = vfmacc_vv_f32m2(vs13, vle32_v_f32m2(wptr + wstep*13, vl), v, vl); + vs14 = vfmacc_vv_f32m2(vs14, vle32_v_f32m2(wptr + wstep*14, vl), v, vl); } // Calculate the sum of each vector @@ -925,8 +901,8 @@ void fastGEMM1T( const float* vec, const float* weights, vfloat32m4_t s0 = vfadd_vv_f32m4(vle32_v_f32m4(sum, 15), vle32_v_f32m4(bias + i, 15), 15); vse32_v_f32m4(dst + i, s0, 15); } - int mvl = nvecs - i; - if (mvl > 0) + int unroll_tail = nvecs - i; + if (unroll_tail > 0) { const float* wptr = weights + i*wstep; vfloat32m2_t @@ -935,43 +911,27 @@ void fastGEMM1T( const float* vec, const float* weights, vs6 = vfmv_v_f_f32m2(0, vlm2), vs7 = vfmv_v_f_f32m2(0, vlm2), vs8 = vfmv_v_f_f32m2(0, vlm2), vs9 = vfmv_v_f_f32m2(0, vlm2), vs10 = vfmv_v_f_f32m2(0, vlm2), vs11 = vfmv_v_f_f32m2(0, vlm2), vs12 = vfmv_v_f_f32m2(0, vlm2), vs13 = vfmv_v_f_f32m2(0, vlm2); - int k = 0; - for( ; k <= vecsize - vlm2; k += vlm2, wptr += vlm2 ) + int avl = vecsize, vl; + for(int k = 0; k < vecsize; k += vl, wptr += vl, avl -= vl) { - vfloat32m2_t v = vle32_v_f32m2(vec + k, vlm2); - vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, vlm2), v, vlm2); - vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep*std::min(1, mvl-1), vlm2), v, vlm2); - vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*std::min(2, mvl-1), vlm2), v, vlm2); - vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*std::min(3, mvl-1), vlm2), v, vlm2); - vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*std::min(4, mvl-1), vlm2), v, vlm2); - vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*std::min(5, mvl-1), vlm2), v, vlm2); - vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*std::min(6, mvl-1), vlm2), v, vlm2); - vs7 = vfmacc_vv_f32m2(vs7, vle32_v_f32m2(wptr + wstep*std::min(7, mvl-1), vlm2), v, vlm2); - vs8 = vfmacc_vv_f32m2(vs8, vle32_v_f32m2(wptr + wstep*std::min(8, mvl-1), vlm2), v, vlm2); - vs9 = vfmacc_vv_f32m2(vs9, vle32_v_f32m2(wptr + wstep*std::min(9, mvl-1), vlm2), v, vlm2); - vs10 = vfmacc_vv_f32m2(vs10, vle32_v_f32m2(wptr + wstep*std::min(10, mvl-1), vlm2), v, vlm2); - vs11 = vfmacc_vv_f32m2(vs11, vle32_v_f32m2(wptr + wstep*std::min(11, mvl-1), vlm2), v, vlm2); - vs12 = vfmacc_vv_f32m2(vs12, vle32_v_f32m2(wptr + wstep*std::min(12, mvl-1), vlm2), v, vlm2); - vs13 = vfmacc_vv_f32m2(vs13, vle32_v_f32m2(wptr + wstep*std::min(13, mvl-1), vlm2), v, vlm2); - } - int kvl = vecsize - k; - if (kvl > 0) { - vfloat32m2_t v = vle32_v_f32m2(vec + k, kvl); - vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, kvl), v, kvl); - vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep*std::min(1, mvl-1), kvl), v, kvl); - vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*std::min(2, mvl-1), kvl), v, kvl); - vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*std::min(3, mvl-1), kvl), v, kvl); - vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*std::min(4, mvl-1), kvl), v, kvl); - vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*std::min(5, mvl-1), kvl), v, kvl); - vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*std::min(6, mvl-1), kvl), v, kvl); - vs7 = vfmacc_vv_f32m2(vs7, vle32_v_f32m2(wptr + wstep*std::min(7, mvl-1), kvl), v, kvl); - vs8 = vfmacc_vv_f32m2(vs8, vle32_v_f32m2(wptr + wstep*std::min(8, mvl-1), kvl), v, kvl); - vs9 = vfmacc_vv_f32m2(vs9, vle32_v_f32m2(wptr + wstep*std::min(9, mvl-1), kvl), v, kvl); - vs10 = vfmacc_vv_f32m2(vs10, vle32_v_f32m2(wptr + wstep*std::min(10, mvl-1), kvl), v, kvl); - vs11 = vfmacc_vv_f32m2(vs11, vle32_v_f32m2(wptr + wstep*std::min(11, mvl-1), kvl), v, kvl); - vs12 = vfmacc_vv_f32m2(vs12, vle32_v_f32m2(wptr + wstep*std::min(12, mvl-1), kvl), v, kvl); - vs13 = vfmacc_vv_f32m2(vs13, vle32_v_f32m2(wptr + wstep*std::min(13, mvl-1), kvl), v, kvl); + vl = vsetvl_e32m2(avl); + vfloat32m2_t v = vle32_v_f32m2(vec + k, vl); + vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, vl), v, vl); + vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep*std::min(1, unroll_tail-1), vl), v, vl); + vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*std::min(2, unroll_tail-1), vl), v, vl); + vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*std::min(3, unroll_tail-1), vl), v, vl); + vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*std::min(4, unroll_tail-1), vl), v, vl); + vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*std::min(5, unroll_tail-1), vl), v, vl); + vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*std::min(6, unroll_tail-1), vl), v, vl); + vs7 = vfmacc_vv_f32m2(vs7, vle32_v_f32m2(wptr + wstep*std::min(7, unroll_tail-1), vl), v, vl); + vs8 = vfmacc_vv_f32m2(vs8, vle32_v_f32m2(wptr + wstep*std::min(8, unroll_tail-1), vl), v, vl); + vs9 = vfmacc_vv_f32m2(vs9, vle32_v_f32m2(wptr + wstep*std::min(9, unroll_tail-1), vl), v, vl); + vs10 = vfmacc_vv_f32m2(vs10, vle32_v_f32m2(wptr + wstep*std::min(10, unroll_tail-1), vl), v, vl); + vs11 = vfmacc_vv_f32m2(vs11, vle32_v_f32m2(wptr + wstep*std::min(11, unroll_tail-1), vl), v, vl); + vs12 = vfmacc_vv_f32m2(vs12, vle32_v_f32m2(wptr + wstep*std::min(12, unroll_tail-1), vl), v, vl); + vs13 = vfmacc_vv_f32m2(vs13, vle32_v_f32m2(wptr + wstep*std::min(13, unroll_tail-1), vl), v, vl); } + // Calculate the sum of each vector float sum[14]; vfloat32m1_t zero = vfmv_v_f_f32m1(0, vlm2); @@ -990,8 +950,8 @@ void fastGEMM1T( const float* vec, const float* weights, sum[12] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs12, zero, vlm2)); sum[13] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m2_f32m1(zero, vs13, zero, vlm2)); - vfloat32m4_t s0 = vfadd_vv_f32m4(vle32_v_f32m4(sum, mvl), vle32_v_f32m4(bias + i, mvl), mvl); - vse32_v_f32m4(dst + i, s0, mvl); + vfloat32m4_t s0 = vfadd_vv_f32m4(vle32_v_f32m4(sum, unroll_tail), vle32_v_f32m4(bias + i, unroll_tail), unroll_tail); + vse32_v_f32m4(dst + i, s0, unroll_tail); } } @@ -1001,14 +961,14 @@ void fastConv( const float* weights, size_t wstep, const float* bias, int blockSize, int vecsize, int vecsize_aligned, const float* relu, bool initOutput ) { - int vl = FASCONV_BASE_VECSZ; - int vlm1Max = vsetvlmax_e32m1(); + const int vlm1 = vsetvlmax_e32m1(); int outCn = outShape[1]; size_t outPlaneSize = outShape[2]*outShape[3]; // now compute dot product of the weights // and im2row-transformed part of the tensor for( int i = 0; i < outCn; i += 3 ) { + int unroll_tail = FASCONV_BASE_VECSZ; const float* wptr0 = weights + i*wstep; const float* wptr1 = wptr0 + wstep; const float* wptr2 = wptr1 + wstep; @@ -1033,20 +993,27 @@ void fastConv( const float* weights, size_t wstep, const float* bias, int j = 0; for( ; j < blockSize; j += FASCONV_BASE_VECSZ ) { - bool tail = false; + const float* rptr = rowbuf + j*vecsize_aligned; + const float *rptr1 = rptr + vecsize_aligned*1, + *rptr2 = rptr + vecsize_aligned*2, + *rptr3 = rptr + vecsize_aligned*3, + *rptr4 = rptr + vecsize_aligned*4, + *rptr5 = rptr + vecsize_aligned*5, + *rptr6 = rptr + vecsize_aligned*6, + *rptr7 = rptr + vecsize_aligned*7; if (j + FASCONV_BASE_VECSZ > blockSize) { - if (j == 0) { - vl = blockSize; - } - else { - j = blockSize - FASCONV_BASE_VECSZ; - tail = true; - } + unroll_tail = blockSize - j; + rptr1 = rptr + vecsize_aligned*std::min(1, unroll_tail-1), + rptr2 = rptr + vecsize_aligned*std::min(2, unroll_tail-1), + rptr3 = rptr + vecsize_aligned*std::min(3, unroll_tail-1), + rptr4 = rptr + vecsize_aligned*std::min(4, unroll_tail-1), + rptr5 = rptr + vecsize_aligned*std::min(5, unroll_tail-1), + rptr6 = rptr + vecsize_aligned*std::min(6, unroll_tail-1), + rptr7 = rptr + vecsize_aligned*std::min(7, unroll_tail-1); } - int k = 0; - const float* rptr = rowbuf + j*vecsize_aligned; - int vlm1 = vsetvlmax_e32m1(); + + int vl, avl = vecsize; vfloat32m1_t vs00 = vfmv_v_f_f32m1(0, vlm1), vs10 = vfmv_v_f_f32m1(0, vlm1), vs20 = vfmv_v_f_f32m1(0, vlm1), vs01 = vfmv_v_f_f32m1(0, vlm1), vs11 = vfmv_v_f_f32m1(0, vlm1), vs21 = vfmv_v_f_f32m1(0, vlm1), @@ -1057,107 +1024,107 @@ void fastConv( const float* weights, size_t wstep, const float* bias, vs06 = vfmv_v_f_f32m1(0, vlm1), vs16 = vfmv_v_f_f32m1(0, vlm1), vs26 = vfmv_v_f_f32m1(0, vlm1), vs07 = vfmv_v_f_f32m1(0, vlm1), vs17 = vfmv_v_f_f32m1(0, vlm1), vs27 = vfmv_v_f_f32m1(0, vlm1); - for (; k < vecsize; k += vlm1, rptr += vlm1 ) + for (int k = 0; k < vecsize; k += vl, avl -= vl) { - if (k + vlm1 >= vecsize) { - vlm1 = vecsize - k; - } - vfloat32m1_t w0 = vle32_v_f32m1(wptr0 + k, vlm1); - vfloat32m1_t w1 = vle32_v_f32m1(wptr1 + k, vlm1); - vfloat32m1_t w2 = vle32_v_f32m1(wptr2 + k, vlm1); - vfloat32m1_t r0 = vle32_v_f32m1(rptr, vlm1); + vl = vsetvl_e32m1(avl); + vfloat32m1_t w0 = vle32_v_f32m1(wptr0 + k, vl); + vfloat32m1_t w1 = vle32_v_f32m1(wptr1 + k, vl); + vfloat32m1_t w2 = vle32_v_f32m1(wptr2 + k, vl); + vfloat32m1_t r0 = vle32_v_f32m1(rptr, vl); - vs00 = vfmacc_vv_f32m1(vs00, w0, r0, vlm1); - vs10 = vfmacc_vv_f32m1(vs10, w1, r0, vlm1); - vs20 = vfmacc_vv_f32m1(vs20, w2, r0, vlm1); + vs00 = vfmacc_vv_f32m1(vs00, w0, r0, vl); + vs10 = vfmacc_vv_f32m1(vs10, w1, r0, vl); + vs20 = vfmacc_vv_f32m1(vs20, w2, r0, vl); - r0 = vle32_v_f32m1(rptr + vecsize_aligned, vlm1); - vs01 = vfmacc_vv_f32m1(vs01, w0, r0, vlm1); - vs11 = vfmacc_vv_f32m1(vs11, w1, r0, vlm1); - vs21 = vfmacc_vv_f32m1(vs21, w2, r0, vlm1); + r0 = vle32_v_f32m1(rptr1, vl); + vs01 = vfmacc_vv_f32m1(vs01, w0, r0, vl); + vs11 = vfmacc_vv_f32m1(vs11, w1, r0, vl); + vs21 = vfmacc_vv_f32m1(vs21, w2, r0, vl); - r0 = vle32_v_f32m1(rptr + vecsize_aligned*2, vlm1); - vs02 = vfmacc_vv_f32m1(vs02, w0, r0, vlm1); - vs12 = vfmacc_vv_f32m1(vs12, w1, r0, vlm1); - vs22 = vfmacc_vv_f32m1(vs22, w2, r0, vlm1); + r0 = vle32_v_f32m1(rptr2, vl); + vs02 = vfmacc_vv_f32m1(vs02, w0, r0, vl); + vs12 = vfmacc_vv_f32m1(vs12, w1, r0, vl); + vs22 = vfmacc_vv_f32m1(vs22, w2, r0, vl); - r0 = vle32_v_f32m1(rptr + vecsize_aligned*3, vlm1); - vs03 = vfmacc_vv_f32m1(vs03, w0, r0, vlm1); - vs13 = vfmacc_vv_f32m1(vs13, w1, r0, vlm1); - vs23 = vfmacc_vv_f32m1(vs23, w2, r0, vlm1); + r0 = vle32_v_f32m1(rptr3, vl); + vs03 = vfmacc_vv_f32m1(vs03, w0, r0, vl); + vs13 = vfmacc_vv_f32m1(vs13, w1, r0, vl); + vs23 = vfmacc_vv_f32m1(vs23, w2, r0, vl); - r0 = vle32_v_f32m1(rptr + vecsize_aligned*4, vlm1); - vs04 = vfmacc_vv_f32m1(vs04, w0, r0, vlm1); - vs14 = vfmacc_vv_f32m1(vs14, w1, r0, vlm1); - vs24 = vfmacc_vv_f32m1(vs24, w2, r0, vlm1); + r0 = vle32_v_f32m1(rptr4, vl); + vs04 = vfmacc_vv_f32m1(vs04, w0, r0, vl); + vs14 = vfmacc_vv_f32m1(vs14, w1, r0, vl); + vs24 = vfmacc_vv_f32m1(vs24, w2, r0, vl); - r0 = vle32_v_f32m1(rptr + vecsize_aligned*5, vlm1); - vs05 = vfmacc_vv_f32m1(vs05, w0, r0, vlm1); - vs15 = vfmacc_vv_f32m1(vs15, w1, r0, vlm1); - vs25 = vfmacc_vv_f32m1(vs25, w2, r0, vlm1); + r0 = vle32_v_f32m1(rptr5, vl); + vs05 = vfmacc_vv_f32m1(vs05, w0, r0, vl); + vs15 = vfmacc_vv_f32m1(vs15, w1, r0, vl); + vs25 = vfmacc_vv_f32m1(vs25, w2, r0, vl); - r0 = vle32_v_f32m1(rptr + vecsize_aligned*6, vlm1); - vs06 = vfmacc_vv_f32m1(vs06, w0, r0, vlm1); - vs16 = vfmacc_vv_f32m1(vs16, w1, r0, vlm1); - vs26 = vfmacc_vv_f32m1(vs26, w2, r0, vlm1); + r0 = vle32_v_f32m1(rptr6, vl); + vs06 = vfmacc_vv_f32m1(vs06, w0, r0, vl); + vs16 = vfmacc_vv_f32m1(vs16, w1, r0, vl); + vs26 = vfmacc_vv_f32m1(vs26, w2, r0, vl); - r0 = vle32_v_f32m1(rptr + vecsize_aligned*7, vlm1); - vs07 = vfmacc_vv_f32m1(vs07, w0, r0, vlm1); - vs17 = vfmacc_vv_f32m1(vs17, w1, r0, vlm1); - vs27 = vfmacc_vv_f32m1(vs27, w2, r0, vlm1); + r0 = vle32_v_f32m1(rptr7, vl); + vs07 = vfmacc_vv_f32m1(vs07, w0, r0, vl); + vs17 = vfmacc_vv_f32m1(vs17, w1, r0, vl); + vs27 = vfmacc_vv_f32m1(vs27, w2, r0, vl); + + rptr += vl; rptr1 += vl; rptr2 += vl; rptr3 += vl; + rptr4 += vl; rptr5 += vl; rptr6 += vl; rptr7 += vl; } // compute sum of each vs - vfloat32m1_t zero = vfmv_v_f_f32m1(0, vlm1Max); - // vl is required here to be at least FASCONV_BASE_VECSZ, aka 8. + vfloat32m1_t zero = vfmv_v_f_f32m1(0, vlm1); + // unroll_tail(vl) is required here to be at least FASCONV_BASE_VECSZ, aka 8. float sum0[FASCONV_BASE_VECSZ], sum1[FASCONV_BASE_VECSZ], sum2[FASCONV_BASE_VECSZ]; - sum0[0] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs00, zero, vlm1Max)); - sum0[1] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs01, zero, vlm1Max)); - sum0[2] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs02, zero, vlm1Max)); - sum0[3] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs03, zero, vlm1Max)); - sum0[4] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs04, zero, vlm1Max)); - sum0[5] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs05, zero, vlm1Max)); - sum0[6] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs06, zero, vlm1Max)); - sum0[7] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs07, zero, vlm1Max)); - sum1[0] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs10, zero, vlm1Max)); - sum1[1] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs11, zero, vlm1Max)); - sum1[2] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs12, zero, vlm1Max)); - sum1[3] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs13, zero, vlm1Max)); - sum1[4] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs14, zero, vlm1Max)); - sum1[5] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs15, zero, vlm1Max)); - sum1[6] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs16, zero, vlm1Max)); - sum1[7] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs17, zero, vlm1Max)); - sum2[0] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs20, zero, vlm1Max)); - sum2[1] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs21, zero, vlm1Max)); - sum2[2] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs22, zero, vlm1Max)); - sum2[3] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs23, zero, vlm1Max)); - sum2[4] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs24, zero, vlm1Max)); - sum2[5] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs25, zero, vlm1Max)); - sum2[6] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs26, zero, vlm1Max)); - sum2[7] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs27, zero, vlm1Max)); + sum0[0] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs00, zero, vlm1)); + sum0[1] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs01, zero, vlm1)); + sum0[2] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs02, zero, vlm1)); + sum0[3] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs03, zero, vlm1)); + sum0[4] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs04, zero, vlm1)); + sum0[5] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs05, zero, vlm1)); + sum0[6] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs06, zero, vlm1)); + sum0[7] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs07, zero, vlm1)); + sum1[0] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs10, zero, vlm1)); + sum1[1] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs11, zero, vlm1)); + sum1[2] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs12, zero, vlm1)); + sum1[3] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs13, zero, vlm1)); + sum1[4] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs14, zero, vlm1)); + sum1[5] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs15, zero, vlm1)); + sum1[6] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs16, zero, vlm1)); + sum1[7] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs17, zero, vlm1)); + sum2[0] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs20, zero, vlm1)); + sum2[1] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs21, zero, vlm1)); + sum2[2] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs22, zero, vlm1)); + sum2[3] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs23, zero, vlm1)); + sum2[4] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs24, zero, vlm1)); + sum2[5] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs25, zero, vlm1)); + sum2[6] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs26, zero, vlm1)); + sum2[7] = vfmv_f_s_f32m1_f32(vfredosum_vs_f32m1_f32m1(zero, vs27, zero, vlm1)); - // if VLEN = 128, so LMUL = 2 for vl = 8. + // if VLEN = 128, so LMUL = 2 for unroll_tail(vl) = 8. // otherwise, VLEN >=256, we only use fist 8 element of the vReg. vfloat32m2_t s0, s1, s2; if( initOutput ) { - s0 = vfmv_v_f_f32m2(bias0, vl); - s1 = vfmv_v_f_f32m2(bias1, vl); - s2 = vfmv_v_f_f32m2(bias2, vl); + s0 = vfmv_v_f_f32m2(bias0, unroll_tail); + s1 = vfmv_v_f_f32m2(bias1, unroll_tail); + s2 = vfmv_v_f_f32m2(bias2, unroll_tail); } else { - s0 = vle32_v_f32m2(outptr0 + j, vl); - s1 = vle32_v_f32m2(outptr1 + j, vl); - s2 = vle32_v_f32m2(outptr2 + j, vl); + s0 = vle32_v_f32m2(outptr0 + j, unroll_tail); + s1 = vle32_v_f32m2(outptr1 + j, unroll_tail); + s2 = vle32_v_f32m2(outptr2 + j, unroll_tail); } - s0 = vfadd_vv_f32m2(vle32_v_f32m2(sum0, vl), s0, vl); - s1 = vfadd_vv_f32m2(vle32_v_f32m2(sum1, vl), s1, vl); - s2 = vfadd_vv_f32m2(vle32_v_f32m2(sum2, vl), s2, vl); + s0 = vfadd_vv_f32m2(vle32_v_f32m2(sum0, unroll_tail), s0, unroll_tail); + s1 = vfadd_vv_f32m2(vle32_v_f32m2(sum1, unroll_tail), s1, unroll_tail); + s2 = vfadd_vv_f32m2(vle32_v_f32m2(sum2, unroll_tail), s2, unroll_tail); if( relu ) { - vfloat32m2_t vr0 = vfmv_v_f_f32m2(1, vl), vr1 = vfmv_v_f_f32m2(1, vl), vr2 = vfmv_v_f_f32m2(1, vl); float r0 = relu[i], r1 = relu[i+1], r2 = relu[i+2]; if( i+2 >= outCn ) { @@ -1165,33 +1132,17 @@ void fastConv( const float* weights, size_t wstep, const float* bias, if( i+1 >= outCn ) r2 = r1 = r0; } - vr0 = vfmv_v_f_f32m2(r0, vl); - vr1 = vfmv_v_f_f32m2(r1, vl); - vr2 = vfmv_v_f_f32m2(r2, vl); - vbool16_t m0 = vmfgt_vf_f32m2_b16(s0, 0, vl); - vbool16_t m1 = vmfgt_vf_f32m2_b16(s1, 0, vl); - vbool16_t m2 = vmfgt_vf_f32m2_b16(s2, 0, vl); - s0 = vmerge_vvm_f32m2(m0, vfmul_vv_f32m2(s0, vr0, vl), s0, vl); - s1 = vmerge_vvm_f32m2(m1, vfmul_vv_f32m2(s1, vr1, vl), s1, vl); - s2 = vmerge_vvm_f32m2(m2, vfmul_vv_f32m2(s2, vr2, vl), s2, vl); + vbool16_t m0 = vmfgt_vf_f32m2_b16(s0, 0, unroll_tail); + vbool16_t m1 = vmfgt_vf_f32m2_b16(s1, 0, unroll_tail); + vbool16_t m2 = vmfgt_vf_f32m2_b16(s2, 0, unroll_tail); + s0 = vmerge_vvm_f32m2(m0, vfmul_vf_f32m2(s0, r0, unroll_tail), s0, unroll_tail); + s1 = vmerge_vvm_f32m2(m1, vfmul_vf_f32m2(s1, r1, unroll_tail), s1, unroll_tail); + s2 = vmerge_vvm_f32m2(m2, vfmul_vf_f32m2(s2, r2, unroll_tail), s2, unroll_tail); } - if( tail ) - { - int maskbuf[FASCONV_BASE_VECSZ] = {0}; - int rsz = blockSize % FASCONV_BASE_VECSZ; - for( int i = 0; i < rsz; i++ ) - maskbuf[FASCONV_BASE_VECSZ - i - 1] = -1; - vint32m2_t vmaskbuf = vle32_v_i32m2(maskbuf ,vl); - vbool16_t mask = vmslt_vx_i32m2_b16(vmaskbuf, 0, vl); // mask for tail - s0 = vmerge_vvm_f32m2(mask, vle32_v_f32m2(outptr0 + j, vl), s0, vl); - s1 = vmerge_vvm_f32m2(mask, vle32_v_f32m2(outptr1 + j, vl), s1, vl); - s2 = vmerge_vvm_f32m2(mask, vle32_v_f32m2(outptr2 + j, vl), s2, vl); - } - - vse32_v_f32m2(outptr0 + j, s0, vl); - vse32_v_f32m2(outptr1 + j, s1, vl); - vse32_v_f32m2(outptr2 + j, s2, vl); + vse32_v_f32m2(outptr0 + j, s0, unroll_tail); + vse32_v_f32m2(outptr1 + j, s1, unroll_tail); + vse32_v_f32m2(outptr2 + j, s2, unroll_tail); } } } @@ -1236,7 +1187,7 @@ void fastDepthwiseConv( const float* wptr, float* outptr_, int out_d, int outH, int outW ) { - int vl = vsetvlmax_e32m2(); + int vl; const float w00_ = wptr[0], w01_ = wptr[1], w02_ = wptr[2], w10 = wptr[3], w11 = wptr[4], w12 = wptr[5], w20_ = wptr[6], w21_ = wptr[7], w22_ = wptr[8]; @@ -1275,11 +1226,11 @@ void fastDepthwiseConv( const float* wptr, if (stride_w == 1 || (stride_w == 2 && dilation_w == 1)) { + int avl = outW1 - out_j; if( stride_w == 1 ) - for( ; out_j < outW1; out_j += vl ) + for( ; out_j < outW1; out_j += vl, avl -= vl) { - if (out_j + vl > outW1) - vl = outW1 - out_j; + vl = vsetvl_e32m2(avl); int in_j = out_j * stride_w - pad_l; vfloat32m2_t v00 = vle32_v_f32m2(imgptr0 + in_j, vl), v01 = vle32_v_f32m2(imgptr0 + in_j + dilation_w, vl), @@ -1313,10 +1264,9 @@ void fastDepthwiseConv( const float* wptr, vse32_v_f32m2(outptr + out_j, vout0, vl); } else //stride_w == 2 && dilation_w == 1 - for( ; out_j < outW1; out_j += vl ) + for( ; out_j < outW1; out_j += vl, avl -= vl) { - if (out_j + vl > outW1) - vl = outW1 - out_j; + vl = vsetvl_e32m2(avl); int in_j = out_j * stride_w - pad_l; vfloat32m2_t v00, v01, v02, v10, v11, v12, v20, v21, v22, unused; vfloat32m2_load_deinterleave(imgptr0 + in_j, v00, v01, vl); From 81a6d1b0d4d93a3330cc99fbb1601c90b9516a9c Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 2 Dec 2021 13:32:59 +0000 Subject: [PATCH 156/226] cmake: update SOVERSION - OpenCV 4.x doesn't guarantee or maintain ABI compatibility - We should increase SO version on each release --- cmake/OpenCVVersion.cmake | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmake/OpenCVVersion.cmake b/cmake/OpenCVVersion.cmake index a2b4926cf0..6ffaab94cf 100644 --- a/cmake/OpenCVVersion.cmake +++ b/cmake/OpenCVVersion.cmake @@ -10,8 +10,10 @@ set(OPENCV_VERSION_PLAIN "${OPENCV_VERSION_MAJOR}.${OPENCV_VERSION_MINOR}.${OPEN set(OPENCV_VERSION "${OPENCV_VERSION_PLAIN}${OPENCV_VERSION_STATUS}") -set(OPENCV_SOVERSION "${OPENCV_VERSION_MAJOR}.${OPENCV_VERSION_MINOR}") -set(OPENCV_LIBVERSION "${OPENCV_VERSION_MAJOR}.${OPENCV_VERSION_MINOR}.${OPENCV_VERSION_PATCH}") +string(REGEX MATCH "[0-9][0-9]$" OPENCV_VERSION_MINOR_2DIGITS "00${OPENCV_VERSION_MINOR}") +string(REGEX MATCH "[0-9][0-9]$" OPENCV_VERSION_PATCH_2DIGITS "00${OPENCV_VERSION_PATCH}") +ocv_update(OPENCV_SOVERSION "${OPENCV_VERSION_MAJOR}${OPENCV_VERSION_MINOR_2DIGITS}") +ocv_update(OPENCV_LIBVERSION "${OPENCV_VERSION_MAJOR}.${OPENCV_VERSION_MINOR}.${OPENCV_VERSION_PATCH}") # create a dependency on the version file # we never use the output of the following command but cmake will rerun automatically if the version file changes From d1b923bee9f6809e0cf1ca21d20b66ea49db495e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20D=C3=B6nszelmann?= Date: Sat, 11 Dec 2021 08:03:10 +0100 Subject: [PATCH 157/226] Update name from Gunner to Gunnar as that's the name he published his paper under. --- .../js_video/js_lucas_kanade/js_lucas_kanade.markdown | 4 ++-- doc/tutorials/video/optical_flow/optical_flow.markdown | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/js_tutorials/js_video/js_lucas_kanade/js_lucas_kanade.markdown b/doc/js_tutorials/js_video/js_lucas_kanade/js_lucas_kanade.markdown index a86bf11223..f4e4f231b0 100644 --- a/doc/js_tutorials/js_video/js_lucas_kanade/js_lucas_kanade.markdown +++ b/doc/js_tutorials/js_video/js_lucas_kanade/js_lucas_kanade.markdown @@ -133,9 +133,9 @@ Dense Optical Flow in OpenCV.js Lucas-Kanade method computes optical flow for a sparse feature set (in our example, corners detected using Shi-Tomasi algorithm). OpenCV.js provides another algorithm to find the dense optical flow. It -computes the optical flow for all the points in the frame. It is based on Gunner Farneback's +computes the optical flow for all the points in the frame. It is based on Gunnar Farneback's algorithm which is explained in "Two-Frame Motion Estimation Based on Polynomial Expansion" by -Gunner Farneback in 2003. +Gunnar Farneback in 2003. We use the function: **cv.calcOpticalFlowFarneback (prev, next, flow, pyrScale, levels, winsize, iterations, polyN, polySigma, flags)** diff --git a/doc/tutorials/video/optical_flow/optical_flow.markdown b/doc/tutorials/video/optical_flow/optical_flow.markdown index 45bbfa46ce..a9faf9be13 100644 --- a/doc/tutorials/video/optical_flow/optical_flow.markdown +++ b/doc/tutorials/video/optical_flow/optical_flow.markdown @@ -136,9 +136,9 @@ Dense Optical Flow in OpenCV Lucas-Kanade method computes optical flow for a sparse feature set (in our example, corners detected using Shi-Tomasi algorithm). OpenCV provides another algorithm to find the dense optical flow. It -computes the optical flow for all the points in the frame. It is based on Gunner Farneback's +computes the optical flow for all the points in the frame. It is based on Gunnar Farneback's algorithm which is explained in "Two-Frame Motion Estimation Based on Polynomial Expansion" by -Gunner Farneback in 2003. +Gunnar Farneback in 2003. Below sample shows how to find the dense optical flow using above algorithm. We get a 2-channel array with optical flow vectors, \f$(u,v)\f$. We find their magnitude and direction. We color code the From 252ce0b58194f76d1a6b700dd18a39cd022757b3 Mon Sep 17 00:00:00 2001 From: cqn2219076254 <2219076254@qq.com> Date: Mon, 13 Dec 2021 21:43:13 +0800 Subject: [PATCH 158/226] add square layer --- modules/dnn/src/tensorflow/tf_importer.cpp | 22 +++++++++++++++++++++- modules/dnn/test/test_tf_importer.cpp | 7 +++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index 5fafa2b9d5..e37e888e35 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -598,7 +598,7 @@ private: void parseLeakyRelu (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); void parseActivation (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); void parseExpandDims (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); - + void parseSquare (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); void parseCustomLayer (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); }; @@ -676,6 +676,7 @@ const TFImporter::DispatchMap TFImporter::buildDispatchMap() dispatch["Abs"] = dispatch["Tanh"] = dispatch["Sigmoid"] = dispatch["Relu"] = dispatch["Elu"] = dispatch["Exp"] = dispatch["Identity"] = dispatch["Relu6"] = &TFImporter::parseActivation; dispatch["ExpandDims"] = &TFImporter::parseExpandDims; + dispatch["Square"] = &TFImporter::parseSquare; return dispatch; } @@ -1252,6 +1253,25 @@ void TFImporter::parseExpandDims(tensorflow::GraphDef& net, const tensorflow::No } } +// "Square" +void TFImporter::parseSquare(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 1, ""); + + int id; + layerParams.set("operation", "prod"); + id = dstNet.addLayer(name, "Eltwise", layerParams); + + layer_id[name] = id; + + Pin inp = parsePin(layer.input(0)); + connect(layer_id, dstNet, inp, id, 0); + connect(layer_id, dstNet, inp, id, 1); +} + // "Flatten" "Squeeze" void TFImporter::parseFlatten(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index 117177a860..1c4beb7468 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -670,6 +670,13 @@ TEST_P(Test_TensorFlow_layers, batch_matmul) runTensorFlowNet("batch_matmul"); } +TEST_P(Test_TensorFlow_layers, square) +{ + if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + runTensorFlowNet("square"); +} + TEST_P(Test_TensorFlow_layers, reshape) { #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) From 692059e899de958b3f601528946851c8274ae603 Mon Sep 17 00:00:00 2001 From: rogday Date: Mon, 13 Dec 2021 18:41:23 +0300 Subject: [PATCH 159/226] initialize members --- modules/core/test/test_arithm.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/core/test/test_arithm.cpp b/modules/core/test/test_arithm.cpp index 4a9522d3d1..014a0cff0a 100644 --- a/modules/core/test/test_arithm.cpp +++ b/modules/core/test/test_arithm.cpp @@ -1390,7 +1390,8 @@ struct MinMaxLocOp : public BaseElemWiseOp struct reduceArgMinMaxOp : public BaseElemWiseOp { - reduceArgMinMaxOp() : BaseElemWiseOp(1, FIX_ALPHA+FIX_BETA+FIX_GAMMA, 1, 1, Scalar::all(0)) + reduceArgMinMaxOp() : BaseElemWiseOp(1, FIX_ALPHA+FIX_BETA+FIX_GAMMA, 1, 1, Scalar::all(0)), + isLast(false), isMax(false), axis(0) { context = ARITHM_MAX_NDIMS*2 + 2; }; From 456137fa24d94fd515b2d9814fef894363de7a75 Mon Sep 17 00:00:00 2001 From: Polina Smolnikova Date: Mon, 13 Dec 2021 20:12:49 +0300 Subject: [PATCH 160/226] Merge pull request #21166 from rayonnant14:issue_21105 Fix intrinsics processing in case USAC parameters * fixed USAC intrinsics processing * change mat to matx33d, added test --- modules/calib3d/src/usac.hpp | 14 ++--- modules/calib3d/src/usac/dls_solver.cpp | 9 +-- modules/calib3d/src/usac/pnp_solver.cpp | 9 +-- modules/calib3d/src/usac/ransac_solvers.cpp | 11 ++-- modules/calib3d/src/usac/utils.cpp | 70 ++++++++++----------- modules/calib3d/test/test_usac.cpp | 27 ++++++++ 6 files changed, 82 insertions(+), 58 deletions(-) diff --git a/modules/calib3d/src/usac.hpp b/modules/calib3d/src/usac.hpp index 06a0ff2056..6dc79fdc55 100644 --- a/modules/calib3d/src/usac.hpp +++ b/modules/calib3d/src/usac.hpp @@ -116,7 +116,7 @@ public: class P3PSolver : public MinimalSolver { public: - static Ptr create(const Mat &points_, const Mat &calib_norm_pts, const Mat &K); + static Ptr create(const Mat &points_, const Mat &calib_norm_pts, const Matx33d &K); }; //-------------------------- AFFINE ----------------------- @@ -164,7 +164,7 @@ public: class DLSPnP : public NonMinimalSolver { public: - static Ptr create(const Mat &points_, const Mat &calib_norm_pts, const Mat &K); + static Ptr create(const Mat &points_, const Mat &calib_norm_pts, const Matx33d &K); }; //-------------------------- AFFINE ----------------------- @@ -571,11 +571,11 @@ namespace Utils { * @points is matrix N x 4. * @norm_points is output matrix N x 4 with calibrated points. */ - void calibratePoints (const Mat &K1, const Mat &K2, const Mat &points, Mat &norm_points); - void calibrateAndNormalizePointsPnP (const Mat &K, const Mat &pts, Mat &calib_norm_pts); - void normalizeAndDecalibPointsPnP (const Mat &K, Mat &pts, Mat &calib_norm_pts); - void decomposeProjection (const Mat &P, Mat &K_, Mat &R, Mat &t, bool same_focal=false); - double getCalibratedThreshold (double threshold, const Mat &K1, const Mat &K2); + void calibratePoints (const Matx33d &K1, const Matx33d &K2, const Mat &points, Mat &norm_points); + void calibrateAndNormalizePointsPnP (const Matx33d &K, const Mat &pts, Mat &calib_norm_pts); + void normalizeAndDecalibPointsPnP (const Matx33d &K, Mat &pts, Mat &calib_norm_pts); + void decomposeProjection (const Mat &P, Matx33d &K_, Mat &R, Mat &t, bool same_focal=false); + double getCalibratedThreshold (double threshold, const Matx33d &K1, const Matx33d &K2); float findMedian (std::vector &array); } namespace Math { diff --git a/modules/calib3d/src/usac/dls_solver.cpp b/modules/calib3d/src/usac/dls_solver.cpp index 0898734fb3..0abb26cecc 100644 --- a/modules/calib3d/src/usac/dls_solver.cpp +++ b/modules/calib3d/src/usac/dls_solver.cpp @@ -49,13 +49,14 @@ namespace cv { namespace usac { // This is the estimator class for estimating a homography matrix between two images. A model estimation method and error calculation method are implemented class DLSPnPImpl : public DLSPnP { private: - const Mat * points_mat, * calib_norm_points_mat, * K_mat; + const Mat * points_mat, * calib_norm_points_mat; + const Matx33d * K_mat; #if defined(HAVE_LAPACK) || defined(HAVE_EIGEN) - const Mat &K; + const Matx33d &K; const float * const calib_norm_points, * const points; #endif public: - explicit DLSPnPImpl (const Mat &points_, const Mat &calib_norm_points_, const Mat &K_) : + explicit DLSPnPImpl (const Mat &points_, const Mat &calib_norm_points_, const Matx33d &K_) : points_mat(&points_), calib_norm_points_mat(&calib_norm_points_), K_mat (&K_) #if defined(HAVE_LAPACK) || defined(HAVE_EIGEN) , K(K_), calib_norm_points((float*)calib_norm_points_.data), points((float*)points_.data) @@ -870,7 +871,7 @@ protected: 2 * D[74] - 2 * D[78]); // s1^3 } }; -Ptr DLSPnP::create(const Mat &points_, const Mat &calib_norm_pts, const Mat &K) { +Ptr DLSPnP::create(const Mat &points_, const Mat &calib_norm_pts, const Matx33d &K) { return makePtr(points_, calib_norm_pts, K); } }} diff --git a/modules/calib3d/src/usac/pnp_solver.cpp b/modules/calib3d/src/usac/pnp_solver.cpp index e095eeff7c..37c649144c 100644 --- a/modules/calib3d/src/usac/pnp_solver.cpp +++ b/modules/calib3d/src/usac/pnp_solver.cpp @@ -263,7 +263,8 @@ private: * calibrated normalized points * K^-1 [u v 1]^T / ||K^-1 [u v 1]^T|| */ - const Mat * points_mat, * calib_norm_points_mat, * K_mat, &K; + const Mat * points_mat, * calib_norm_points_mat; + const Matx33d * K_mat, &K; const float * const calib_norm_points, * const points; const double VAL_THR = 1e-4; public: @@ -271,7 +272,7 @@ public: * @points_ is matrix N x 5 * u v x y z. (u,v) is image point, (x y z) is world point */ - P3PSolverImpl (const Mat &points_, const Mat &calib_norm_points_, const Mat &K_) : + P3PSolverImpl (const Mat &points_, const Mat &calib_norm_points_, const Matx33d &K_) : points_mat(&points_), calib_norm_points_mat(&calib_norm_points_), K_mat (&K_), K(K_), calib_norm_points((float*)calib_norm_points_.data), points((float*)points_.data) {} @@ -362,7 +363,7 @@ public: const Matx33d R = Math::rotVec2RotMat(Math::rotMat2RotVec(Z * Zw.inv())); - Mat P, KR = K * R; + Mat P, KR = Mat(K * R); hconcat(KR, -KR * (X1 - R.t() * nX1), P); models.emplace_back(P); } @@ -374,7 +375,7 @@ public: return makePtr(*points_mat, *calib_norm_points_mat, *K_mat); } }; -Ptr P3PSolver::create(const Mat &points_, const Mat &calib_norm_pts, const Mat &K) { +Ptr P3PSolver::create(const Mat &points_, const Mat &calib_norm_pts, const Matx33d &K) { return makePtr(points_, calib_norm_pts, K); } }} \ No newline at end of file diff --git a/modules/calib3d/src/usac/ransac_solvers.cpp b/modules/calib3d/src/usac/ransac_solvers.cpp index b7f3e6e0c1..fe64907ec0 100644 --- a/modules/calib3d/src/usac/ransac_solvers.cpp +++ b/modules/calib3d/src/usac/ransac_solvers.cpp @@ -774,13 +774,14 @@ bool run (const Ptr ¶ms, InputArray points1, InputArray points2 Ptr min_solver; Ptr non_min_solver; - Mat points, K1, K2, calib_points, undist_points1, undist_points2; + Mat points, calib_points, undist_points1, undist_points2; + Matx33d K1, K2; int points_size; double threshold = params->getThreshold(), max_thr = params->getMaximumThreshold(); const int min_sample_size = params->getSampleSize(); if (params->isPnP()) { if (! K1_.empty()) { - K1 = K1_.getMat(); K1.convertTo(K1, CV_64F); + K1 = K1_.getMat(); if (! dist_coeff1.empty()) { // undistortPoints also calibrate points using K if (points1.isContinuous()) @@ -797,8 +798,8 @@ bool run (const Ptr ¶ms, InputArray points1, InputArray points2 } else { if (params->isEssential()) { CV_CheckEQ((int)(!K1_.empty() && !K2_.empty()), 1, "Intrinsic matrix must not be empty!"); - K1 = K1_.getMat(); K1.convertTo(K1, CV_64F); - K2 = K2_.getMat(); K2.convertTo(K2, CV_64F); + K1 = K1_.getMat(); + K2 = K2_.getMat(); if (! dist_coeff1.empty() || ! dist_coeff2.empty()) { // undistortPoints also calibrate points using K if (points1.isContinuous()) @@ -1011,7 +1012,7 @@ bool run (const Ptr ¶ms, InputArray points1, InputArray points2 // convert R to rodrigues and back and recalculate inliers which due to numerical // issues can differ Mat out, R, newR, newP, t, rvec; - if (K1.empty()) { + if (K1_.empty()) { usac::Utils::decomposeProjection (ransac_output->getModel(), K1, R, t); Rodrigues(R, rvec); hconcat(rvec, t, out); diff --git a/modules/calib3d/src/usac/utils.cpp b/modules/calib3d/src/usac/utils.cpp index 1c781a7d82..c4d74eb663 100644 --- a/modules/calib3d/src/usac/utils.cpp +++ b/modules/calib3d/src/usac/utils.cpp @@ -8,9 +8,9 @@ #include namespace cv { namespace usac { -double Utils::getCalibratedThreshold (double threshold, const Mat &K1, const Mat &K2) { - return threshold / ((K1.at(0, 0) + K1.at(1, 1) + - K2.at(0, 0) + K2.at(1, 1)) / 4.0); +double Utils::getCalibratedThreshold (double threshold, const Matx33d &K1, const Matx33d &K2) { + return threshold / ((K1(0, 0) + K1(1, 1) + + K2(0, 0) + K2(1, 1)) / 4.0); } /* @@ -20,22 +20,20 @@ double Utils::getCalibratedThreshold (double threshold, const Mat &K1, const Mat * 0 k22 k23 * 0 0 1] */ -void Utils::calibratePoints (const Mat &K1, const Mat &K2, const Mat &points, Mat &calib_points) { - const auto * const points_ = (float *) points.data; - const auto * const k1 = (double *) K1.data; - const auto inv1_k11 = float(1 / k1[0]); // 1 / k11 - const auto inv1_k12 = float(-k1[1] / (k1[0]*k1[4])); // -k12 / (k11*k22) +void Utils::calibratePoints (const Matx33d &K1, const Matx33d &K2, const Mat &points, Mat &calib_points) { + const auto inv1_k11 = float(1 / K1(0, 0)); // 1 / k11 + const auto inv1_k12 = float(-K1(0, 1) / (K1(0, 0)*K1(1, 1))); // -k12 / (k11*k22) // (-k13*k22 + k12*k23) / (k11*k22) - const auto inv1_k13 = float((-k1[2]*k1[4] + k1[1]*k1[5]) / (k1[0]*k1[4])); - const auto inv1_k22 = float(1 / k1[4]); // 1 / k22 - const auto inv1_k23 = float(-k1[5] / k1[4]); // -k23 / k22 + const auto inv1_k13 = float((-K1(0, 2)*K1(1, 1) + K1(0, 1)*K1(1, 2)) / (K1(0, 0)*K1(1, 1))); + const auto inv1_k22 = float(1 / K1(1, 1)); // 1 / k22 + const auto inv1_k23 = float(-K1(1, 2) / K1(1, 1)); // -k23 / k22 - const auto * const k2 = (double *) K2.data; - const auto inv2_k11 = float(1 / k2[0]); - const auto inv2_k12 = float(-k2[1] / (k2[0]*k2[4])); - const auto inv2_k13 = float((-k2[2]*k2[4] + k2[1]*k2[5]) / (k2[0]*k2[4])); - const auto inv2_k22 = float(1 / k2[4]); - const auto inv2_k23 = float(-k2[5] / k2[4]); + const auto inv2_k11 = float(1 / K2(0, 0)); + const auto inv2_k12 = float(-K2(0, 1) / (K2(0, 0)*K2(1, 1))); + const auto inv2_k13 = float((-K2(0, 2)*K2(1, 1) + K2(0, 1)*K2(1, 2)) / (K2(0, 0)*K2(1, 1))); + const auto inv2_k22 = float(1 / K2(1, 1)); + const auto inv2_k23 = float(-K2(1, 2) / K2(1, 1)); + const auto * const points_ = (float *) points.data; calib_points = Mat ( points.rows, 4, points.type()); auto * calib_points_ = (float *) calib_points.data; @@ -54,15 +52,13 @@ void Utils::calibratePoints (const Mat &K1, const Mat &K2, const Mat &points, Ma * points is matrix of size |N| x 5, first two columns are image points [u_i, v_i] * calib_norm_pts are K^-1 [u v 1]^T / ||K^-1 [u v 1]^T|| */ -void Utils::calibrateAndNormalizePointsPnP (const Mat &K, const Mat &pts, Mat &calib_norm_pts) { +void Utils::calibrateAndNormalizePointsPnP (const Matx33d &K, const Mat &pts, Mat &calib_norm_pts) { const auto * const points = (float *) pts.data; - const auto * const k = (double *) K.data; - const auto inv_k11 = float(1 / k[0]); - const auto inv_k12 = float(-k[1] / (k[0]*k[4])); - const auto inv_k13 = float((-k[2]*k[4] + k[1]*k[5]) / (k[0]*k[4])); - const auto inv_k22 = float(1 / k[4]); - const auto inv_k23 = float(-k[5] / k[4]); - + const auto inv_k11 = float(1 / K(0, 0)); + const auto inv_k12 = float(-K(0, 1) / (K(0, 0)*K(1, 1))); + const auto inv_k13 = float((-K(0, 2)*K(1, 1) + K(0, 1)*K(1, 2)) / (K(0, 0)*K(1, 1))); + const auto inv_k22 = float(1 / K(1, 1)); + const auto inv_k23 = float(-K(1, 2) / K(1, 1)); calib_norm_pts = Mat (pts.rows, 3, pts.type()); auto * calib_norm_pts_ = (float *) calib_norm_pts.data; @@ -77,10 +73,9 @@ void Utils::calibrateAndNormalizePointsPnP (const Mat &K, const Mat &pts, Mat &c } } -void Utils::normalizeAndDecalibPointsPnP (const Mat &K_, Mat &pts, Mat &calib_norm_pts) { - const auto * const K = (double *) K_.data; - const auto k11 = (float)K[0], k12 = (float)K[1], k13 = (float)K[2], - k22 = (float)K[4], k23 = (float)K[5]; +void Utils::normalizeAndDecalibPointsPnP (const Matx33d &K_, Mat &pts, Mat &calib_norm_pts) { + const auto k11 = (float)K_(0, 0), k12 = (float)K_(0, 1), k13 = (float)K_(0, 2), + k22 = (float)K_(1, 1), k23 = (float)K_(1, 2); calib_norm_pts = Mat (pts.rows, 3, pts.type()); auto * points = (float *) pts.data; auto * calib_norm_pts_ = (float *) calib_norm_pts.data; @@ -103,20 +98,19 @@ void Utils::normalizeAndDecalibPointsPnP (const Mat &K_, Mat &pts, Mat &calib_no * 0 fy ty * 0 0 1] */ -void Utils::decomposeProjection (const Mat &P, Mat &K_, Mat &R, Mat &t, bool same_focal) { +void Utils::decomposeProjection (const Mat &P, Matx33d &K_, Mat &R, Mat &t, bool same_focal) { const Mat M = P.colRange(0,3); double scale = norm(M.row(2)); scale *= scale; - Matx33d K = Matx33d::eye(); - K(1,2) = M.row(1).dot(M.row(2)) / scale; - K(0,2) = M.row(0).dot(M.row(2)) / scale; - K(1,1) = sqrt(M.row(1).dot(M.row(1)) / scale - K(1,2)*K(1,2)); - K(0,0) = sqrt(M.row(0).dot(M.row(0)) / scale - K(0,2)*K(0,2)); + K_ = Matx33d::eye(); + K_(1,2) = M.row(1).dot(M.row(2)) / scale; + K_(0,2) = M.row(0).dot(M.row(2)) / scale; + K_(1,1) = sqrt(M.row(1).dot(M.row(1)) / scale - K_(1,2)*K_(1,2)); + K_(0,0) = sqrt(M.row(0).dot(M.row(0)) / scale - K_(0,2)*K_(0,2)); if (same_focal) - K(0,0) = K(1,1) = (K(0,0) + K(1,1)) / 2; - R = K.inv() * M / sqrt(scale); + K_(0,0) = K_(1,1) = (K_(0,0) + K_(1,1)) / 2; + R = K_.inv() * M / sqrt(scale); if (determinant(M) < 0) R *= -1; t = R * M.inv() * P.col(3); - K_ = Mat(K); } Matx33d Math::getSkewSymmetric(const Vec3d &v) { diff --git a/modules/calib3d/test/test_usac.cpp b/modules/calib3d/test/test_usac.cpp index 6e6d8cecf3..8297ab1de8 100644 --- a/modules/calib3d/test/test_usac.cpp +++ b/modules/calib3d/test/test_usac.cpp @@ -452,5 +452,32 @@ TEST(usac_testUsacParams, accuracy) { checkInliersMask(TestSolver::Homogr, inl_size, usac_params.threshold, pts1, pts2, model, mask); } +TEST(usac_solvePnPRansac, regression_21105) { + std::vector gt_inliers; + const int pts_size = 100; + double inl_ratio = 0.1; + cv::Mat img_pts, obj_pts, K1, K2; + cv::RNG &rng = cv::theRNG(); + generatePoints(rng, img_pts, obj_pts, K1, K2, false /*two calib*/, + pts_size, TestSolver ::PnP, inl_ratio, 0.15 /*noise std*/, gt_inliers); + const double conf = 0.99, thr = 2., max_iters = 1.3 * log(1 - conf) / + log(1 - pow(inl_ratio, 3 /* sample size */)); + const int flag = USAC_DEFAULT; + std::vector inliers; + cv::Matx31d rvec, tvec; + CV_Assert(cv::solvePnPRansac(obj_pts, img_pts, K1, cv::noArray(), rvec, tvec, + false, (int)max_iters, (float)thr, conf, inliers, flag)); + + cv::Mat zero_column = cv::Mat::zeros(3, 1, K1.type()); + cv::hconcat(K1, zero_column, K1); + cv::Mat K1_copy = K1.colRange(0, 3); + std::vector inliers_copy; + cv::Matx31d rvec_copy, tvec_copy; + CV_Assert(cv::solvePnPRansac(obj_pts, img_pts, K1_copy, cv::noArray(), rvec_copy, tvec_copy, + false, (int)max_iters, (float)thr, conf, inliers_copy, flag)); + EXPECT_EQ(rvec, rvec_copy); + EXPECT_EQ(tvec, tvec_copy); + EXPECT_EQ(inliers, inliers_copy); +} }} // namespace From 762045648614691240e950bcfbd141d19fdc334b Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 6 Dec 2021 09:54:48 +0300 Subject: [PATCH 161/226] videoio(MSMF): add queue for async ReadSample() --- modules/videoio/src/cap_msmf.cpp | 87 +++++++++++++++++++--------- modules/videoio/test/test_camera.cpp | 15 ++++- 2 files changed, 72 insertions(+), 30 deletions(-) diff --git a/modules/videoio/src/cap_msmf.cpp b/modules/videoio/src/cap_msmf.cpp index fe3d261d34..fcc9260584 100644 --- a/modules/videoio/src/cap_msmf.cpp +++ b/modules/videoio/src/cap_msmf.cpp @@ -31,6 +31,7 @@ #endif #include #include +#include #include #include #include @@ -308,8 +309,10 @@ private: class SourceReaderCB : public IMFSourceReaderCallback { public: + static const size_t MSMF_READER_MAX_QUEUE_SIZE = 3; + SourceReaderCB() : - m_nRefCount(0), m_hEvent(CreateEvent(NULL, FALSE, FALSE, NULL)), m_bEOS(FALSE), m_hrStatus(S_OK), m_reader(NULL), m_dwStreamIndex(0), m_lastSampleTimestamp(0) + m_nRefCount(0), m_hEvent(CreateEvent(NULL, FALSE, FALSE, NULL)), m_bEOS(FALSE), m_hrStatus(S_OK), m_reader(NULL), m_dwStreamIndex(0) { } @@ -354,12 +357,19 @@ public: if (pSample) { CV_LOG_DEBUG(NULL, "videoio(MSMF): got frame at " << llTimestamp); - if (m_lastSample.Get()) + if (m_capturedFrames.size() >= MSMF_READER_MAX_QUEUE_SIZE) { - CV_LOG_DEBUG(NULL, "videoio(MSMF): drop frame (not processed)"); +#if 0 + CV_LOG_DEBUG(NULL, "videoio(MSMF): drop frame (not processed). Timestamp=" << m_capturedFrames.front().timestamp); + m_capturedFrames.pop(); +#else + // this branch reduces latency if we drop frames due to slow processing. + // avoid fetching of already outdated frames from the queue's front. + CV_LOG_DEBUG(NULL, "videoio(MSMF): drop previous frames (not processed): " << m_capturedFrames.size()); + std::queue().swap(m_capturedFrames); // similar to missing m_capturedFrames.clean(); +#endif } - m_lastSampleTimestamp = llTimestamp; - m_lastSample = pSample; + m_capturedFrames.emplace(CapturedFrameInfo{ llTimestamp, _ComPtr(pSample), hrStatus }); } } else @@ -396,32 +406,45 @@ public: return S_OK; } - HRESULT Wait(DWORD dwMilliseconds, _ComPtr& videoSample, BOOL& pbEOS) + HRESULT Wait(DWORD dwMilliseconds, _ComPtr& mediaSample, LONGLONG& sampleTimestamp, BOOL& pbEOS) { pbEOS = FALSE; - DWORD dwResult = WaitForSingleObject(m_hEvent, dwMilliseconds); - if (dwResult == WAIT_TIMEOUT) + for (;;) { - return E_PENDING; - } - else if (dwResult != WAIT_OBJECT_0) - { - return HRESULT_FROM_WIN32(GetLastError()); - } + { + cv::AutoLock lock(m_mutex); - pbEOS = m_bEOS; - if (!pbEOS) - { - cv::AutoLock lock(m_mutex); - videoSample = m_lastSample; - CV_Assert(videoSample); - m_lastSample.Release(); - ResetEvent(m_hEvent); // event is auto-reset, but we need this forced reset due time gap between wait() and mutex hold. - } + pbEOS = m_bEOS && m_capturedFrames.empty(); + if (pbEOS) + return m_hrStatus; - return m_hrStatus; + if (!m_capturedFrames.empty()) + { + CV_Assert(!m_capturedFrames.empty()); + CapturedFrameInfo frameInfo = m_capturedFrames.front(); m_capturedFrames.pop(); + CV_LOG_DEBUG(NULL, "videoio(MSMF): handle frame at " << frameInfo.timestamp); + mediaSample = frameInfo.sample; + CV_Assert(mediaSample); + sampleTimestamp = frameInfo.timestamp; + ResetEvent(m_hEvent); // event is auto-reset, but we need this forced reset due time gap between wait() and mutex hold. + return frameInfo.hrStatus; + } + } + + CV_LOG_DEBUG(NULL, "videoio(MSMF): waiting for frame... "); + DWORD dwResult = WaitForSingleObject(m_hEvent, dwMilliseconds); + if (dwResult == WAIT_TIMEOUT) + { + return E_PENDING; + } + else if (dwResult != WAIT_OBJECT_0) + { + return HRESULT_FROM_WIN32(GetLastError()); + } + } } + private: // Destructor is private. Caller should call Release. virtual ~SourceReaderCB() @@ -438,8 +461,14 @@ public: IMFSourceReader *m_reader; DWORD m_dwStreamIndex; - LONGLONG m_lastSampleTimestamp; - _ComPtr m_lastSample; + + struct CapturedFrameInfo { + LONGLONG timestamp; + _ComPtr sample; + HRESULT hrStatus; + }; + + std::queue m_capturedFrames; }; //================================================================================================== @@ -902,7 +931,7 @@ bool CvCapture_MSMF::grabFrame() } } BOOL bEOS = false; - if (FAILED(hr = reader->Wait(10000, videoSample, bEOS))) // 10 sec + if (FAILED(hr = reader->Wait(10000, videoSample, sampleTime, bEOS))) // 10 sec { CV_LOG_WARNING(NULL, "videoio(MSMF): can't grab frame. Error: " << hr); return false; @@ -912,7 +941,7 @@ bool CvCapture_MSMF::grabFrame() CV_LOG_WARNING(NULL, "videoio(MSMF): EOS signal. Capture stream is lost"); return false; } - sampleTime = reader->m_lastSampleTimestamp; + CV_LOG_DEBUG(NULL, "videoio(MSMF): grabbed frame " << sampleTime); return true; } else if (isOpen) @@ -991,6 +1020,7 @@ bool CvCapture_MSMF::grabFrame() bool CvCapture_MSMF::retrieveFrame(int, cv::OutputArray frame) { CV_TRACE_FUNCTION(); + CV_LOG_DEBUG(NULL, "videoio(MSMF): retrieve video frame start..."); do { if (!videoSample) @@ -1082,6 +1112,7 @@ bool CvCapture_MSMF::retrieveFrame(int, cv::OutputArray frame) buffer2d->Unlock2D(); else buf->Unlock(); + CV_LOG_DEBUG(NULL, "videoio(MSMF): retrieve video frame done!"); return !frame.empty(); } while (0); diff --git a/modules/videoio/test/test_camera.cpp b/modules/videoio/test/test_camera.cpp index e82285ad5e..135eb8eea5 100644 --- a/modules/videoio/test/test_camera.cpp +++ b/modules/videoio/test/test_camera.cpp @@ -25,15 +25,26 @@ static void test_readFrames(/*const*/ VideoCapture& capture, const int N = 100, const bool validTickAndFps = cvTickFreq != 0 && fps != 0.; testTimestamps &= validTickAndFps; + double frame0ts = 0; + for (int i = 0; i < N; i++) { SCOPED_TRACE(cv::format("frame=%d", i)); capture >> frame; - const int64 sysTimeCurr = cv::getTickCount(); - const double camTimeCurr = capture.get(cv::CAP_PROP_POS_MSEC); ASSERT_FALSE(frame.empty()); + const int64 sysTimeCurr = cv::getTickCount(); + double camTimeCurr = capture.get(cv::CAP_PROP_POS_MSEC); + if (i == 0) + frame0ts = camTimeCurr; + camTimeCurr -= frame0ts; // normalized timestamp based on the first frame + + if (cvtest::debugLevel > 0) + { + std::cout << i << ": " << camTimeCurr << std::endl; + } + // Do we have a previous frame? if (i > 0 && testTimestamps) { From 542b3e8a64a1516657ec84fd9b2acaaa2baf6440 Mon Sep 17 00:00:00 2001 From: Vincent Rabaud Date: Mon, 13 Dec 2021 23:43:49 +0100 Subject: [PATCH 162/226] Fix harmless signed integer overflow. When computing: t1 = (bayer[1] + bayer[bayer_step] + bayer[bayer_step+2] + bayer[bayer_step*2+1])*G2Y; there is a T (unsigned short or char) multiplied by an int which can overflow. Then again, it is stored to t1 which is unsigned so the overflow disappears. Keeping all unsigned is safer. --- modules/imgproc/src/demosaicing.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/imgproc/src/demosaicing.cpp b/modules/imgproc/src/demosaicing.cpp index 03bc781046..27dfc1520c 100644 --- a/modules/imgproc/src/demosaicing.cpp +++ b/modules/imgproc/src/demosaicing.cpp @@ -603,7 +603,7 @@ public: virtual void operator ()(const Range& range) const CV_OVERRIDE { SIMDInterpolator vecOp; - const int G2Y = 9617; + const unsigned G2Y = 9617; const int SHIFT = 14; const T* bayer0 = srcmat.ptr(); From 952f8e38cf9f29c2b38b6a8a59fa82cc90299626 Mon Sep 17 00:00:00 2001 From: alessandro faria Date: Tue, 14 Dec 2021 03:21:31 -0300 Subject: [PATCH 163/226] Illustration about realsense --- doc/tutorials/app/images/realsense.jpg | Bin 0 -> 75046 bytes doc/tutorials/app/intelperc.markdown | 2 ++ 2 files changed, 2 insertions(+) create mode 100644 doc/tutorials/app/images/realsense.jpg diff --git a/doc/tutorials/app/images/realsense.jpg b/doc/tutorials/app/images/realsense.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5e4ec67fa7ae231aada066b10c7ae8b4314c77d6 GIT binary patch literal 75046 zcmbTe1wd5YyEnQgfFYfs1SE!`n;Af9h90`RQ$bW(MU(*nhwfC6F2N#gB&8$;u>~mw z6+uBoxogn>dEfh;d%k`^+StIz0D{3_5DEN2yKk|<`lkZ? zA;`!`5<)=`!~pGs!6ABp62XlIm>E1X!OaRoKrry6-aIgxy=N)F2maz3fW`l&fdh>H zoA)Zf1c2GW^DemA;Qy2pZ~?hj&KL;)Mh8L1PK2KbJrU#uL1qA9ofWW_L0+dJDye(v z{_`|6GPA@IWbyK{cmgP0o`6@y6IAhvSa}6iys|1m1>%E>>dz=*%$ zQPjMDaU8&O*?;N<5TgH!(Ev02&6^A&qO<8BJsJqg1~?OZ0g*t{;4AAdeg$y$U%W_- zEc;6{US1wn_K#fQWq;(_xew6u7mqPQ(Ef^lc!yQe?a@!wC%o#PaKyhbvg!|CA3!Tm=I@~ZL*Sb4lE_$c82AH$^BAZQr@K}CP_X#)bN`KcEzxQl^M zsPSeNwp2rfL56mKKo9})A-wHa_mo5>8nT1`VE|9Xf0-AdV*H*d!>Ho3BmPO_K@GS3 z7xwy_288?XLZChnCk{cue;C^5l4*!dR83!Z&q=(M|jQoQUPeC~zdktqV zufONNu-Ct^&%dzmKlm_Wk4GQ?M8x4g{IURi#-F$s4r~IEbQrdWLE}WE+pX+jpi@M_ zVfY^A1i0cbat|YHjUnhT?H_zl31FH%%mC7K__O|@8Ahd-`Js|4q*d!Y%$_hzPLdzp&N6u=T&NEd=<0U|Q$;KKR@H#pI9G z{`UXD4kiBk{-j6bfHLX-V5?Ao8UEl0fM&a;|B~mD-NJw2OLoiu!SL61i~qq0Nnf=+ znQ#M{botusVMTzS``Z2|9JvozCe`*37UsZ874|S(9t8L*{tH{}VWhmQGUSW?gPrtk zpb(Zl%nSMf@sK~^pgkh>>@feaVVGsezP&WS=8-lb?Ei&*LOAy@;>=kH3faGhfej$T ze8=`MkTUX=zxy8E4{)-t`hTS<2R#*)W}s1Ijj!b%mIQdp7rTd50R9c)AxldK8-j`~ zUI8ydI3z>;f^787bS)tKAsOX^GI#}o4E_)vl0T(*5F#k+5)=pr4?@NyU1cQ&ypDqW zpTt-^fnaKZ7bki7g+&}cECJd)URh2_4i8$ttdgP{7B7oeR8&$?!z#%tDk>htt6?F9 zznAQvryBPEJ>Y*XjG&?zg9i^m ziYf~72Nl5RaZp8BSy2fxw=ma-NWP(1n{cmiKdhOT&yj%Revr49&rz9UUIE8t!1&}5 zrgW4LbSTUVA}A>;K;l?=tOORXa7b3=;6XL)pPwoOSw&?kNq=JxDaoqHgP(s>%gd^$ zC>He0zn0@0z~vDy*x;1h+&XK}lXwMHzzr*C(~VrQV=%!~TpE)SeOc7bk-l zI2g&PouVs9g7`bCQ+sFHe_@Wlc(3z>|0%~Cq=)~-G+ZD9xKMk4yZ@elPxcm(+QczAiS!m?smoU{-xulOMeX?b8z ziUMLP8p;H9Sp`J`wGbFRJv|d66Bjcx7eSC$a8JkoM_;?05C#P$!vM252#$dvFtFVo zNC41DLmh;`F!c8VLqC#+mJUVFzz7nwf@TdvAmB&@4Gj_rQlF(x4UiZbRzbWrEt{1W z9WI2O5Svww64JTxki&ZXv#^49XdFEQ=YB439uZM7aS2I9rGtl*RaABL^bHJ2M#eU_ zcJ>aAPR>5Qe*TA#1O$eKM?{`Hbvo*Nd_v*{N>XxmPHtX)L19sGMP*fWO>JF$!_DTF z*0%PJTerKqdwTo&9}NsnOin$So_RX^?CsqAyM@K2<(2pAUp6+se%spq{)5UFmCyfN ze;E5;_`(3b;7B9_Nk`=i20saI1O`bXh^J-MwxaV2VZ#w(QS3TdQN=Qx)`#LbthowkEryz zB|jA8B6(S@;+b}Cb;cTg($6}t>Iq&M&zbaPin+*BdwTsqzaTQ(CSGk z#5D0%-1@380N+=7WafQW-HW5D`fbr`#?x9`)$seCr%P>a8}SW%>=&&RO`#dB@?Wn_ z^#1v5q+DzD5i?<^?;PLM3C@YyL!W z`D&f@pZ3Z6KPkJ=OM$`4Blqf=M({svjO`Yai+#!%E;1i5bD1@2enyr&rZ`b^mHk@{ zS1+8#ILo@UP9sp?B9PXA^%83h=CbjrjAyYNJ>lyDt&SMmI7z5cs2&;NU{Zhzmew!dlwqGrI1)PR5w{ z&D9;ZacP~*CvlMT#UlNYscCgDZ@4P9TfVq)W|UWfZDyeTqSg=8mFrhM&cA(CatvWo z_ue9bTMe>G|8+F+i9TgYm?!X&jCNPi0pXvUO#^eE=LKGUxZO0q3we(T{9@21Ug#nb zx}7h+&+60C=p)}Md3&p=d}`{&8woCJb4aOvuAsTpG2w1jk&3kK65_gO*?xX5Bcd7q z*TkrSI@nI-?^cbz>Z*rQhKJwQ9Y$1Tgb1DAg`(HWSRxO^GIsT$oeoIP{_JY|Dib(8 zZ+1jb=kdD;k2`Lf_9QX500%+Hldc`WYZ(cY)ar18Qji8@A zR|FF8`Y8q#9Qu^_NoFPJoZk2u+9g8i{^{qxy-rGMxG`i*=j|xP?EkUVa^aav$JFnd z+alW`b;duhDydmo*lgVRZRZ=Un6mxVJLctu9W$vi70$6r{o&j9i%Yw_ccC9Br7)d| zpqd%!Yz=#rnvLAX5N*Mi=S$x>*a>u};SE2XkQG>l??P#jnyAn2S=H~gg4)v^I?OrV z1+D!EF)3o{uax$S^wSTE3v_z7Q{Vj-7FcpGy@p;6 z+vKiIOWTE{OD_D(4WujTUPm}hxQgDZ_?U9{TP^k+pO%%|{e(FUu^##F#b+l-ozdf^ zeCH~zjnvM*kZ=n6vd<&C!vh_y?pUo_dqM1|$#(i@^4g1l*ySNUlF(I?&QkeZC_Z_1 z^%FV8u5HfoQQv3f7jzpO&pcr{BwEzUTa7FG?bb6cv=F;~+O``Cd>L^$vkScz676vF zm`^V{c1v+Y^JqRVT~Yke?@X5$i2RNFIrw@Pno>nwu29NSL!Av7=;n?tX?V=jB`Yxx zF%L(ETz=b5-1loUG)CHi<4nWrK*IO-f-noQt?JPJ4aLqaErIRYPM&*Z?f0`<>2Zq+ z0o@Xs=?1Tt+_mf7Pn{U@om9L?d3-8+@zbzYcEkKeqd?I;GsSwofaIRV&lz7s8d#ni ze*K`c3sto_mc0C#W9fYv^K56cj8XI@osb&oO-|>|ndy7&lpWo}NAmLztuf)5#2vHD zH9JN>IC*`c+t}PWbB?3^t&2pf0Q$M(^Sf39p{8EdHZ7$uV|YBR3mn$f*NT+xnJ!)L z2-d;|=6;LHdwq!Q#mFw?#?`)Y?SugRR|kNvxRmWeH6b~_IlQ&v8DeHW*a-M$2u!Dx z^Roq1zj*$lUor6g`gPBv%Y6I03sm!qj)e(#a$6o^zY&cacc68-b8Fwv&KTp5(&5`J zKgEw>k%p`ZuB9Q?t9|KFTF7hbj{}F#F*keMUyb2eZYw@05M{5ca@Np%e3R07WM}rN zpU|jBnCtJFhyxa~bJI4*o5qFS2KFa!>DM2u7A|#bLFqNd#Iqbtzq5V*ipMjl=ST1Z zJ_0|Qr3+6Y?$dB0j{c14p`Wzeh3@BHx-YTL5OF8re*G@=bcRu&m&>VZ;O#pxyIiExZB#cvu_u2 zzqPWX|Mn?NVfOv_n>SetZCX}VviII{_w%H!icOe+f`kmE$?q@p9LUqsEb554{`&Xr zB27US^_Q}CU%S~db|O1=p|)*roA`COgx1EywEnLU^Dm0E+$lY(Ub|3~_2pr!A=%Ez zkZ9eB;f4bAC_E=^A++r0)-S1sbW&}CU(%ELkUYP47ui1N)PJ7U;@LRs zF&~Y7G*G?r>ImXO-e-l)UNzL$oK~1W7B-dR+SfY23o$M~%wDog4=~)hd44JAH}Afl znYiuq{XCzNEPfn&>v*~cy;G-o|3O-4*dvp9(Gv0}rB8g019J(}Eow$VANQAxV83V7 zTkSMEIrnGm=r<+qcfQnja2LuCDM|Ux0F2j7wqRzpLSin_ zGV^bDvL<{Sbj?&~HHhhaQZP|<=~y-^qwGRZ1b?fI#RI>1y=$$Vee`|$;SWA%KdL@_ z-#129ONOrOcgt8&_vrxU0i*L?T!{?9r|QnWcd1W%GBugGe>ilT?`D`~L=|Dj&yn`$ ztv>wkFz4Gev)9*IE{2wUPYEs*jM_-A{ovHrHp9mCwv93>bnT_3`97s(P1@}kxo-=t zAFmJb^u6+ao^GVkyrS!^nP~R7sQSxjtH>R{K%40*|8tcwEJX}=%S#I1+8K)8xRBZu z`@OMd2X)nFp0o92)zhbE6Fp?@BWXGx(@G7l?pV#8S&e_dwEyS0P>gYatJ%}7^?4WA z^u*HV=f9QI-3b$}eia=S{#L)3qu(A%RobrhJ=6N4mJTDgZDZ5tydm$=CO+C|^qbvL z_Dn|d;)J?x)hj1`wH{fYKBs+y-L>XVmd+ba1#O-`dcei(;K8bLili3CwR^kYInFI3 zgz3Sz5#leXHm2)DL&J%m8n-;Gn|>=k=~|9eXKv^XZT1#@IPkqU!daasKWmZ6i!V5G z&3oUH>$6ioZ9+MY$rKH2o>;H<=;yg4%$UOHxzAK;-%JG0d86flR<|35*w6{}_-trk z=d9GFBj@>HbSv(opDS%sdXCn)yl|!=eV$kL%K6IidgQGckGIJz(c&&<7g|>`j$Bp~ zP&*YUCERhMb#-GOrkQMejxPCp5B|AkRo4?AhP!OsNP)rov6tWOLKi;5Uiuu{HZc6+ z1Os!K5U_RqN<2Qv;mGigC>liR)0 ziQk&^wp}V+Mix)=`>WE6dDFdNk5OG&3!1;)?&!_XPzmAI5X+Kc!ryo_-Di}pygY1M zy7=*MWY*ByD0Er-`@S*`^mTo0AvYb1Y*9PcHuKeE%FAaqp0z4TiHxfUgdLW)B6Nq2 zuU%GSctyzu%7w4BX z6o}l)nz`jHQMS(mf}!^kuF*BG64ZnpO~2nz&fI79TR+0#$L!1#nz`@o`jOlmL}8y) zvRcS@hs(WcI-waYKa$7HBG(-laZ5j+pKkK9>vhVK47hXAU6OOUjWe?DVU2)-Y*_5* z_V2s_>rurVQ+Gqzo-am^k1#wAZZPwGIf&k5pUio6RVF32sL(*;$8TGI;j@`Ej|uJ; z%i^#$Rl8S}p_wjvZZFQE{32nfa7+pG++r!oI_PE4{NX zD6nT$%)ePU8*5FZ6I#FKyge79ncK0t^xYwgMKZt3SLRk>Gpo^iGpw3`zuF??HxlTPBp`bjIK6=tw zhY&~4ZOeE{@8FeYU6qEb&aWz8Gu@CX`J7m)ILff{?de0uSS=eD{Tr56_#D3X2Xk$i zz9I%p84au~?+8UUj7WT~b9RQloJdoJwX7p~dIBg{$E31$A&vbRQ$Am=@;=&n%;EO# zr?ae2>gglRvO4FVu8;D!ZeB>;Q7;wyJTelCc-*(m^(|y_V+AaU7@5Y^Z@YdF4-sWm zeKUU-8FphT>Kr(3h*}b3&|sm#7t$VjL1ZFs)*Ku7%DBbig)3w5Hgjd2p=%SQ7i2B07t% zs%AOiTeozT`v*%PE#os3Qdz6Hvxj_odDOY^U8wcVYI}yj$MhK0j_^svM5nl(j=u5E zdi3+>o*3`*3#9Ni!p*Vq0kebJVK2n)18c|P{ z=f^frdY^QAA<_Bhz_~54FYnGC>2O?#u08%he3N5(N7?J}yqbzb>T%?6js%sTcF$X^ zI5xQQQbP^Sgt%vYUV2zC{vKgHB~yGL_`#i*55-oDVpn6198NP2$+9ZacrPJr>zd7> ze|X@E3+*$m(ph+gP3lSC!79OZwSBkN*$-}93^y#bdSV$~@w~@!BvfW0nIx=%m_D0W zcr#4x;1MjwP$bGoaKe1!P6!PA_ z%R_pBCD?G@b@(zvwDi(1vq6D?m>*A`OD7w-rlwcieD?nRCx_ArZl6SDD>8A}2jjA*GH0@{PKP=a|ybCSl-;8-<74!9d>I1|lgXkvrry1Y!9(3Ic z`)NHaocg8ZoFmQoawOt42VdWnqs%SM(xqN$C)p9mjJ}lXH|DK(IHnywzIN6c%d(ib zk3NbCoc`i&a$IJ-8}BfH6)^t#dW}A=@5|HSdeMW|CA&+D9@dJMGtQbs3^qD#u>V@< zud8_L{e#fUYCtzw2Gb5Uq;2RIdxkZteB6H^@vH>xLeQHDZlPA z&Y7$l5j|6|W3YYnU~A;F`^ju;=2|hZ`P08YZ(TF8ug|JOEOfv5#M(E)xKeIKL4YqXLXjbyHIJ!ml)Np5&M}DozCE+S(gn82cvI78fl_WdDjvXssjQwM*|c3 z*B=ZYmFu~!*Z#DlEv=|yUwhrA*GH)bt)G9BHVA#UrDSMy$d)u)?AcynODaKvcUcFb zx(?j2`}Mj1NAGG^z0$9H4+i{DT5H#@`D~}Wp}cM9X^nSzM-jl&t(OZBe!EHaing8! zcqZmbxP);QD@#-xQaheE@U7{W>XzAL&|DGD=<4TLL!ICkv1{oUW>y@H4yYyAN%xmo z<+!z9(B%2RGj3GtapPI($$_a1FI@!Z*%<3Pxs<1d53tfl=pU-Q`MSAtqYnwy}yHKJ(jE=T35({||yzNp)3d8dCM?Ld(^&)FIrwk;R+bhz| zaEwY4mO@iryepxyeYk7P7|dZbv;eYxcck+v?*s3m%`h2FWUJfPamtwdh4&25)*lQ zw((eP;p4#%5?pcz@`TUPmbEmu-7w2vm-e}@THJiQ7Gr0d9b#|fSk-8Cdwl+AMEcBv zs9)Y}VZQQKebx9{?MCW^iO?V<{mvH(#rHbBG`W9kQ1W2b0eynv>kl^mdV%;)owY9l z>cfPkzxWCph;jXVlYHZT+9u3!^>&qg+_M8ub?TnoR6HCkFjzLVc`=3Z;@NPL^o{Rf z$Cq~^=I;e^R>=_QgK8K_<-)8K_PN9;oiH;+sL#g=uOwK5A)?y?MMoT^tn$j3^8!nl zZS3pq>fnoZ1xR0848ZjWnUS zb0@wFiDN!BGTN2$VHES?_d08jQQ<;ZPWx5p+lqUK@K*IJ2}6?gr ztJ1R<0$1)1UUetWmNC+-TufRaT`9F-Urpd*XFC3BH6VR^#dySu+Q5pv0_khU;2#x> z6s<@I(Oe~wLyFL^zJjxcCd7TO4Su8-utUE#FRtJX#G#jFGA79f#}JG*M)e1i%__)3 zDm6S8k0~}muFmJ)rv;oDgAa)cDL2T-IOlvU9y!+nQ_e0GZm75<4j(Icm7&^DT+HuG zmKI%HDyyGLsOv>EL8iVn1xf;e66EUo_{H2wGf0H1x+0Ugmo~p}3Su)VHlJdEoSiL< zox!)Z36b_lMKYvdq+Jn9Z)9Fo<{f5+yG)9*6xyAIcIV*NjdXQAEUhgJj7;@`&jI)g z%7T23g&~1k?bz|~P-{aSEZF|W(k(zt5DRb#?}NC#e8PgYEiFy|vkCv-pWmjb{sah` zl%baO-{b$cA^vM843{7m4ao%3~&ZA zEfzvUb!e=hIK`SQnJg%+mu#Cnj%EM{mx3W0ECfU8a54FCK{koN71RUShcMD%U)K65S7OsZsyKhDm0~45b?zqG4<-;z@Zh}G8*Q9t)!EUqzXHn<5NmJyn;re z41R$~wzL$4${H?6m~jn2iFk4!gcGTRu~5A95U_D3PXt#V0?BS@nk6Jt>Tvqo)9D*o zk_t|j9{J&A6?=T}Tn@1KVOxEYXPF}^m&6W(${n?jBv(6K^ z9i($*q}Yp8yl%GVx;&XPoDp|oijun@37 zI0qa<>I3xv$%&KMiL~?I1zE3hYaC-b7t&&mkf&>eZ>=w zsVz)1z7wfOY1x?Dh6ESD0laxsDGAzqE8HzNjm>}7Dfvn63xvl*-%Jvc)njf4_LmD!mz z*U--Qi(wIwGu0;aLv#7{Fl&jM&goq)%|=9RZ7jMDLJwt(pT5I^VHMPA!x44RWH26} z;bdW=Al0;eKojwVKsB8{TwRn*s1E{$;x`?m;-;hTU%bk9Wa=Kx%L8)~?~Zoac_aOE zW@j67@6*?oc;6z~7Ia)p54!8gg|gS?b7IT2CEJL`R!tmL%ulll`8w=&1%q>>vI#>G zK?A7+-glxTgJ5@=4=h!plgcP90+xD2iXnI<6~YYm5q?>WHXXUOHgh#jr7~C8Hn{P# z9Vs3%QXQC4ITp598lgWN1o|KH8g0H)`N4~6{&9N5Ba zvxgTSbn67Gjrn9v~#L1POmyFQ{_hdW1!e9mrkI;q7ZDpcf)$`p~MJe~|XL$hN_JZT|SPorkf>b#eO2@|O` zfF^)KLV^%V2UF;qO9Y~%1q@M}GM7za`m`b){JMdv4`S)(Xh9RbD$e^xXI-r6EmKh; zjJEAf4a}lFO8m@~%T{5y=I9xUtwIrhsAZlDk4@B?ng48f@leIWP2L6!hO?weRjKh7 zIvx^aB7(|*!Nd!~tB`{Bq|8oQe5!9EM1s-yP`fOyelwm}mSW}bK6!Qi=JogGo^TW` zMm779r8UJmJ95=LILUTW`}PoXZK!j#FN)*h{J8gHnj|MK9ZNJ?9Kv;vtxPrrsN%42 z7?2_8e<89q$r1yz*4*c}Fg~VQ>vYCyjH?eYDVd8;h4(cE_Y>c7jN#L``XHa?jm1j= z*6hTairGoX!wrp;J_O>u^KiD5p1*4iqSP~0pMlI$>fx|dqesL-)_nxu>7E7wwNjWdR1MX!v&RW(uY2d#G*9bYbzxpQry;RC6NKRE^sRrWMc8;4?92TU|HbIG;kBTHT`sAucN{po2 zy8L<*s-Jj;1Ox~bpWb}O?H3*g%aM~&%x6{HpAQT4IF?WoH}NnPv`ld!UL!peH;Y;5 z>KidBV`j{Iq372sopftyj2V2$pn=ARwuo~<0}!i3hV-eZjkeHy)+ctE5VEBRZ#&lz zK#1!q&t}N%;>N^EUW%$emvQlKtF*M!tQXEVi76{9I@>ldU9rPkN3wy^8z*rEcVjyA z=F=%lQW}Ypre9r|^1gd~=7=}u-Iu`5WD!a-Tb6s^W7!H1Uib3y859$ zFynorB9`YNcDT^B)rmc>0!L&gG9#B0tSL+h$B155Mt0fHrF3eL1O1LaWg`?jGQklc zPs>wm#;ZAX!PW1&(D}kvl4PA)R{dkajk!V^wx*X_?Sx~+**jLh<$bEdP%;M~47O#U zaDeyr7yHv`(wgM6lG1t9HU(FuW3qjtQ0f(3|1RCk&X?4Pk%Wj4tzf=Pl9ccbVSByh z()4~*!{9V7d2m9?Ev|0ua=?g>EKVM4N`4r9>s{6AjJ*bTr#m~5Tn%Oyq(}9bxC-&K z8>qA_b`je$;q1nVc?GAp+@rge%Y$$vn%JObhJ)dfcs;ymL-P6D{aG2nUXr=oJ9p$1Y z7w+8}Hua4I$CSC4S*NUn7q-9VIel~dYW;ctqoMgLX?}io?5TCO!sXBfV}>I{7njDC z`cQTqh^HmFigVM4A%Lc6ZQ-YHx~-%(Lzg`m=%7x>6CtHOeD8oJV%m;>Niy5yjTBdU z(v$3GTaKTRW+d_yg7>er5+e(myjtFfV)$&&=Z zFUfP!*vKVlo(WS>pHVs5(3r$3--8Sa5u7qG#<2^zOx8QCUNWxoO=Z&&e|;Rf9)C3X zeI#C7=!V>KY7HKR1R4f24Qg-BG|xSnadNaogP*@~-Tr%BRcg3R9i4;g9o7&z>m$u# zhMH4(Ca#isXQbtFT<d~0Ic;HFe4O~)!-%jtMQI>2zH{&)%!k~e59+o)A$ z)tmMmU=wz~c)=#`lJ@)P4=qCcCzR_0q_Pc-71>ntEwuxhQ^rlptSqG`Cl@2vNndV< zoYB=e^f_>N=r~Q}uiEqXSSz@(z!qw`Eu>F14;Cp&w&li(UXqWacrw7@tfrvdk~R+! z+?<#4kDA>|Yy7wf7X6=cy^}ZTo)H!jT&$za#Riw4wB5%e2bb3Q1$uCCg)lh-!n)}wFr^?R<(kY%0 zje)`<$NUwMyu3!%8;~?bmoM&7h!SgPz5$vE(z{@~)*N-^tnH}z-Ly!Xp`pdFa+nR8 z{De4nlY^~EZiT5yT38TD6zb%PxF{8VIA7S<*oc#rx0%6YGpo-iCZR;pt zEyDz5b*wm5Wn13;9z6IKCDF<8RHrX|_*;Z|+NsqK7mDkMzP89oYY}!kACK#Pn?^D+ zPADB%>h{Q~qlJ#Bqc~z*Ig=GW|H;HNN7GzJuZYb2@oW>0?u>dqhQ_#uI6;h{p9qIp zt5e~KeZPm2-}TqVO0hI=W-eRocQujEGIE}P7pOqmmOXLgn+6JZoLdR+Z8(O;My4*+ zEt-jE)+1?CRP7J9g*1$W`z`p9A9~9}O)Dkdc_nvW#}wRGdNL3=tOIeQw)sl~7ZpV- zjeT7^8DhD8?}Rko9^o{OrT1aTrZD;DrRJWkJoK908Z$W2~1@XX_+qX6<)~ zSMRYNnx_o1;(1!4R`R>?#OCK8OP4c}Kd3)QS^g5{8zsJB{bj=&!I9*LVaR5k@r1`+ zlH_1B!>kBB_1~OhohEKNM>I}4$5Fu)J6AH z2xdEp*|(oKbo9_KxyvgWn(h|;KRY6lRl>jWx@)Rntdkuxb2+eNdn~=9WlllB`^qN^ z*TgZn)&i#$#TNlhLSSi{l)0^0%VtFnOR{99p%)&9sd&OMb|Pr^r1z6Um!^w^>M#Dl z$wFmF>ikm=LLx~5m%{rf5p7Op8}#tZB;9%zxoc)&kDG26pN)QV z!Da0GxzfqYqZ?O`B_YV;Q>30MI0H!>3xkmGS`?Cf!INPB%VS?zJ`=@ z)J2WZ#;Z%~CAy_iY#DAUf#wqBo&};d41@M%S;@j>*(QopK~pQPPhCV(hSYbScnMkR z);)TXOipsD562OnRZhQMCEUbvG~DKBnRF;lLtt2O=tNRiG@VU}w}FwZAg%>0qT|t4 zkTuGd#Wu~?`RwTaaa%Hk6h!IZBz0hVM4fAtm50|eT<jid=U%dI_g4+Fdo)2d3M-{=$lahso zDU?RrbL6djx(il}G!_>elYgPgr< z{VVmiPu0H$2)nvo$}3)?jG)K?B5^G~yzEIRj0pp&-xJ-KJ%16ZArulFOVroNq>;~XD|j~T4VH0&U~dJ3 zj@Q#cr;FD1zQ6sd%}_(G!0yJ)&G{j(K>L=fj>UDfoD!Ba>F6XWSS(Uk#EVVLf8dyq zzs~JCqq;iZ0cOvjkzbette;uS!8#OJan^{I)_dQZ074L%2pdf@LAu(UL?R zaH5%#%g&C=t>qLrHuALsthF#eBepnkN|I%~9^07HI>UNf!%JDVnj-UoB{L+WcX7=l zliWBZdc7lD*s=;@V#uVuk?=HjHg|9}0Rvj1;r%whW zDbod(I%5W=Cz*qOgrZdc(2F#6WU>xgn*@hltp2&^#^Vb#3``Njp=T`_kHf*l%A0!~ z`i|E(``+t*%q)5jAsIFko@akP)7}yXxnbH6rCn84g^+dN?TzN>5MFX?f!O#;?0)rIhrgRwjq87FOI|5)pRq z@U5tGkY2djlP28ADLlkzmc%t=>r_K}Ze*o87G!1=!deuPCzDW*jU}3h{Ah_PeyT?y z>!i+pT2OxwRWw;S*?)HF@xsTZ`@2v~&hxb|PdgcU2t@fBWm-HFk-gkHUXaOC5SV_p zFx3oE$rU&p(2Kr?NbJV>1^Y?2(CVd&sH$QAiSFV%K9_QeU#HJbUToY?;HYqP+Mhq` zG&BZlsyp~cqAMRHZ`D5O1z$~G__T5=;`UBL6;Pz8tQj# zbS-p6%0dBcy}6?4dgs3$+WZogVbL{G`rBe?YwG^x@%10S z&Ycm4!ZI{D#8*qKm>|$XQGhPcH{o!$N!HFt(3RR@&{-H;ijD@h^ZQzK>9Z?V+HAMP zSgz%txAtxtk8=@y_bQUJYB8wkMtrxQ?Eoha>q44yam8j4&0tHGjFQ2vGw*Oe)o*3@ zD3N5=KU!1+{fl4Wy26`MzxiosX;ZZ%n$_1sYw0JSd#PrI1sgY(`|Z(Ne3L30KYk$n z5k?)cP62`_WU#rC3!9pu;-^?`tGXl(Qaq`ULP;)1QL<1JvS9@mhXDQLdAp$(SMlt# zlBQC}+zS_yBuGxAU0ghM?ig=JIiJ9ZiPx)J@fTc2{iA+spa*Zuc3BFjU-RPAX{b%0b7;72K1n$*C7hdwlbRx|@b=sY_74aif{k-RVsHRaCuGxEp`I zTaWPvi6xAUwx{dK>aCUckPe1zUqSD{_gzTlfQI|?sBii0*Tz5i4&GKj@p9q(_+^iE z)o*uIdtaRzy*E_3Fmq%3o(Mmw-%Yvf-OSYtG50U(0=`ix0G^a#d}&V{@ zl%IEXXQR#UWpEZku*|vALQgOSrP1s0RG_Tv=iRvA#uN( zewy(#M)~;xLLpfPgOR+RuF#MkEeRqv*ktSILRrWd|} z#meJ6__|-0UiYR>YJfFh6~6D>+CE)*$--rF`RnWFV?DYS_^-Ft&(0>s8>`+oi@KaM zX^2Gt#~@gdgIPEPb~qD3^WFP^TG43KD%f-+=pqCY@k4R$A+DT+4Zi?6llNbK7{WaY zjyOGnkstEJ_vnqwMQW^nkj~s2y}?XLm_)=1ad47RSAx#I37kK%2sXRH_5mDtvLHxY zmC6x3=5X`@@Q#GG!P&Z zUj}1GMu36OH<9FT!)w`Ek*}X`dcCcs*rbmj#&OO)tX&xc8^d85*M>e>IFH1!#zANd zFj9IZFinfc!kCZ{m=BYg*dd~~Ju24#YYBW?XmL;%D4mSNq9xJvFg`r+#bk5ibQEL* zusFRTDWkIMo^UC|fXSD8kIj6g6%BHl8hkcaCoLoozj*`r%ifx-x2}}FA9c>bfvt657|;dQ8XS;1&>L`$Q#a3C44}*M;S|-qb*NG#046E-}=+nxq549+JGl|3Sc0z^Da! zTS$jP8CW3I{=wBrHAaZ~{yCHgNB2-^1U?8j?7)*sVWr?zA1=Lk2ycr*c~Vo+psgXa zHn9gn4nXlZ`FbExEYLZb3?e};V0b`P)aFI~B}f59195|9X7J=lA7B?^;ex?$Q#yf# zqQRFW5+no7+Z*%%mhqC5co+)Oh0ru;U;%It#RarYq80@TlShMMKn-vhl;$H0Hm!ZQ zV5mB<;|HVy9J@$(1;hXsc9gGzK@QZ?w zO=t|HNbWP{lZQRjv$GpSNkd2k5Fn5VvJA^1UL;=zRzYAPMqP_x6D2`8TwrPq%Ahhz zEfOe#LTv&NhY(sEP9zEu`v^n?ELlpTs(~nhR0X~#T9A~L9E^+EvYiKED171sl_nkEF?0x*h@3InRE3ZDvj%G)D|Lt;I!~}0h}0X_$?OpXMd9n@iAadx z7oUhgUmyW}Lu4JQ>acs6h`=oZK~#AGBdL#vfCDrRLU&PU^gvAvSTqkkRYvG!$r~8- zp5Ijo#!6wz3h)Fqf+HJ1k2;`g>JMUB7LyL}2m^76r-BC{kDbVl0@PLP(Faz5N4Pq@ z(GY{AcwH6*e04sMEow?QKc9&fI72%NE zIF&A7Qa}$7%o_Ja7r4#g?DkfcI7nH7%oK=2Gy4{B0m>~q4$35$ zIrsYLib)9|8eTgyEcc0ANk314m1x# z7%9$-+DF1C<~T#GkU>Y?hZTNjykj}QO4PkLf=UX`7?YQUMRvXEkc9KGb4o_aE1P_)9=3vi5aZ&uX7c{r8<K$j`+X0xFRGZjwC)e87n=L-(q~?~6P>p8e5*?9#*RQwlkDf$mlSJsg>p|_ zu?XS{c(v$Bw4Xzd&iwYi@VoFrV-s9n<2cJfw=G-W#xaW~vG<;*q0+T_w?PZfv-g(o z`BwiBc=)ORWzUfcG_uCN=smAl5}UlAVNzOsu4kYxK3-sD!0+0M`WFqIYO%m>h694# zDWWGs1GqQ8@+UH4(>J49PZWO1Y#rOD9u)Q{ZQ)>bf$044TZfoW)x2rEmK_j%Gi87G z+h=9p75r;|Xg!?cEqOZbcaX!rI=Eh!Av!i`f;D!fE#5y>I$!7DO~d_nhMF4MhIJS5 zu;Uwx{8x_rbapkykZb~-Lj4IJa^wqMTzvZh?6-f{;FL?~!iRR^qa*oJmXw^+Pp`}l zC1}Y-*ZRud;$95ZycexuHAA?Owomp!tl^5New=iMD2=hBn@r;+W@|qoL&>p{{46aI zuYu{MP=a#B*~|L(p7JC{i@ZRY1)?(|L&}?j5`N-x3)tII;~H9j=SqODE*=8wCknz zg^9VjctY;8e1_wIhu|VQ(d{r{n`pUH9HZJIRZJ*WcFdMSFjq@7l8bd?)UANnP)kR4H`*+Zvf| zr<8eh#omonA**#}(QEHROXgDQe~S+0?)-Kd_#Tuq+4MXrtXe9`^!TgyS)Wf<-rhF6 z5Y;|h6nbWTD0P71p9qrnG}E*@^)T*Ijx>@tgpp^PV}LKX=T7b;l-<_4_zhh z3ltsdUaxtxf_!0*ue0h`LIo9R5J+J#f&+}1Jku^1}wyOn@}FR< zbfJ6;XrYaAjZ?F{b3z+6@O7W(bsCFHYqIAzvkTyAJABY(3{7*Gs6SxrKrA6yfIC8O zkxr^ro~*2kuMAr&@mGNLDrC(1y#6AM;KPsMO*@-T3-DHoy(G%U^)7c(#H`C{Tr}c` zPWfz{9q2aDMC^w?y~{V3xU;IUMK}V`H2EQs{Y>C@XAK&wN`fJY6LFBnr7m8|5$rbX z-Jd8aDH(Rr*TNZg(Vk#&C&TP=G$<>8P8&>NBnKcCAoe5fBA$o<;YDIZ3*}i`v=1qZ zNoreeZ0_PvS{+L_?elfxuxaADhO*eSj{+IroOP4iU{5FWSQ|Fds0Acf36Z0Tvgf2p zjIw&2;9bU-Z1MTbk;B-+G7k~V7?T5f&#r1uBxgZXI8>OX^t{Sp>ZTV`Il7v4H5JJn zrKz?H45@}C@l&7+4vPIl8>9O;^5f+8a9Vz=U^Qn^K4B%K z=)Ly*#bFoSe?edF<|;OFcP|SOon_#H91%Ti4XJhU2Mf7fNfvT&Ygv*ymS+xW@r6PA z7^vFyQk2Mfse14xL6JH7B1hIBCYr&toAacvYYlYOgdI7^Xj(_Kp|i<0#u_Cdub}4Y zpPZ5V7BgX|7=Ag(l)>oID{$~_ehkobc!{&h*w{twDR;^1 z?7I;xz&Sn)Z-KQ>ZIeV#!JKH;xMW)$s8%yQ+LTnJqkAi8S!GDk$B;QRkEuo0QB&x3 zfu{2Fx4_6Jj-DS)@w7aj0?>$z(5fxUzt+NfZUB|UD1fk@xW=FrgM=;z(HiXlcEiuR*hDg3Z-} zXswlS667juz?u3Q&Oy>}xrK>8RwKdeuOlAH6X#WF*YhD}*pmEaT=Xsnsn02PFlQAH z**MX$-fhw3yz#M9zJpdoLvuvi{h8im4t41Jdz#(KCgV$Thd})f*fCe3bD=|PPMIrZ zy8u}`^_;OG8?)xqUgAheJjn(a1rs;ZBU}%BE$7-@$zl6CK$wZAOh6}8l%90yD=6D% z7pupsbza|dJ%+BnhWTG`2b%x`_vM6mda5$PD?5wC8giKhK~%);Ks1k8&T-x7E8Zl= z1taLpOC}awIpwI!Bw^=-S*tJc!MeBnP%Sy5N_mW1vifP1&*F;(#WitM9fR>j-#MOo z&fr|qG9*BaJ(|&b2jmH*i|GaNjq0qQ#~o2;6@DmW?LNd04+FMV?{!!w^GasAOgk>W z>sX5A<&cmDqxrm zP)^lGjNWUq_1D>yCHrUTha4?#*j~YIAhqK-x4Z)Kv`YwckEU4Ea>UG69GbUv z0Bi9K3}p;6T7SJe4biG48_7>ewO(a*j;q&fKeZH55|O>4RFR_8Iftw%#lB=)MN^AkqSXX?1`Q@LxTi=ES{fXh~QZ=NpFnNG3m}RE+MKP^d zX|$lS+VwQ++VM-48H4;TQBUwQtS^uV(5@q(!{=B-XNW)c zq;mkCBT@o+*bT)(Z8_eL{pLLYjoIJB8E%s*JAHua%4S^|mE1C?D_#v@vOM}IMqllS ziy*F^OIO`5ZTu={&0Qj9-Mwr$q6mjK3Al`7IGyBb9361Zd6#xLr#EE@^{Je&W|(X4 zw-R!lGko{eU02bwazdxiMu2I=0n9W>dFkZdKeF<53@&iTrY?Bimv@UX;so`(~Rf|o?t-_gXS~# z^F%=DUx>&D-B(Wzb3U@NuVFhG?G|3oYr1Oe4$K#8ZUL^vUF7nLDW|ioMFcft_d0L) zz_AdJg37)2JjLfjRd0*k}4l*Dla4bX)`mOP;$lRP~)oN2zdvl>PqSzE`cFZ?7=y@#sc;YMN|0<;!t3SOC zk~ft_pvHwU)S_-_>2v$=3F$al5_>{gJSZI>$xLmClfm2Zv!nR+<&pTw#YgvniJ3xJ z3x!wom2!1VcA8Ag`C$$mmJP^HT5;!kLD<=-1NVA*2)}55X@odxIu@1{v0P99pYm>O)*qi-H8ZEAqRVf3s6&W9z={guV zM}0QI>_@--!S@%vBAnR7jxTc z0s|?kBR_70t0&y&1F+le9Q_u4EdQVyt+(%vQvU&zC(E09#hR@2IB;8fJl>s~rhN-3=SSCXN`Cbm)v=jpEvfZ@?r&d`CgYX16)7Ub0t#jr zKtauWF+<7QqI1n7JapeA(>*T4K2W7-Km%%&cDk8|oN*hD{6a%9JHMK5Q|*~F8EU-h z^Fv5n-J6UqAL1}~Z~Emita6I%8mD@?l{*?+wJ5@)Z(^=fzpyMVEf>xsvWlrz=@&^g zNRwi6V2oU{0g**xoYJUMEZl5W7P(~4qkQ-1Tx5DcrvZkVKGo%HlLa)<8PjT_>|eK;_LWsP`d$eY1otexjDEb{`X9NE*=;^G?j{F}}v-)MZ|D zU^Cm|=(8I<$2mmYN&7he@v(`@pC06x)tm6dd6`5HTYxrXU|Gm|b#XD@n46U2@=n4) za3^PLB3Gvi|7mIIb!gyNZ1YoT!I9U>Yjxk7;R?9U$^t#<{>Yh&n-7; zN-`-X_HCFRBidK1F8fcT&Wx__OkKR0l_pqGjlXJ-b4(CGOQ%3-IA5}nVQFVBER{B` z2%FXg)+q*B0Vx*5Z#~QIPd-hYj*b=)24oUh1N?MYJQhAEaTAl0C$f)6E&NQZxb^C& zt#VRr&}{mgp%vI`Me=7`zb#IP4NuW_{1V!+HD__%#lk9^VW&}KG?M4IDM;llI)S85 zDBqj2L$|e4Hxsd2OPfvW(<^^3Md^aZwIkU6IVri;SG+APnQHD`{9=^^A!t#?NKuVv zlpNf0ROE_hY!~SfF~Pq2P(#Ky z+d{3A@0`memVs%9Zfb{RCIWdGW;ch_R~^I^6EL>pg!dPM@F%+SC~1d=ZRbE6LbyJy z$#r(UwxyEge)yxAkA-K;bPtl zyRtB=k8|x%b?qzy2sC%5_!7OK>LBhxm8UOFx9=u!d`fSXo}@4tglZf0{Y?UBQD&=z zM`J%2^~0|n)BZ5I;OH(-SX5hto^5k2EiDbKtsS(sExddatw`Xh5gB^PmYSslVeIWa zJn5?tdsZmft;e`3#@ZPwzL}W+vDNuwG<6dr+voIibbz+ZB%4a%Et+^7qpyuc>fMv> zYr-+>VSRW8+t)^fqBcrKN~A7P89wl^dwj~FbG`kI8J?39m^*{i{*$)512al;;wn=R zTE!McR}yOmW~rJZXHr`bK|l5N68dOcCu1rj(<$UvA8V20&4NDbUx>s#-_7js1S2f& z!H!}EkGHc0r8-l>dVkzM_KtpLGfBr*$;e@i9QApcc|r2B)zqJ8#2(5sbJ$yYfrW<; z*m$AfFP%0ki=aChA(MWv>qk98K5|t{V)w83vd}`S6z!f3lcGxG?KWSdDCF#R*=&(? zW7YdW_S-e*BK2@BuUwz&&36UPA zDwQn`&YwuASi!O9>j02(kz4_edyLn>9_z-GhbVNcti`q97Fa41=x}hrWsz5+Z^$3Q zYp-i$k=%q=~w)CC}HVS&b9f+|_Ba753ltxrs zd9r<^5#s2#NJdhw|8=E!!NP+m(gSxzyIBZZgxwfF3=y97ze(ddR0;sfs#!>P)7DFu zzx>Pyk!kdpva>37=~UYtntj%>O+Hg-qCnEu)860p+Yf^Ktl@7e+n#{|&jI>Tt9%Eyu8V)~vih;a8~OD>sXB|PhRw9B>H+a{}2Cbm+jRbVJE zaK>v-FRZCQq4w<2SG{H%vt@STw$UGMRVA!xjGf<&_fBRRYVO4(K%~p8;l8KLfnUF25lfMRikiP}rzhC|#mDOd<4okNO>I=dg#a@pbiY~xLUSOrupDjVi z&%F^a6ez+KrH$7{&EpP+9g?;!R(2a%8}l=DVWGShLxrC3oVWXWU@sHMbz{VqQCTx; z#rJRS;XUy=m$Bd%H!6rpR#$!MFz+zoqO*udR|o09j!665&1<6!!!ucUY&2~6QwwXH z>vJHJ$2Y94V~;sV?pYyefs@iEn}+=1e57ow-0!Kap&IDDHuTHpwVi7VcKR;KsIIfU zKuxP?p-eq%!xGl2L4d_`d|P@?0e(8Ln(lOPCu_qR1^zW+-Y2z3O8R(G7*x$+tTm%a zX|@;ncro@h8qcYp+*7QcRX4;iYznCd6NZ!~74+7Jx|`6vf7=FSsn|bRze;W?8@9h- zBTq?zu^J+NL>k2ilXz6onKO7kk9UILMIz$r5&aWGbK<=HHGPBNEvNQceWVy*nJB4C zW-m{LDfOs`^IA%4^QnfJEL6j#022ig6N-&irnUK-YQwbi2z*A!$3~jTcXC5N)v@Sn z+0~Ozvt$&qiZPXV=Uf0HX4?-NOB7|$uRATXh=kN;zIV}`3wQr&Q@8+Eq14-;(+Xkb zYa!{s5WhwwXc}%n4~e0>Wh1}LOP|I#^FTwA?MHd_rhv`h7VxO8Q z&{;1ZeA?$=S4!kLNDjb}hfQyWj$@ytqPrbe7N$AfS5Dp?+VSi^H7QV#tYKGpUdV`&NsGJ?>vyx0 zS?5HETtJv1BpBR&qJucat;x!As5ar#FZrh`qy0r1m>koC90Pai=ZX{S}XF{k!))BtHjYVyt zVFk8fb0a%M^EVUnXBm;BrxF1t2o;uj%X1&X4we|1S=VQBs}mEu`Bh)7M0*EQEZadC$OeKN)XBgEM7* zea8#PD~S3XMs-joa6bnKesMMm9jEk5Z_=v+rJzJ_75lhpes zqx)5LXFP zaVm;X`*=~^vH{u#9Er`WdA&TeWff4~%h?<$E6HXE)yko^iw;aZN{V(lL>V*)I(62t zqyLo}C*=1DTxzn2UBI!(T&!s?@rf3FeM7HwvtVm^N5vHL86-mwJ(bGPXx7`QGQzF^ zJwY}{+q^{SPdR%T)=&9E1fi7xeaK|ONht!qq;DdwA}*%eF7v>v(JUAc z|6w-6!a#vn`~L;-YBUBkj4?VGwOK?3n~dVCD6FdUH!L=_g#7p<`B}&r&-7wq1&vGCJjq{NSL~OdzBiR|Uk_G3;jd6D_!Uw0NQ@tFBg=;~?HVqRCBPOc&J$=Lj zUVk_fAu`1FwElAw2QD^{(nGSnw5ue=H*#k#W(J1rNMk-Sq~W|2$B3>*x<$&iV@>I>r1 z;_r66*xiXU57W6e+s)sqm@OtQ`MA%0d&s^Y&mwzZ^-)VfLv1R2NGXyFBg`mi? zlKFxpf01XRa0jT?|3xl^i)0(ODi&3FqH)TV{;fo#w{&5=YsVMhWPW6DND|JRn1GYJXA$%>)?{hpTC>I$=6~vxih5B z(qF{(GF}w@U64k)>}h%0$7b*}u63Fgsqc_Bp=XVuoo!-2WZp{wnjwwDPv%s9NkaNe|Vp(m6!6=;VXJq z2Du}b5ksWFSnpQ(SWIIsxEE>9(;3XuKgAG%bW02NOM1Q>xI$qZ7kRiWb~nCdxuWhK zmDnFwx(~vj5OHp_5sPI1EUD6u{oadsqbxS`o-(1yJ+60%A|mC}%v>NPEPT2abQlj7 zLZ<&AY=44+0;4zlpA=$1L8D`esK9`CM6eoTIb(}bsIsyDp9q3C(+NXvSbe}rp^L)& z?}u?dBZ~ZQr?#Zb969(E#udf``_tn0SIF1#_52XQad@(Lmk@aHn(`YqINH8Zmg-e^ zu*_4IR+%az=BF8JdY-DbQWE@S6D!p&e+;_RWQ*Icx7nsKs!L2cKFkQUWlkVQrz};H z(gZ)Wp8HerB<=N#D4z1%Y_lvM9a#wcQ4IJ(7WF+K7fo;>kkz zT`es={bOG6@9;-7`#-6{FOTt64+MrPvI&t=oLMlQD7DHiHp28key$7$c=0DyI(C95 zuTQ6DI;1q!I4hFekruFsrBOUHE8^LQ_@8)HxPEvcW+-Hqan5Keg z7##~cD;gxeP`VW-$!=^NOHLN{$voGcRF@rs$Ggh{VeUI0E zseD^y`|EE1w!1+4ZHh(537aKtV}tJNyP55g6bnT-U|6jiXTzhnU8gN4N_6lqM2*Q$ zmGicFlFt*h9V2m-j!t*nTY4yOUk}~%8qetpR0>RfY=J=gZeeZO?Tpr?Amx}ZTbidy zl})_&hYebT^D56$fy`pNq!*>b9CzyKCF4vk75CKu+WpUSEVs?7I~|@?aYp6LwOJtb z-S6MK{=~2k{kMW#oegPLJM!z^XK+%{%=3iCt;{7 zckRjwhG@%vlZ|?9rq4(gD)fUcq78t8rwCa6xhb-$&EX9|i`6yudS<{$E_{7u5X`V? z!*uk>RnZFt_S+S?CZ!<2tZ6ZzpjR#t;Z6gMAtC3iw*RGes=$wO%;7!F{!n}RK^;Rm zv;K_#D(l6rUXR)yr@qa%a>)3%R%1Eko>t?E-KEfg z4I|Po;C&p)_Dx}Um4U@phT?aK%k4GFQN%_}CoUeIvO$(JeChTgU2mxMYJLA_Jwo>=f>dpdVK(JKhxwD2@4-Q%1tU9ghMah&Q6Nz2* zctO}w0NFIakEvzM&*_fgNb1;!rH+imLVwJJQ$@^apv%VdNOJ*Xilaqrz;3MBuMrL)=T$@M!L_3HnR3cJ zXKPn{PSd%M`2z4`ZW#-z{0kALKf1@O|H|P5T0Br8sBJ|G#95b5Q1b*}EgHt7zsnWp zSep^Z$3ajEZLzHG@C1xU>V>W?`-9|$PImz0U9;fAamaW*Q4Nx+BQpC)2m`5?ENFzr*uM-4lad3TzHLh7gWPGx2DamNvsqW z7}m6{IjQNh@Tgmt6XzIgqnh5+zHSnFf8Zl$hedhyM*VivQcJ~y5h+QdS_b8Oqd6+Y zyb6nPnZM!a&(ui*BIU9O4TW8IB)<`^%FYWh+{=hh8BG|sc;v|rJZOEST2=O>;kxRn z8XN#NQW@Vt(}f#8wRI<@J|MS~i)_XOHqUp4MDn&|+_eo}SU=|2SCDJ^$UejGoFg{~ zuOmuv`qk4{p-O*EMcglu6BN|+lp*6QD?N4;vX42P-eKrOt;m~tn&)HYH{=6ujiR3> zl{s-wcKIFRSrsQJ$H|)?zFxGuD2;dG4>Hx`d6WG-s0pKt`F#PBalF?EIb6k2f$G62 zFd7G(4At3;C(zU$Txx{R)RVd!j{u)qhs2h3*a`r#|8Yd&TV{`vZEa@$Y*QXoDohJD~y9?MXFt8R) zaLtW8hfxUl%eMwqUT$@%_=E)#5|370`r(XD3EAfTC^Vn#;wh9K9LB>~e=+D7loL^? znzK4$_&w1_r)3h)SD{-5sVGmu&j2dm=O0+}>)_NMGwtyRlJW&D{vm{MGcZ(!%#};T z+fr2H)q4kBRMJ0#PU?g|e<$5Ha3tVv(8i(X^9LDxq1H}1>au*TU957bvSyO@y%?#y zki!Z?_xZA^EI3-{l-YeWbcCxv!ajmthRA4e{0^NzK^9s6Ef2N8ZdF+M^=qi)LI1AQo^^i=Je=APndj%q!5X(!ERc}P^p!Zlk+%S!U0lr7-tr!xqwLH$+TBf^Wg3e9}s@#NUnvOJu{=CKK z@TBU|MO^n%999D|mx5tJrDm0H_M^rSlCAiDfe&*M;UqkaVkbA^`7GA&;}_n4z8n<_ zvglxRLu^$CSdQ+AB}9*g#)6jm;gXV#EH+SE-yk(QD^!OV!T>a#>w?SlaN1E<~Y{PuA(*J1o2va(QSD6Rr4sC~K!?=)``AdOQs1F_U4mKgV_|?R_rVUG?H|$H1zS<?QDzJ!p&~r4Xyc8?S?8yVH)((%?Z=O_+_%d-$1zw);IL(cy zMpsg(BhE{1Y-`5&mYM8UW?N5MWu6Tdg6Kr+BSOy21mAx-3LZUWCP3$!*{Xj6030g# z7c4DRNMq4BwWDhRNKFO_8sUplVG(Zm225IVVz-j@2oY#Uns*u(AOmCo$u&TxL0GL+ z6>CwQ&h?V^pb{?-#}}Ec&b;Q2TnGDmbPpmOpqaP9Gv>GLrLk`%AHp+;`ruFuoGnfx z_D&;uQEc$2OE~jFzY6x_yUA9@51!Uc!h6=45s8+mVG_co05#TLE@fhZs$1YQV%@Vh z`>ktYthuuT_83)V?i&XVcND`y3O5+fg34v2-vV858SeEj#D0%pHcnATfNJSESGQb( zW=+ z8E}z{|6A+MNV@XiHc;*+a?3%@2DpEs8mM6PEa#7_-mcDCqG|(XvJz5Jl+4siMNfbC zvW@*Sy+}CW9^m=cl$IAhhoVt{|L5r`H4|?;e@kEmC8vhVZMqc6Ny6y0=z1)w^}*X| zt$=z#Hc`3(rb#-?b40>SOGf@KCrt-OJ8P>%dB<*Lve=hKb5_2L5S6jUG9^D2g>gJ; z9sZ6rRBOhX7+zR`oMQg1%sz~fDXq>=VY7pSKKn&B?bak*4%QuEX=Pszqnr9*)u2Gm z`#l&mOM0NfMzG*aLNDc9jjh>6$?+9^HLrnn%C~->*2<;c4NqB!ZkcPFn%9gu96Ado z7i`V_w6nNG6U$(kJ}esP!hD7G9j8XRuva6~J|5QR;EcK|^^!1f!RREcVl_uDn`RYa zbFcK410H(|2Rjk}D#QCUNjZwHWhVr=N|vD=u`f(N7LbZp~ zVzU;xRi`$s5z0mM^^{G@M^%s96@tViu$HL;sZS@@?_25dG!QJ(i3~9)dPxajZ|nj5 z7C!sc4@AKJh-jZJS5otYaS|=9p^<6TrsZ7C^o~`@Lts*EM!^bh5~tLVIL-*Yi3Q|2 z%WH-Fs5$pde~HCc%8Y?pMY511qT%oxpO2zd1q2rTX8da0lFO4rI5z1oB~yhi$#Vn| zSJUF7-=dX6ZF5-jK8(J6Z`N<+b7(lX#>%nu7mk9U+T~e zA0QRDV;+ZycNFNGIEDsmC{J`l7R;}Vb=aqcIpM%(N_DXvizBYyENWb|U^+!*SlK7R|VDw(xUJOtrt_pHyC%qa_d>zBtLX z`*&)NUc6I6kN2fX5XtRP?TBxw8i1ntg$WN}j0W(daqUU&y04lEy3i^Rzt=;)V@Q6` zGUZ{AQjdJ%RjpqOv{lj8-S)2Nl5%uLoS;Sjz>)u|x7Q^-tWid7c>It)aNWpLrKDyI z_~h0*Ir`OUhp~#{Vl^T43)E&J ze|JCGFw9lzRB}S*dbMuT( zKP~GLhGf=dVx5%g8Tr~YkVw8W6lXq`%EMLT3i4Ww=+J}!%G(HP8y)vNu1JC9eOFzt zC#Z=W0Vc`2J?K+83e5}^;PWZGBWP|56O}0AfUMndmzb;@j;FI?Ax|F8VKZeYL;UU_ zKnJ0K$B>^icTz>6arCYgLdtD&8lGR&Ryy}yz8KytHq`AEKrI5fN=Lk8r?VAuetTFD^1zPf{_{kNWOh8y>v>WTy zW%#)4%x=J;tET9Ii1(F2`)*k|J2Tv}Bgn$PD13S!jwExn`+r@^3`T*(l9rntAE%7;3PDAQ);HVMQv< zvC~UEmZdYh&#k`o%g7deo3e>xrt}u=Pe4CE7%Mf(umZ?z&NV&V2HoeAg*0 zGCxB1w@!hH=CHJ<*fVK@%qphHddJQSvD3XdZCz-mtN9Kikww+NKdg$EsIR}DeqQdj za894;n8LbO%Uvv>U&lWoxBm4pYnY3QuVsbdiLUI1ghFW|Qr#uUF=h=?jKYyL3$GOBmR#j!~ z6!#F#7hxW^?UT3NG4dC}zGypR*YJiSM%%4r&?P&$XqesQg@N2&&YFwn73P^M^IQO1 zn;4IJPhIGjof`G0_EGG|!GKn1X_sraRU)~+5DUW8OmL9-sD;=r7ksv69a2Qvs@Ky| zhg%fl`~z-E`#HA-5E881WV;=rA>^njfa|m|1$>r-=}gEKvK1%PEUPhzZxFrI0-JQJ zct^RV9Tio{63f~z-mr^Jla(1K%YbCz5A7>8yfWF+=>SNMx5aA~w$9gMULGIx4NGYp zm4-Vd;A3h>S9B~Y6WLShIPUK4v^V7J2)|)o3)10O`t-rV3`8R3uBN_~3#Ny1oYYbOa><~t@%VH<8fx-J z8%^EFQCL~O;l(I4j~L<1HE1zC(!Qu~zpVQgf~>Rk1*E=7Hu%oIsQ6QEN4uu63P5YX z_YM%NrnA}wj1>fe^pkK4-@ZN=Bu5O@d;ivmygjH$0Z+Tok=SNPupg8^l5&coJ=Bi- zr>*8zL5?Tnopj1{$%%J&L_lHf5|4<>(Jja%TdapklZzo z$kZlj>~(JDoDPR5=4|QQXxp||WN>)b`B!kA+{_kM_B+p%R`qkMb8-{S_p4O!Se|Td zH)ucggE&mYb9!iou4#K=SmLm+V)}=b7kO`VOi6Z%v(UfC{V1;w6>VMT+{{?Yp@8|V z!C-AtpjO-B(I60VwtiH(SB|}<(J~)C(fd}8rH4j`k#i}dVgs@5D+yVVYI_VFX!7nb zlM{?Elhc0OrR50U9EEO+b!RZ`J~lO*QxxJx!;+!O3Y_FNr^m4z(c>dU-lTIbJ!C%9 z)u^ma$N2(cPTb8Yq!}kOZm&4UM{8`Qnfr4xIIgqG(HrS!c6Cax63;|o~4lJ{Db~RfjqcXk0p?|pd+rQr{g@g#K5)%3zp6yExKYZD~eWY zcEF*`S?ScQ6#)L1V5$yIP|`J%@)Lc}~p{(XvcQAE&}B<43bGunlL9AC}C#b#Xank+IpSvH(tQ zP-3H&gJIx>UNVA3W+=?BU&ozoku}o=IV70^%5T*S*stW&)Fmv5zeIS=K(?@rV-4X` zkuQ^ZCYw?v#K)nIlAsk^7?!|vqmBcs=Q!=$_%n^?3USMTxE`+SdvttC-hBaPoLAwH5@HKAb5T*8I?!DQ|(ugVtOTCccYY0)EMvojfNqJ^Ef z8&IWQrIw6)lLXEN$$&}3X3vo1yeNYP#RlwDko6v`66BG(W`P0Xf?SyS?3uqtF{?JE zMd8+=rU6fFCf-peUPP*8*x@#Eg!Wp?=WHq9ym7KesjsV@T-He?GY%Qf*NtwCxUpdu z8;w;SJ8@kaMJQ6kcv>QI@v0QoRKSEFTWjT(C0~tgg3{fdVhLU*CAPW5C83-({M2>ld1(umT|`hRF}kt5^8IP1IGd+kkIzXv{Z$SH}{#^z%@Y6Ls=uaOB-dFAM`yV zpZ`aF;6Ligs5r4Em3)szIAaID+9pdmCYZ8ET}^9-t+se7Y7t<~pq6Wq=-W%QgQB@wHc=Z>8!R7VX`7vlb0h1uPNYViA_atrbFR9;oxf#Y=p<|ef zi{C0>RS;LSnQ&4OkmooA02CDR!FfUFr;3L(n zM(;A+g50j?iaS7I3ZDPwhsORx?QYk51qZ~Z-RfqJZtQT!1XP~`V#%144XaFX7y_Y# zWr1ax*(vu6(hRufv3~Ihe4wbI@3yz5d!cL}erg7PeY4oz zi36v;6=wHW<~|c z`M~e$J}DQ66`qG2EPcz+ux=$!=c4_w{S)|v`Lu* zoY9(^GqKCf;e2H<;zILPKB!OWVfw>=iWE`*~GCaRl>WtS`|(wqM4Z6R$jtQ zA4}m(27QYqi8c9MR&RvMZ$Xx4eNxPMi5f0X!fbKej_Y(gE8ZtLSQUgWq%DG)`00yW zt*@06tFHuj|M8jp6HT&=7}n`+F?vgg#On;W(G@N0f#fL5&ZO|xds(^HKYktjJdrMp z<}RVjkTV2r8YE=A!#meq;wzCY3({+$H<|~0r^9_*IxlH~;c(2mA&s6*mJ&n3s>CV< za}lBVB!rxe8NG9E&JJM_q5410+A2!gMaSQ8nwhsL za1LF~ag=T%vWlw?^)QX@hE&{Nmzn#IyMw3UUeeMGkbr=s?gJSt3f7(M|4Ib&4)|(= z4m<;5t_nv3g(9j5X*gnpWhs0mJ&PZy{EM^inYY5Y0&)s;+O_Ei3%bfifgvhkoeVB+ zp@|*?(YpR*1xy}5AQu(<6~)2IBOqk@ z@+9pcxB@`$wIy7B`qd#ex$z&T{v(Pk6-*1fSO8!Vp%^U0$^~Hk4Ur&AXeH!*kz{R0 zw$tBL5VyGD`APG2!=BQM_OXX2S!?KanHM@|hqM6PSZJM7At}f$;Djx!{Z4zm^lOMf2AVtdV-| zjt`}MoOxn)Xd*lqc}%Q_qJN@!6q2)j{S32A_lRk4UVC8X)XZEIh_)~L&Nesr@-^tU z9F5R@hytmmcUJR`usqH?bEHwhptW*)qwmah#k0zv512ra?Dt>76>$05;d|;5zdv{j z>{;)@w&_0_Xy>efWC;mr5*QqLQvdhvUqpoKj@Q1dFzY+Ao${Z;+{=4WZcFB}HEuO- zSK%njrM^%fkBhb5d)(m%i0I7pH162M;Ejj!@vrdjsXn<4+=Ik8043MlD9Q998$z~>H+GEqVL;gGx z!JVQ4?#nLGgJ4s+;069dxXUQUF>4+FekWb)c!oqG{58P6e+CDHZ#e37h&t>6z`kGZGfQphP`#LEnH69v1kr)A)8${9S{rnCyR+ z=0A18QjHWt4<#$}0879>i%!#|tq8V2ik8XRV(6J{c^Pd`RRTqK?icSVf`lw>lHbNl zFA`=O{F)wlM+!gBzKm{(cB)&O2Ul!Nj2Qr@a&GKVo^f(6C1wr|BHB%qocgmOeiRoE ztkCGG`}j!5NsyjP2@h(FWb&OLqaJ?A{w z>Znh}qUVg(+tz6%wiucV8hQ9lw_6H{5K8vE#Jvxsvun?(mz@0`mSI?)q@(u9tj<5c z&d2dZlpl*BwQB!7=O4g#a;8d)a(`XGg7l5slRojg_=Rr04#GNH*o&w!yp?Sh5W`TK z>5Y^+w)`_^U{dY+NkzX?XlH-xkF}!4l?O(Lo=B_5dI3czjNE(q2~;RHqWrE@K}NH( zu0Il*ybz2DF+hBn^Gd3hv$dUlL#ZbpMf@R5-HDr|2PAYx%wSHIGM;^2+gW8y5qpA5X^{`zqz6`3Hq7rYA&Qc?7CF8(X`PlD zy~Bd}+~gGQDENQDXOIy0B2r;*_{yvpNr@dr5+wUD+_z{r7FH~0ie{>*-S(Rg&Cr)U!2r4$~LgQ7s#`stD-mThO#IUHa1C}T5ygtC_*}#Jm1V%7i)=& zFBe$ze)<7MH?N_z^JGI|tLVA@E2Fa2{pEeRqA(7bA3`%CTX(y=#qxS11)0B^Wwd2= zP)v-TzBFG;u*XKer`VOYa^Rk$Yq1qH5f*ZRY1Lo^)UD8ixo==!F<|fT+{F9qD&a5P z6dmHoP$~yh_pvV{_NdU^UiPD2@fqa$!w#fxw%zCinN_YAn-f7buqlH>^zF zkG2<-y_eu(d_Xl_qp(+KK%xoG{yc(KfQTfEi)t12@WyW3bQyBYR#ud|wEZC>Vtwzm z=4ScoyUrd;IV_c*lUzY$oji@)PF1#%R;h472j;~~m#CbivIDv0=S)7RVIWooqLNvj zrdTi~w@uGHOO{THWJrlb&A5!C_L-AR#eb%C{3RCb#=C)Vz#)kE!m-cJ2P^W{pDIAm z!Cy|(b#fZ%FCfQr#JLV)W{t=$=7l0xbMs$E2}2+C2P{R$3I7Y@J=a3tg^z!NkIELB&kmHlYQ%P1c;9)-qRgtTyL&SM2x0 zon_~T%C2aJ38C{U+wN&k(b2BHc83y;3-_0Eh&MWdgnW|xBE`Fo!6VAiZq*`%tPk(H z@wHD+XunALrk~U-#X0CFdhP%9092qEsZHx896%T^RZ6=+_$@nFf9oZ?f(zlC}2 zW-MakwDAzCRQP0m{VR|BNzt~JXE^ib*wcAQtkL0LqKVO|-bk~7w#=px`}HYN3ukP` z7f0A5?v^%qYx-OR6m*MCaW{POJrI5f{?cgGjZpayN$<|@CRF2VK>2zs=nI%#?G5ES zr@kVNfFxB!NHh#v;w5qK32`xV>{SA8Ejep&R3Rf8{g)D6VLxBacJ4`)J4Y` zZfz0zUiKPY6cZ$iu?n`tvy#$xS^2?VWV49X9ux*}ajk)9c0>aTOjad?qaK;6dB*u8 zoE$5r7R=fQ|MRh4?^_ar$!WVj1S**~Ziz zO7<1m9*Le}Ka4&1yu+e;I)f#h`-OHA0 zfqsc3;ipL@k1>a-DRFNzMLNHSe9Q7vq3Ocogzhk{dP-D)K{L-d0t#iK?%bjMj_>Xh z(;myX#&5gYY41u=t^9hw@%00ZlFJ-32k@Qq-_`x+^>)xDDo$sv|uZxP<+}*+5T2 zjmiu5wbd0HkMl!Z56wsJJ=~As`4!(VxH2BxGtcH0;f7^2HsUxGSDjvtecO1I&pn7% z;0}ripqtoCA?285qCfN(cMOnfP5<#rHAk&RYm1m=sAl3oo@Pv#P0yW^b^5>(P;DbD zOFL6@!s8It*6PBC$)yR3+19b*qep)RJ>og%I@8B~Da8}Vu|OC+xTO~$6V*Rd9Bowp z*m!5T%?}msn#n%8HCS-J2vM!j)pKVhQg}h4Y0=V+WfPXOo%_t?eD>|#((Y}IfRXo> zoi!WtiU~Fv-9Xjn;US6DTXDpYv&ywe*JVcY>kOok++9)JZ@7ucxAA0!BowB{Y)KUTXH|m zye)<}DQUD$A0utR{OUSp4a;!K4UawKc8Bj)B}&wRdjf!Rt7Z#AYb=n4&NgqFa7l(Z zKo+epR<2rH=AR071t<;4z4%LX2c!L)^rJ5KZiCRfkX6l&{hoJMYqATl6EYkx>`&@WY2Yw45gLZIR>OXfb>|qm*Sr54e57FHDf6Nc6 zDv^+F)cC4fJC#3aCa2=YB0?RP771S`uUvzQdso?Kpj;h3)(sTC&ZI7srzVfYF@j8+ zHS-}p5%b}lW2wsi!&wtqFhN-B6LLQIc3{jxM5w}*#PI5K>oZqme6KaeO(6T+&RqPmwq3DfMhe|HDldGJ`_XBtsj&I;AOYF5IAG*Ea{d0ekJcBTq*h zBaXV=UjN#;7a-xzjsLJIypBre|BVHk9B0`ymW4u7USfUPyl zCHN@d-L6OR|0%&f;$#aJ-HP*f67qperq1aW>Ayrq8t`{l@k!WDkz#zfN$Fw1IL+K9O^mW=&L! z2a84>7mNJY*8aoFp+@{#xYgrJ=QQWG(M0*NvlOME_HMNZ+pSczvo8Mm|pOQa~Y!`mE(r3aDTO-gia`VWE-|?dd(|R)+xp&GiQzg|` z%)^tvrkNlk9Yr|$OH|`$i`?d{3)DGP3H0H_K3hh6)_uCW>1 zHB`m9`gCMc^A_}xIq<0!7+cI_)xg1x!-&R%6s2PDt-QTNEzE@eZnI0gWG*V&8*6d1 z*(RA(ou_a#-M{O-gQ6ShW@rCU(>{DM_1;sAtK9Y|DyXman=g->EXl;@QH7jDmFiUo zW~EA-;bp6>@m0r^DfguK+E+<#o>NlD{%46w9Y6gAUJ?SNA<9^9 zIaX`_&3KgJ^+%e0;yN_nxpY}P^wnE8E&=U?(V&8z1Yw4sgd1n|+r^79abZ;bV%>>l zq=y{Qs#FVy(dPC%DUG!32)T~PB}FsTDW%aG&gO!#SIgv4q36d>+y#oN_W5%d?NQsEHXq1{? zpyTJ%+MpjB1=0TdG{UpK2A>T5{xrow|9XHtK} zyWd|Pt){X5evYW>6Hu;Ck=x;v_)|~mWZp*|rUS{x4lLWA!f{hbtGH* z@^-kV=-g(kJD<89!esJ&37@)9k{hRBZi1`6_nX(BBeO}dA}041bKc!d_%k=}BG4iu z+`u#|U!JktR!foGVeK6>Su5gWQ0f|E8G0&K6!?YdIPUgzy|3&MSm2_^H=0kDyf&X~ zqT|weC?0~6YUt(gt@fW^flVzGw^C_~j*1HPVnIeHEuG7sU~)JMGPhY!c}iIZ>rcxhKFJs>f4_B~qiAS}~BpoU>v z?wswvVG&y}>@vCN=M$ZjO;r!i$o{cpazE1f`j$hVUYE)E#Ilgxy(`LN>_CYp+50d> zNGjn0BE?t{*d6;A>!krgpqJJqEcL`@>?xz1r=!7d&lWS6dQA3mLRXpu8QC%Qf@Jc* zXf~P>KR6r9XxL#MuDrnUk}jAe$^Qq*gt0KGs@r`5+JIi*e$lU#sHZ8FYXUXtrB?R- z;phc*Wjh)xM0pyE5gU4{39k5@*-o3no6wJba98;AV+ub~69J>dunrfplEg26*or^& z%!fGUN6g5yYQ#=Na+_GV=eC7&GrXFqP6Rm5r~cy2b%>=^Shl-t!}qRIVl?TKm&~Ng zmI{|j<_6C5%e}fZUd_wX}rw&WHamC6}|P`=ok`xBLSAZ-~XxU^{b(X&G{v5UTS5Z?yIbY>uc7B;xhMk zssyvAW@D9guDdFW+N@!_298!m z`R0y{-VV!HO5TbIpveAW+Vy36_*>36$JYuQsUJpfj$hH+gkzbS8Mr4os1i0Qp7HEc zCl@|!KXq4RpKahA?3g+$%f>2__@_PNGJCMQOzC`=>296TwiK%3&S`C_IfcjVkvPA$Ld7wL$H6#F;c}rAmold|C-rI)=4`C281WY21>f zdKB`xc>VkUw5siw`lX8W!xfI)`rEPAyjvPC4hI)BvU%soR~^H%w|4M^Tbc-rQ>b*;?Fs88Dn&X0t6xr1MdD*9j85Ys(*p|*;vt}% zP5SJu-dmD7gQ8rz`C&PYK;*4f)=yi=T5>CI4<;A&X;a?)%{*!;mH*y?wwPjC6(jPdbMP-%xH!VhcBVG3Fi+bWMQ(a_-U0BK7;f7SCZ(h+dZClO6(`MmH8-P zz`4GoT$vB2Gr22#gTec86}s9i8n`6|ZW-E9F3R)qx0ft~_=EUucKuz4&Q}xOE54DJ zq@U}q{_28g_8l~>ZeQ41t)~xMlEBA=qO>Dx z4QaY+^#E5O@jjzpZ%18^F}fMfzgPWWoRgL~{emzU{wkkQ)=-NS;NHi%aR)O9;5T{w&>sS|PV+`2SUwYChoSG*|*>O~nw}hTR1G?C%?|4=o#dZ)0wN*>$KGv!} z78yS(W=gT(AmPQmoL#MZU>9etE1Jb2y^txgQla_audDm<5*H|%B@zU2J~vP9>E6}IfSlyY!49_3P)%JpHY37rh}T28(s z#58Dc?o$Wmobs=^K8CyIpAT*Za!L)%sI}=$!<-Daw==#=Gw3)JNuQR??r40Q#gi3H zc9s|I_;(VX5T~vqg9xvJ7VniMjrGt<0gk?q9i?ZFGsHS2!d=El+~Z zB^bYN-P65I_b#yKP;M{iLpGD*;ER`-M1~tnwB!%UiBbnVhUk-qryX{=o>mHfU8|Ej z4)Z`m6QacL{2%})Os~$W!S`>)-eRpZl1v|0bPzMmPees2*PdTCt9 z(aK5w3iI)K&oAC$L}_dRmdW3xFnSQ^@wYC`T(CV*&RVJL@YR6gbl=tnESMLHeC@K| zudZnt9maI@OY1Mu^04*tG_6mP+%E>Axmf538OtjC$r}npK&hibr&BO_txCD$lw|s@ z3W$+zbLp-5@O*7&0`-YXa~%Y{X7+`rx{1uFYE~$w?rvPQcxrdqR^H-F2bm@t`AhaKwC&Vw>sVhu zmFD1J4+(zt3S-6B%Q(OYzH1@m@a$uI9Dht&z;fog2slxtytP?b1~EYhmEd8vnaZ`p z6})s}(xYjQ$Xse_?fN2&iuRIF=88}6%4V8fhGfJ~->W?42?Ns83QG5TQ45E5lpZN7 zRL)9%@qH2mMCc2^ZjmSuzf|h%w|LJPAICZQGnOKoMx=#nFNCh<&+Uk7@YI&YbAo30 zT@$xWhtY9H^w9Ow72{fKSw(@4U$&<@l81L<#CS#!odqg2$ug3cPg2wr8&v&uGsKcm zPueINJHGMG#p@Mk>2U9sv0%Uy@5<}G?kRFR)$6BnkVO>rUp#*Rbiw%c?taaJzC;i1 zChc-XY?u1auASUmEH$M3oWPOHBr-8{_ghi=ZTgxb+H?@vtHyo#Y1>8D#x; z+~6J$EiAyGxHa;{;S@g;FSTNcHb93u|{^Nf}%gA#jj;uPcx}N4ZG{r#}56mq%EVXo|CHcGGEIIaf7adEdqu<>m+}H`OsD46F z>N~;T{%TH(lf1;;CFZi?DpauJQ&xM4a;>)OOXhaph%7yPc#E@vtz7V#h-)s`^g;XW zdCh{qL^sp2rCH2;Ze}FjNdKseiLKLaunb`NRy9o0p)>&-W1z^pZ>w$XspYOKF{<0r zuc-0TUw4bW5JUC;(UMr|z2Rq7-kaZde&U#VMlYXyu$|~1zErTAK^7r3c9yII9jZMx z^-+|H0$K?+h6}Pe^03(TK4yx`2RwdvS@s_(`-!kLNJU6We2iLI_sHWgDd3ZjOgjJc zgri6)?s@EHFr}qt-9gCB*U1ujzDs6Jac&{*i1;su`g8+D;*`U_^KToU^LmvW)v#wZ zG=c&MnG8Jsx`1rpsM3|G)w%ReCv8j@r50KQKk%N^(LPfEPre9N(o8T3egmG z8(+1W!^Slpf>JCbPi0Yb*gG!=r#o0c<#21VelUK5fEL#aP9nIgqf8h1n{yxIsE{ziVpp*r_a=RR@Ft8(>+Vv6}g!XfW z#zPTS@xE26Ri>|*#Nv;lHIx7^7oXgo)DQJs&u|%0gi-QmSyeFv>TEiXkqu()+KKCo zzeft~)m~GIg@v=_V{7VvXd#e0$;SQ{`Mcxuy1Ipmsx7qw9hh8xmj}|+^kw35BySr; zW^(WKzu;m}YTFKtU$B1BZN^)&)1tSdQF4`PFm=UR`OVp1h!pI3aGQ5QqwHa+Nm%D& z$0L>1H=`yDnD@V&tb6mx2IaWFPQDkqFk9$0SOv^bH!H?_`J>g+#!4TK=Yo<=2Cj3{ zyDOP`+r}-_XYgvPmutY^T8wJ##zCx+D6TjF&vEK0@{1rcN!2;HwVJ zC$oe&9{EY9@cwG~&jH(o^DL|Noh4lx&lgW?Xnp?>;-ue69CFWH#83Q8{FLnzu^caz z|NP# z3DL+6v8g(Fs*{QGi9=HM*j93Dlj!tCGsYs;z`{0#O;aJx63(dk;#@S-LHnx3MVZyb z(31!?6eEh#ZbVk+#(_0YAl+sALgUnZYQePe?`2a-9jPAFy6_{OYll+?_4`_H6Xo8T z{~$zHPfXs6ByHh!o(tpn(AcHhK2kxiApX;S-q$2nN)5q^1^ZvHodi_FVKFhz{`+w) z-)LQ2S|q;hv@;N34m4c}z)9$-Ow=iTwEWE1*Y>AW)B=3!qD)Y+*;1ZcSRrBv^no(b&xG`f&-3l3-fP)Q-Pv0^{td z`3Jo;w_O1Pc{@OychsMn~yiow|Uye&R;Gv4`$)Hh#uSCw=zDKWHuyCqYn za!&rzQAK}juS5uR&}*NwId!|)9ZaH~zjQlOt%m)-uu=a@>#Yb3jBy9J}41Hm;msB|Ivq%0)*K{6N3H{r9QKmEU}m00iL(~ALs~mFT@}8 zg&WWakq-VNm#j3CC&)bV|DFE}vAgenp3f!c zm*|u(yduO-ppK$4*@T(SN<5)5?_MLBLk;Zo)fE@2>!;Dl^84=v-{L+Q#xlhGjuTXV zX+3FwR(Dq0UW+yp;V!%(JWoMzHAU)3keBv&JEY$Am_`XVETCo%RIbrhZMnAu#u^L> z@W7-w?6`yz5sA9`LPeM+ewOl@Uu0$A)1i}QZJ*mpwQMqBO>6;2bZO$2Xr`HE{Jt!A z$WQc*B)eIRg($?jK7}>;>33DF7Hy4p$TM$a3~@2k*##ctJ;Pm+qQFt-@AB#6szzh) z0$=Ms6D)p?pjMX!<9B1<$Ik?`oN^rsoN5O+1*7l{DDl&(Vb|*Az#Ft}KeDyWXBD6r zCbg|cci3OhTFRq64v723hZE7k?%uEDW&_}t-v(W(L9p;de$X4y-1#8|6D)uB5$xDk zQhsEK_I09#Eh87os9|bZs-ok4+WCPP&9ZID0k|retA?5wCZa|Z`6K@}L5r(STFX8{ zq`z&!EBj60&>&7{4mfFj(?;0a{spt0UC1HQvrYM!!8Ne`qTuqkZG$4u*PabeehwB)(uV)oQ+n*9Q;$=$b<*}X+o%l%Ja3r>*)J|P zj0FKZfgzWr;$xj!(^gln8g)CYhFybhQ!dllZSkp1B0-uXdZ7FiOXTShd<~`182>kYh*+@8vKwhDNzb zY}&rZF5g1?*>#xMg@y^FfWD->xuwW;P%cacJ;;Sb{<4(41<~;F`Aam5yRHq=f??05 zr%hT?hg=#=xD7Me{_EQNDN<8KLa}m4<>+-jDNeWwZ}8rcLE$^lduSZIxZ#v-MPD}E zsJP*kFGYY4?3LflRa!cOeKgB?NX$+f%u|lys&~8sebC-|(D0Wi$h%6*?Jp7jEDLun zKH~Uds*0bf*nTZUbnQb`Gx61k#o>e1u{37P85RX-Z~k~A@F_54c-cFay^7?R;tCMa zL=qKwo`o;_oEFz4>09+HNE<}@OzTOR*+P$6-FeGA%s+y8HRADRzrgf7U(GUa&1?+& z@Q@_cizg{VT;kmUNm6qpl%v*@A823=uuM*3W=E|clE^fThE^tB)rKLB%>Pu0sD7MM zkv;yd=Yugp2d{l}%2F$JZ**BP#SBKSzwlH{OL-^w5|wl}j5K|i|7VI)2RjEl7mdq_ zi?B^jnvommCWssI4S}?#mXnNic=KPu>y*fLj6occfJ==@(nsI!v7;|g;hfIGOe)8r zx!3liPmI-WErr!G^W4eWQoa_kg&K_Oi^S?BB^|kWnFGaVxY>93lx?oAyUOVM-pw_Tk6dSFFCyKf3E(LAMV)ug(*LU zt-XF)Vu3coUHyiJbHrElIX4}3)>|X41(yb6hME9wA(^F^+RSgqG+M#G&I71WHtmh4 z3UcBhLe#N|+Z=}Z^o5dN9cN|PHMrAtSzc=_zB$Kg3LWQfDJ$bGYqOflclfrMgQgxig(<%fW#VW{^>XRo3aAP-g9}Kti z)`;5EV%5&agF^S8i74GmX1Srl=g$EXUwm7&h~4}~8%JXQLW2k@e&g=hmutn*FE{^# zHV#etpmC%cEK*?D>!GxJFS1~Kb=hilsj&E&QG%dSWNyWqf)1TKL3yaYn*t^WmoAU( z>i?`-QU`+{bz=St6>>Vqi!Z7{P_Pb~7)FC`WRHeRoy_Qy(I2Bjyg);zO&lV?F6A## zg8iT`*DJ;Ky`LSaSadOpq7DUXS?Mh{{zbinK-_#7C+vP2`k9x)LbL!f#8t!fU4BK5 zkHd3;oX(Q}FA*xJ;Zn3?A?my>Z&1Zt9c zO%mOb(1?Z+Pf>au{Rfq zLn9i0j^6ucO&@%7jJ(YQX-(qy0#nG%y}t+uIL+9_G-$zULM$8frc*SC7#~_bKL@N3 za7W!5RF^->`t#<)xxYlN<9C&8qrfjl4YRJiZb5F4`0=M}WZipfH0!1x$H?|vq639* zN5dc|50?_~{NK^sutJ~PjUR7nJ>NI@nAt$ei4SfVa%iW25pqk9t6!^Q55OOOoLQOu zUIXwB`q6gS4T}Cv3V?QQ1RNZo2OwHYMUrs%Um~wxXl3F*XComtDb#AhJ2e#&8UQMQ zY@p!hXgJ~0?HnKgAdN^g?0l9f&yNq!!m46gne(>lp=D(D+fsh3fc&2CSQH?AI2aZc zgvA{IJl$&mW}O3I8dkJwmc=4lguLguhUAdR>sI*@PTVxcM=1#d*b01vtcOo07Q zesMwn-iAaUeY;;5<@ZUKoepY1sR0WM0)mJR)GiR! zx?yp@&16Poz26K9I;YEbs)8Y@Z;P`7_OQVC#&Z+_1p{Vcf&o5uwC-VXxOy}UiQccb zhfbs$zKsvl1B(SF0U``*m<{+#G#Au@h9d!z0XY99ia9s~UW)f7bP|0Gy+K4} z0>fr&HDu|)uxLO=aDX8H8wIRR9Dvb)sszXgM*@E)B-)?59bl)1U2IcG!>O~>>Za5h zW_R5h&~6<7X&cacfSdsa2_S+S@PfT_0Q0JHHza@s@YH!Tk_uEJnhh&HR)eIY-Tpzv z0#F-qfb65(8c=TkcnXLHFxBEOky-~ZPnLiluSdGSfHh8OK?q}Y{|6M@4WPz9!2#9- z{QMIRV9o&nkRo&tN~>WC2SD3xL=B1$;c;pWDOwHK1^z4(7yuiK`ZpKWfWrTqk@}a& z?;IDrV)DJU!36*lL_i<79Rz1<)n%!1O$QXC0F4Hu2nUQ0BA_h(VLLh2=Q9KcRtD(p z^G3o*5Ec~>jN4C9gG{SI1~qP`qoe^q03rS_t^c71BTT`O1Ad$7z%#U03y?5_M0 z3$Xa22^=6W{{Ozp4U2m3y?)c-7^Jx}rdWG`#0jMT3p;lJeDDBkvSAI`b^-w6Fgz0O z^AD|t$Nzq}?yV+M2c|7UTVdEnK*P=&0p{U$y#P{o;P%e<#Lodlu!J41tFg)*D(s<;-XtS`XBoMdf5T&1;l#=w%-r?ssrsnN&tG&aS6w1 z9LNF))f)qKlw9}!l}`d2xPKjZ|4$3v8~1^pp5PZkaxX1)twKDK|NrLCRL!2Y3!6j5 zR4&X#_b!c4H1czeX15@=J3sm8u3nNH z@xfsrD3N@>TMNpkwFa@hwmH@+8%!NM ztkdLTS91Y_DsrWp^2yB2Gir86&Kx2Jmikh6t3IpQ00kpD^cO}#7j3?I{MLs#;8csO zD>CCl-$>{;F@mBllB}}$?EQSR%|PtjdHKcd%CerB+Qum!28-V#zuBXv;M_FB!1>30 ze`jiji1ov2JYv}VoqdfTT3*v8fH+#7nle%=w3EQ82q|eqjQwvXmjW>0^)Qt+;$ByJK+qX#B!}aJPTKyi1TK z1~`mRHU~2qhR#}4OPZ4jN@aSaW*>N<4)vc|VByPA*RpHJ$|?k)4ToehB*vDSnd!vC zx~phaw*#WAte+GP)Mz8NwTwJ`r8DX;I?B|Jxoy{}!oflu?>uS{%HwgWs>O76_l2o) zWN4WDf@`y56N9zlX# z7zhu5pkHfn&pr5UK8Zb0u#uG2Q=s_u2gBotv=Twn7W4BcpTg9OfxbHCTU3PnCqt*KG*1FZ2RKjuVd1)wh0!d z0h2%zkn()qv3#CXd9|59b&X1wa$jN>WfaOtC9SD1DWe^oJ+h1aK;DqQBS z#ExXvk@MUdP=KZ}O9|0S16Jg@wWycYrqvWVlAQYTh_as&n%1joH*&VP_WYzOOAd58 zm%5&&^B5{2sJAgv&h|UAXJOi~8K~v)O0uk)$8Lg6Si+M=rT+HcwS13zQi z{*qI?wJmk8!rtB?z~R(x4;k0s--}ADua32Wn;5J=a9r<|zcj~r7sF`xcsAb+`sv>&5 zjr9$i?lc{SRH{LN@U@GTS)n7pf2FFx3wbk@dykI!)OnIJ#Q~83)$x!U32m|T_8)mi zOxxHv-xm^cUWY9cAyq1G3nF^J@n#wY2grgFERIF&9Ljx zK_ynf~h`0Z$!Lp_Ir(VKc5 z3Wo>dK;AH8hbmZjgSUS3G#oOS$ne!#uq&;taF20_Sf1QM{^Z@E&?Y^{Y1<<6!^F!M zDw9>v%cj_SEeYwvJi3&s3$nwJN;Bj)@_xwIXikZ!6_1qI%}L>{u? z{0>*3mo?T`ueQxuE4f7FY3#e-5jW-Q9%;lGDGPZ$m#1T4gAP5W+E#VTG7SjcWve(U zS+~Z$;}Ky6TfYMVMfBm}kxl+haMul1ZD_@z4#T_x&AQ#J5DnFthp(d*DfmuBR_x5| zu*K>edrSCMP@0e(l(r={hQfAUX6>onu$i^ZuMGx?ZDeHt(#kVkGe@9geXX^weR6YV zdVRfD2+uP;uVXaYw=2a%M%9;Q&DfAuGL>crg#3+s9w*+%M;uNzhlBv$u9&FVYzRZ3w@>1 zM{__sJ7!MIMHa1nBZb}1;@FoNCvPVf_aAYKD(J-**aLg4Kw(m@6$k$d`;3P#)m{&?1Lpx^I-nMk zjW3mj?R@JbfA(NjVQ5RXQRX)#ZhZ|y_$E_(^)lc7L<7GE=kYHQ)Judeg%a-D^j&KW z>mA0XMx7%T0;C*hF78@L{EEUW2g#zT@}30AK)O8%Cj@%m@mgV9@YB2#=;`gApMq>D zgoosQKp4gO|H7#MzZ(V6j^h7?QM#5^9tHo4qW*oqfGQAZO=1gv2G}H;*bf-WMf10dg2fheP(Dt`UT{=r55xbOe08MNFj(z~H#gc4vULsIMyg zC~(T=kMsTpy|#spTiZ#=(9}#fRdjUp#jxI%pbQJ|3iNiBV=yMDi;_oVIOvgVkyHvF zVD8tjQK_uo6ih2rRpk<)Zd84V{qLRzk1qy2Uu4VKaqo|b{U9wr#pv7jhfsK!eP^}r ze6|8WUh%2!arF^Nc#ZQKC??Gcr_=t_4rh>Ao8RfM8LH9++>^w|mLOPo^J#5@N_E@A z!B6i4nD*BND(^vxh;^&Yr zoRw?B);qD=lkUgXYg9^!iH8;!dLI!41R?*nk`nYL5QuQD`VQ;Gg@eTl3b?F4u+Zw8 z7E*OwUA>A($~B+k7uPlV=hgs#kQf+?nW$k+Y%HRmN}fR8q6-@RbD}Z)Q(|*TGthz- zqPL}_WC)DR=N=uDw*?CUY&D%qD>-g&&j}=^rn_eg2A!|)F<|)q5@igmF=WQZ)|LFR z9+azq{I4JwkOYWOU|OL0_kZtG*V|e@z_^r5GE%z$T`E5Gt&V(%7MEpN0<0GJX@)%D z_jT^XUGk}${u>*ArYK#)Le&Na7y?S;!N=G&3zNMhM66S(28+zt9l;ZwdlnFljjjMb zyvH$rn1q4G9yi3p1%`GdO91=oggb4*o445xN`(*VsP+B?yR842jg}J+ABp#yN^5?S zG`HC52PFQwZBN?KzMKY=ip5MY0vcf>|L5@1Y*f?mnooUj9&i!Q&7cK-cSm*u4F+4} zL0~SikW}8O50A+Auz3YHkUpzHKmiq>P;}c>|6d!wA7iNE&$+RYFA?E~)+YbDTjc%#dpP+Dw8=uilu4WVqDD%;mF~wufX){fh)hQK zmKATBA@^r48YLFSWEqvi$A>Ip{#W4));ys`URkk zj_~{-Df_?Jiq z&^FN6NRmAJY+G~Cpa#6?w@Oa}{Lol-Ay`?}X$_)#BnkLrXsnIE$Y(R;&3WyYlXt2@ zk6mF`G~qF0&`EhN#mi{ON4{*fCqR3bSPhn5t!AOvNP#B1IaM>H?V8O=z>@|%FMkR2 z@ehP!Mg~FS&Kj?1G=7!b>1chRe5g9ou&n^hLDO75N%>`NGe>4_yWmPJQE<74hK3g% zCfm(bh#&`VK*M(8WUPo|Fx6KpWfF*GXv;t^Tg6DXwY43~u(NmyuNePre@H?NnEQxf<46wN~3ljEt-U24&((xL))P|<@Uu_$XCsQ~b`d5>; zdT&ABvV|b17pAFlExwzz<;ukFmW}jQpB`2GYhof6txe6tKy^%G5E>*J5oe0zn_feZ zVFVB2dKv58uKhBrNE+y-T@YWo2*9LQ0zdK)aa>mN=;p5?O<*k3)QQ#`sv%z3 zs3vBEhi-jV=+&ijWt@6P8gevHZ{5g)Tw!)du_c@*Zhd1|Z+)lG!w=T;A^h1(mt}{| zl-5ky(i+bNb$&Zm9t5MQyvwg6Wj+rOm8m&K=0+`cV2w#nWl%NEaOOey<6k%nZQG&q ziKxw^Zx{BHlM$&?X>QTlZc%;E^kA)zZEA!l{WXlbc%XvT)3bNpVOm8q)4#A$-9Kd; z!bcE>)zRX*kzo|kLYK0)d^CD&?4V@fx=4_$`?6Pzc(k1HJvg~luwqsY{DFleLDcUa zl2}F(x++w(PUU+<`LUbL5$+jtSYyM&%r7qBXcf&v(l)s@FsulfLiG2rV%eu*Uq35Z z%!hy0LL;IBK90u-T=+MRqr>C&=$!>j`tis$)t>&tagtC0)HrZo|+kj zickZdOvaQdxUZDuFA+6qVmbxcF|dXsW*Pdg&S&S)L&p`yfn$NTZV;&J^F+o)CT`s; zDECUBWf2+iAL9q~=ShsIVB?-8mG)U$D-w{ zDu+k-u9K-4o&HZEn>rVakqy^H)-fk`VT8~@F~FU$Cks3KYWZoL9cW_0yGdo)WltFr zfi_RIW$`NI3={|+Y~}G9sQb|I{{yx_Nx%FXFZY3^wyo7LL)WG8q1g?)>QneTtpsYQ>a_Gx_AH_m^4R)6&UzPf3zh!lQpFYdg{YSIy&*d$iSNPY${{SL-PgnG3wffhK{5!9~ zaeOcIFO4r(`E*~xf5oLKOZP7~Z`5CX>i#SHZ^>f!d_8_o!~etpC=dYv0s;X90s{a8 z0RaI3000315g{=_5K&=qfsvuH!64D_;qfs4+5iXv0RRC%5Dp+0Im|plbGhbSZsoOi z1^AZ>bv(<3-Nm9MNEW3!<}e*o5p+uRG7&jTuGcDDq|It({WpnHeyHzLU88HL_JHsu zS8@?Djv0q42bK%DS9x{7J3v?uX-+7DazliJ(9wV5Cxt}fzfc-@!HDGXJV+IYy@hpn z`iPFMtFNBX1sYpXD-zSug$zo^NiTO&^)1t0r9H;_#PyyeCvGJtq^adLj4PpssIL*X zONJwImN1jCBBSAPF{r{giO@w#GOLM(0qTw(yV zi1E8Yg|SkEWr5FkoU~X|Jn$CyotT~op{ZvKQ-&FokbSU5l#9%RX@+@#s859KnP(Z} zeM<|hQt^s_rEb#C3vp6fz=epxk4R;2rRmb8D^tETIuX%CsZymx`i*p=RH;lxrJ+*s zao|qb;}=oKGKt|4sfTWdDcu9K6h@s1nD)oBqFkztqKRp>N^b6@VplnarDl78)srN9 zOzKpla|}#wFe+^W5HB)~z`PJDQF8=)lQ#wi3T1Od1?DYA+qe?3Dka8*P((>v5M_S8 zWzQLd6aqVKnUb%>E5eXHI19=h#qk5-gUnQvB}V}P+5`in9pQ!^E(Ot;H*uo_a5pKc zlvbiv!B2=ewXq7s?!YBOv`7S^Q5dOFAT=qELuOg^FBsDhcXIOI63~p@bQsI@viEe; zl#O;i;wD7AQ8S32`v>%}41PPiT^*My^y~3=a@1h?1j* z;T4%uraMZA1ZA*L$tzN`JwTO2cLL4ADSJ(^4}?2J#HRBvfsE!;Gg+Cqe4)_SR4_c& z08vS865R>{cu864Fmo#rBNcEZ-N0a6D#MpgOdT9~=)jfU8Y<~O66bE1bf#;e-3;mn zpu%Yb8t6=?Q@U499R$*OB&A)F&cqH9O_7PqB|eb zJ%Q@{%tQVu`;-xm;EyXSzcJ5ODhz47Tl+sl$gVfQn?v#;fX3Pe4yw~u8S7f`O|nlI#pOo`MK{bwqm223$1zqSyg<}7s9x#<`G@9t zoZAe>D~Mhl^16w3LkaIR#I;-xa3xBVKqHnq7^JmJN@@ln1Og6VIE1*dFLJqrONM=) z$~s4I%m$3q0^mSEca(RSIhJ0|f$mCq`i@xoAe%oBLo3sPSx=V{xX(C<)W7iDp;G3u zLrNXtiEl!?US>tj%v@1vP-WTov|3mw-Vy#qi_@22^?~6*4O=z^K~Tf8hw>1EJ1l;s zz1!S>W0&$od&Vm*WwPS)7`1G~zcYy65Na(UG+xe0efUE2j-?eF$E9ZWlF>_fG`$P5J1X6c_UsmEpmJV1=HTM zh3B3e6`}yx`lV@cgDmC(1SUeS^FIT$Rxun?sqj#_h%EZvqZFrt#6JtzKammk1%E_23wVFs5Y2fX z*_dO?a9z<--RE?*fl&dUHY9)Negc$cLJ|G;*{XuLC<`;5{hlE8C zET=Op;DIq&kBNzjrkGU0_?Kqkb(lv|;rWG<(`+#BD6GoFGd>_nU`m7l;(3JEGI_B4uu8EBgJY!5eaZwTGh}Wvk!a9mtrd_um5!FpjW8NL& z#^*CrHVbCBeh?teqD4j+VYG6nnOcuXGFBt@l=0BoVKMhBhE&I>lDTCOG0_`|;uS>t zhnPUtAg&-v%rNK_%nR`oh-PyZ8I|T&nNq9NredwDg(G> zk`aFL(5~YM0#?E0UxtG;g3{tGktC`g{wget8>D=$Ap|-U>$iKlCw;ySh>vZXku+v5L+?a zRJbQgjsF0H9*P|`(W-{olO<|9VlaKrNQJ=|a*1(WqB5dXMxaM7ie)t!USru6edW2E zOr}u;tD96pI)H>AJ)v(9qzuPZ3c&HWUoyFs%%~i)vnXy<+E%4^fO&uj2?Mx5iEuXz zFQSQ9g6%44sCYp_FoYy6OuCR%KFkDru(dXfW!xsZ7Y$}_QGCm8a}=lqaN8_U0PQPT z=-b#JAjfVTdVxSiG_M}wq)~L8p%m=K3F17ZNwj?-I@kl4QmBx_GkY?y9RH*M$ zQ_R{SL@mX#q1rpB&o>_Z7qrdJV&krPgdv)e3(`I|2s36q1_&`KCSa8iiHP`(O3ouO zDkQHFF#@Ned`jkF2x8MHuF)3dd4L$GDX2Njt)N*0nPX5M(=A1FEl`Ub%2W(O#4OA^ zK=AB-0Fs28Ve=Kdc7re4rGUyJfC!O&FuYM)5 zAYBQ>XvK2i{lS6*v-~4jZjcOd8BtP@jV<@cuwtLd+_zU>pjWm2A>Ir_TFetb%nv#K z%a_%X{pBL&_a(Cu;V!G~9}p1Vc-7VR?=jAu)`^Frg7ye0Y-Ib0jBaF;BbcnFQaN`$ z(;AqKbmDO_7@Zr7iiV|$xFJ`}qtJ;y$dSr0!g-EU3`}M5m#&s^F;I-jreZ2-Q5%FX z*opX!xQXIE6DB%fgDwM>HiU!1$e6iTfR!$*}Xm?J;uG3ECv3o;ujgEUtTN#K37jYcwBh0KoULa;u zx?;3~@dnt%TWZz*CJ*DU*$@KXc>cjG9~bfvc|583C1;kPepX-z*a!QcJk~#?9hewp zxZN9wQv;*1JtHnL9SNO9Mp;x;TvT07&V@>QOrZ?-NArxTd0a_fQizeGTZL9u2Wg3L zs>}>#8RPK;tY=cQH4EgaOX@j8x>f3a5l}IbZ9AY>Kvb;U2pQ1>nNpZq5olrOpzb;X zjk*MRlew}PryI>C7-EI-6fLJI3=5MNON2;3Vkrf_@f5jcQb7<$c4fx$ArJ~xsH0J8 z#_FTqT&&C-PXKI#Ynpi>Kze8sE`6Kdwf;&31#Vr5!|%eTD!qCzM#kn z@iD%!N;-ddF{1L)S75#vg^q6|%H-+8`ppDwQ*oHKEBZ|kbwqyVK-Q3c+49r}=wok; zRZpOg;=HfzmLYoo0NBD*$NqRrhUa&GYHIz+4@cbbKr9iyS$O%x&9Re4GF7&j-O9Od7 zz>um^xVVKz&xtmmKj1Wl_?zsMm}{ zXXGJsE9wb{$su6bd_r>1Xt|o~;|05h3zsxISlRQ{t z5}1sAL~bNb@gE;jqFhd60_Ogpw)Ts>M7;DX%tx^T3&amBDJroK2n9gSp(_;wysS#? z0*Ry|mk$X}AfZS_`Y#0v!<-<3IK!L>7!_m>s9a!Qmo4V|ipYZ_+ZbF14bd0=iLl<3 zBLU&F_w9~n_Mgi`ifsAA_ZBL=59|yb%}B@@-GXZXb!+hp)T`Woa`RWh1NKWtvw!G> zVR}#ba{vLQ)Ctq07xgMT!$oTGcPvHJfmCP7ic(TcLik8*0nW;EbY^7Ei@Qtqc$IBr zqFbTl$J{4Fq!3gK_*8C}WH2h7+}>RCanSZ*D^e5$^1IVhH!4?{rd@$wsJ%bsV^#8b zLWuCFv3Eb`Q1QF=q*W$!L^^kJ!}LMhu32y8gn>lNK*XpqDpUxWWkpjej%JEYK1BWQR)N+5QG0V)|oyP%7Tir7ioSI*eHADg_QN_;=uNp&twKe%wf$e+|0 zyRPPb2!OSZ{Hj|@a%=lY&9pFt{$RqhNq+`2mg#I0V6SCQ$rYl!Z`#G~ZI9|r2|_&KA1WQ*v8hbGdgaN`rDFbE84 zj6|V87h?1F|}D(FD3OctayrC*Jl{K*%0uW zh<@;2OK~}-3Z~)#9}38NhNS>8!b@rJ^>rg zz^?7T3H_95T;u(sA>XhMh81GGtCZt?E4-&rP67+Gqh~{u>`h^W49bQYgb)D&_EM?J0c_b~spr{{W8>0i@kG z;r*p0fK{vd$Jm&Um4csu{7a3nC2kzdnM~de=ggoWX}WaynW}O%1uW8Cd_n~Hu^*B$ z&;@0;+Dqt14M+0Bjb|u-P?O-Sr|MbzFeQD(%1Yz6(G~=FqxCHtjsE~?mu5Sy%bBQL zwW6H>noq<_59&L#{{YU-%|0bX!V!p(E`v~nc0a&4egZ5e z*us02rJ7g=_Kwq^NxJvrH{0V00BwB;C6BrR-20g^gyL5dXop2yc|;{UN>@e>qFhLk zB~K}YL{}suVje&93$&+w{g>vj-M4bL9Yt&{9?i;>Lu7Qp7)5rK0EyhpIO*t5$Po3M zHZO)h68&9Y@iX`#V6Ot76DNsqfV{#g;ekCsga?RpH-QMGAb=$jr9g#2DjS%n3y7}w z$Y$qp?q5r`z66Ovbp@SzJRq0JXv3;2UZq;vww)uvMqif{Fg5Zx(!EaCH&l> zCY-kA{=^wkz`=fEY)ZVX%rIY;$SHsanM@YNP#Pvgz?;sGETRQ_Op56ROI+?V_i{=O z#MuFTnL@5L8CMX>Q|1XSx`Sq+*oSUi&SK%*L>_xgI)WGVDgi9j>KIxuM&DDN5j!LI zajwuBv4Eye-SGjh!f;R2Gyeb)Z~Y_A*m#sjYqG6^S1v|4nr32dZ2?x^8AsaFWv@9Q z(9U5LV5pz-Yms?!%yNuC@pguqhRQ828}$aOwGFP0Cb)@V$17+9iD=OF3ubd|6Q3=I z#v(B&gxzpawhAajioLDin2EUxJ`r4_aD!JKS`dIFwB)s27G;8RGND1YDUUI`Dv>kr zv4G$PyXs|4wLx!bD8S!jhK;9V{^#Qu%m|P^o91EFMC?ZdQa&d#yu>Oe5{t3-3+X>` zN7+p?y~H?!x}F)!%hLmp#{veEMEI9k!%0+;uWT`y&mb^-kd!WbPXqfMJ{FOD zXv+3RM6rMnAQcfIBvc7dAb6F~J4){Wo~26f1GK7v1EQ4*bQaauW&3VtafU{3Sh}wy zNI-C_@uB&fKELfNi4|?9e=#VcmU$;Ie6da#b}N>BhhtP5h&o4{@hUn9%Aqt{wc=M@SwwXp=SR#{ z>~rc=1&~#!uMnxJQHbp_Y8{A{1Zd_E7$IWe!4z>XTZbJ^-s4a5IDQlW{s@rsUTFqo zAT|(OLmEJMK&}eUyRCXwfDUBtA z-9`uk)~X&B{FgcAHhhUu@9dTeGFT4u7dPSphtYwKPfzIM^DpU_`p*9V63R_YL?RJ6 zVm4b`FW25R(8+YpaAQWrH;0lQ*gNL@!9LciU+z`H_Z94mXc)6o1*gd=`|YTGgfHr4 z3GK-#H%JaHeV~$ah#zn>G0+Js zLuObqfK?+N-CeQae_;|@EKbq7jOBe(fM+ayL$Lam$yT4DJU7(G6q~Z}lM4M1#+7=7 zw$Uo4au&PTnF&ahOgbaC(svla8k6+*o?ynZHXX)g#19ZLG?c;ttV5uGgL2@*z{f=e z-{5xy6KK!;e~Xs`wQuI3^v^DBejxl#N(Z}8PtX&${HvE@*JA`@m7A9qZv@FN5G<7` ztw79m<{xYlnyjs2Sya7GITYEF>ye5mivS9|`+Z$Z$fQuBws*aKlB7cvmZ^){0N=}=GlKt5VI8MZ`1+7k@kMC>QX*TA0yPFSj-J!IbE?SmGw)O6OBhxg5~SD zn_S{ZK66AIs|W!oi|IeI-vp*Ha)x&SglM7ky*1^~OwKtm3bZOkoaftf}Q zp;8oeb1jmRmjmi)h!8VzB>{~oJ|XB2h>RVQpPuo(mjM052VmAmLk5bQxRjF8pCCU#o4ReAVwU_}y(${^Cc!CIn zk^l-4vu+v1JG>e^l<1|#fB#g z9rg|v>19!P#^nOCd4vIe;%;r0H&4`OC($;0=N1a@FwH9UiB~~=#xPJk)C~JsS@_ZW zD4`l4LsFB{1Z;xy2(BljR1Su4p|)`>4O19FPJ%Ta(~0g~GU3KBzCdhjA2Gkwa5<+O zCUon$rTtKf@jI> z{-y;#KkA9FS8}2q_a7H9ZPZI$E)K=&E4RcbHZs9hBXz?O9?`*6L#3;L9v}jghYH+o zX%OUMc9j9zSUgG@C(K34MEJ@(z9t@ir;!G9!?wm&Q$z_>1(c|nMq=koHMeGAlCKA3BeWZ;#k}_9P57Nfjh+}ATwz&%U){>k11z2V{xw~{D|x;gsPdwC6cdh zP}3N6f$=myNN1ve+9BndU?Wy$)?n`^lRTJ?Aq_0=W5fi_kqk8K%e8jp_lln|lI&*+6 z$3ZciKor5ass}1-ni~AWdv?&qL_Y%gsQ3ZIO5OKMiPZ5brx0^*X05B|@|#pKA~6_JRIf6nK!iFK1|?%r4$uzW8&6Jx5l3lvi^JK2RoEvo zw+JEHUVR_9PbySyNc;sBgdW4CTIilmWwHefC~-IxaRU=kASxLFx8hw@NX2rGM|ZRb zr=t3OOA$wI$c_Oac0I_w7~x6 z!w}*lhoV!7ORMnB{{ZqC#34o9a67$;atBCa4`oW5!!4vAL`x;F2j&jEuSdD%%oQ|T z9L?yN8D~+;4CixEW#DRGT8IH~7P#ObsUTFX*>#R9zcSF+8LL#q2JZ842E_Tm-~VIh$vaHBMv180rQ?y{x;I zFm@V<1Zm7y1z;~Yfm;Y8fB6Tw#LFpaDEGC5?k1U=MsB-5lkvTrhG_x7%-fYvCeRX) zp(<3=2~diaT9wR0Rzk8C_CvH-M5oMKEFFKFMBbNLzDl5pk9&#rFG#$wgEyio>W^{p zXtZLZsB-3RX0_16Eiy$kWacWNjm9|Gx75}u4kJeHCBOpmJOQ=vL?KPs?w=ix`Kp8$ zjI{%AjEk0X7aO3C$Y@8%9-z)!lv<$jKgqTdV^rO$JZYq5wa^Y_6o4I$KIN`8Pysun zWbB@0ii1|FiI5c`_5wvsMUlCokCwGD#2^k86dkCQjJL!9!PpGx8nW%W-!Y}AWHN;o ze7W;EmM5_9_>ze2EF-wNia)HAUt;sM$RiIXTP=3#kk zNLE9UZ8&3*zf%bA1&u}1rYMm2G`ZbMJ_xBNheOwSbWIxd7dV!^OB1G&xil3jo9Q@N zRlDv}4J<|83sH+&6cOOym9&FFc!Yc3h#Cl+M6zY(%n~rVQJ7%7x|w<&dF>l302bUd z)KI(9`>s94h%Uysf)hpJGTJPpQ!Sbv)G{m#zGi$C#4R#}0m25v(rsr6CJslvRDSz}a(qOvaSO>++lWzag_yg0K&K34WP*T-P1oW& zpsAxg#@|x4P*SCGr#EZAF+)^a+R$Gu)k@1Nyr^i7 zOU5Be5P&jUB32j84Q>SvZ5xoPvtc-~haznKjzjWJ0R!nT`wWO`EyqL6fxdt^os>Wy zfmrd~;vQHqFPXRCB{UOLRX$#SZK9;Qd;b7Vps&h5f*VQdkT|aa1b(Q0+%C_c{^fPa zOFFfJw=0)R@BCat{{R8vRM5ZUkrOb+(Ff3nT&j24+`OP94{IY4HEF_S_;8t21(%;h z?nqZB!xqVRz7=1x{-C=)AM`}FJeTqteBGb&94Zdzfn8;>m00``P+3>B^O953*f1W6 zLFxxcz9tlp;k$7VykL99GN+F%A{I;1s4FEtQ&Ap~NxhT+?p$Zi8^ zSb^|n6P>E7L~BH+m<}P2GjI!)z(Z*WOp`}+xAMj=oRS+}Q?ABpG^tR%4`eQ0Qs*?G zmz}xPMOun`h5rCY#IPe1AVm+y<~X_sA@lyCp{6g&uhmC5nkCu3jWdWLQQ|s8X*B{Q zF?M5z{{XQ_l-$7R&k>2@z60U_(-}c!iaE`na7W;;kTS4#mQe{r;qHd4drgp8D8DfZ z{pb;lnUOnYJ#B-~o~>hna{hmlV@v}`(rKs%7cg-ZT8r8*x8hRu04lsS4A=0l@Rw{K zNBbK%eSh2v!k!?cuJ93Q&lob#RWKacQB5@OPx^!+HHFh=OZlAcP$&lsPL0JtEti#S zMJg;ontjUzyV#AbyWtPN2zVlvDJ4~bsp@~0;-Xu;u^;Mqo5 zgOfq~NSPPBvip_YaXB>&0!$A!brJlvZ%EW)dl-KQ67r*-@P>g?6lC}PE)*aag#-*r zfx!Z%wH6cJAVSIka;}5SIX%RWK)ZcLiE%}Lv{3c6s}hw%#I+Wxn1x4p(Pr80A(S9o zDf2746?xw*u@q z%%X#f^9e_2_20$<3&yI3wXSHPd&4ldvCp&QPZ`9s#UV!kvjVEW8MruV>iD%v@3%0) z9H3GbKVPX;Q8LS2AL#+8X^s>>cr&;ddWGD)H&f;~V2j+l{S_9bgsflA2EaDT0Ez~Jf8j6Pq9Iki; z#6p$I<_wNcqT!I?70b zy?zv~x;_5@Xv#jatNLae=s)O&+-~7N@3rpBul_mN0I$1%{6L3^qag3RRgbHJ{>0fQ zIhFqa9pN8HAM8lNdSh)j>Rbkzy~fGRx0y?SxE|3_Yf+n0TE>jpVemEgEpSeM(IU%t z-G13)Wa__gU~aE!e@c%LI2eOko-;4;8#cT@(p_qHSN2~to5}hjE(?F$Bc<*V6)hhS zL%hrxl{~~+@mXV6`Q$O z+zP7;cCZ(gyFlV)j&_4m*P4%r^Z`UHwdyR*p%)y|*V_tVh-f*-d+!B?tKyXF)&t=| z3@cIFsy)B5ArQmKZ`u6DE+9SOcbE<*66ULp{EP4rDk=-?uoZp0Lz9^r#S7iF>lINn zhMK`a>)adMVFgxz<~Mr7%F50#dJi*FEJcH%+^FxNh-|!(Hhqg5wKP^5yF@h0ae}^f z&;IUH-1mz)e{I16cxnEoMK^F)@h=-V&X8P()OUQ7{^pnc4>I649fApVA0PRJ>sPG* z08kJLSjEHf8S-TDgKCFgMriwr#UlC(GM$OOZ|*RSzxA6?URjh2^F00TPtk|dL8Nd1 z3dWm?Icz|z{-Bu2gsl3Ig~7A>)lAR%FDd|HT!26E-dQZXpWY`r{0#Oh>Jw#A;;6cM zfqGbB22+bE`HWjG^h+;=ul7&Pz5f7nZF>yLobjtJer_UpLW=jJZifB0?pVS!D<`M*0D9 zkdDL<860E>%vPgaY=3h23AV!c&ZA2mXjF8)zU)Ay+_yeq#0X{}yJAz=MzkvD?SQ^z zvv3YMm0hA@H!$41MofzHD(Ny?kcW$E@_)xL#=#iiI09Qrw6=kUEaD&Bs7cONOfWYk z(CzM3vN@EgPRJS@;RqP91tuzG+b~ew#4%QfaxU}l#76D6vVBI%SP$te<07eE zE?%sch{SZhWB!)coWwa66vNo|fj5Z0V&Wz0ClDoHI)r?nz6?Ji0azDB!l)fVPz6?i z+Z#6SMzP?GYO-kr>RB{y$bR7p@ckitb)NSfTVsUT7W92&QYc368ooO72aFCsoVkq2iSfh z+i|+T7+!p6529v~^h&>UpV&+EcbHDlT|;7zm4pclu9cVxeQ~xMi-7`%EjwYanD!;G zxE?mQZo$~Z4+f%{W?`1@W!k_YcPI`av6!LnEL}_-4NF+KBM@w`s2P^Hm1VQAYt*V> zDhp_U+yqCFexc(yrZ_&!ZSzD%fm*mbpKwZQa2p>ivaB_2<}g;^Z|YLO1%YV)0MtBI zp&9q_IIMVFdfE@7AmdS0Tz7*;<=b^~v6)8`k1+tV>0!W4;)d8SG~6ejdsZFiulbnG zTf3;O#2A#jT}+lsJ7x~xrsoN?8EY*ramzMcOz=VUA2%^$AT53)=RSGU!2}9sO8$sI z01XC+`Iug=mrEYEE?C{fesjTJWWeUx-6;~~>2ZN-VuA_UP_Kl6PB$lJto*>{QCgMa zdSzBfqJ`6!F!IYRn2X$AV|&jeyirAj z%yb-kq7jbs0qy?)Dl)iQmEE1Rj5h92XM9Cdxd4noeq#bCwQ?)(L-iQr1~U9XE+`o( zL9)oa6#St2aCu*b2hGOk61a{DEs{rerA2h+04`R$Pe}HHtr{57MzPu?fq^4Y;N3jj zZaOm<#sXKFq~Wq&nb!g0C4geNP$??{&igye?pZu1cu10tFbMC&1BUS~2I|NAj=ZA( z03a9&V_eJ-ym*GTCNW1aM`>8h$F*~ycbncK6_l8Oe4>KV`HRF0vBYt_%)LaNqP=u= zP%Q%B#~xu#Hq!q9`7UWwTMt|R0Jjc2`qT9P0B(7v{Pf(TylP(2V2e;&)O9vOVBzt@ z?MsLLOZzmbWHFekN-UT9h6T5S;3cgGAy4`e`sHshfd2qplsKysBr8_h37eOhW)X^l_}oUeSeTW`jw?_{ z;wPB5K@IqTd?z@H@X;u262h|`3?S1D>}4kdjNpQ^ZiTMtYC9F+orqtUW|UC3 zM{0uXr?MJ8+4_ryeXFwD^i4bvz2T_ng9g7amTcX-I4UKq5}fKQY#v}|F*`$meZ_7r z3!LI-yl;t^YPu{bL?T;*G0IWo8^lMg0RUq=H4`%zB*Ci*t+7SBCmGD2eCr;uW?c1yw_?S z$sX-gc7YPvLejAoCvWCqssa{%)Gk^K3nQh}a7mtXeb@shcp zZrCQKUCvqL=&lUkG2zx003~^-n8ehkBS92hx{hXYYwjjz`cjoqC1zFP1WduitWGB} z=xz>Ww>aFi&8N)sH8brKvv4uc9SA+*W1ur#Ik9`~!}iM%;$KCb=Y+B%QZTZ&xcSCe zOhm7E`O5c%tb=VN^*U`9%UZH z0GTjq1GU^KDd8PZI?m%HtxGh%U~wB=+v;qGNK!|u#0>ZTARrVlq;I1WE-FnH3px6PRtq ze&YHVth+bz{{USTiK`!bi=ZNLFk^yQ9HnNNUr{kK_bOEqnw6LtiFlPNBq}*&O|x(y z{KD0N%-*ILRG&+KFXA^qi;FOfTzsSF{s6-Z-=V`Xq-WkfQLfwv+_5S+6{|;3wHPyhbQ4YjqZL7YZ$L<%89%O6KS| zwk;@n%7P>Sij9Xy4L?wxXk?ZOMgS_N*wh=WM5g8UE?aTA#JuLQ&$6+4{R#s?< z+i9n3s2+_@NpCa&YxnR4AHaSPh{Bm zb1vp^M!}fq5Fm(~AQgzT2%?-|)CjypTt|u`#9U@BAi>PS{ZBEU&TRTX=P@UuScw2y zU#OXM#d_#CVk%b=ilQV=(JEDODp!aX(ZPawf#xI(%9T+2fz0zrmY?QA{{XX*iS z;sM7N4IOw&#kum2h&;Ph#dHmJ67iSZ;TLtbISE)RZU8_+-~bhX04RrHl?jU|dq66I zcbGmVl8J26u{pSuuwN_;?hA#6u6|`jpD-?#a}2;a3$hHSKH`-(*=%rNIBFmTZDR`2 z+Bw{Lren;*%-M3`3NJBEV+{%x8#q6)XP$lCzkY@e>-9PIVbx zC1xiQiGi%!dHRIna|O$NB3F<>zy=&*#@iIH0p-%p$GI#1TH*i}BP)f*YO2oN2RsjnREM@JEU@%hm9^yjV zwqDY{Guu2Ja7AqE6mmI{zOYsotwBA4a3viUBfc#(fWcY%yDziPKSB5Og z1vLt+GZS9o#y%%8i_A>)Vq|(fH*^DW`1w=WS`v0}wI zi+u&{5f(2IW0M$R=w54(hb%x<>iCJW+YyD%0;5lcrVT*Q}RTnREwZksRMe}pU5U%7br)+c9cy73&X(Ew8N zZc{`>eM(DUM&K&$B56BvjhTmYGO37NMER67Lbs0*Y)dzDd%VU)Efu+o%vPa?5R;7T zhkhYtNH&GrPtM`9TfLw4m#4Eo`h3fmn0Cy+4Ra|`qk-AlPyluqt(erjN=UC!K#V0x zLULSwX!(Vfm|Tfx;%6kP2%^O!W?6SK$C#q@!?dzxr>98im_r9e@PDEN?PSB@6|Cbi z)*v#{o6aUyTE<(0 zV#OBXix%SZ7AU@AGaE(dI7gsw8lDYeEhPavEU{p2Pzo(j1~#%M3fsS5QOJ&2-W03k z2y9oizCszJAxN4?p;pG!^KR!l!8jo3@>O=G@CV@?(2p+Nf!>t8ue`vGgSaY!DC&WJ z)IMV^ZfTO#K)k?QP2-5H%iPVPPl!OXO_<;_EzB`kCgW0+>BBPbJ- zFd_iZz~UpQzM!ef;4V9_5AJ~76xunDp;3&p+CES%FdQ<8h_RP1FbFY8P;r)-lrIc=52ys zcx07`j|``Xm(?K%#AO5X1-do$c>GGhc7kyjv(cAKQY=wev1f5&DleGEG>&+S77{Fp zQTG+j?%@o*L}Ic|9|BsHeBz8i%5urF2Be8ZqnQm-KV6(3Q0^aD_QItzY)Onn?~ijSDBG`V(lJ|W6`G( z?G`$ViKJC2adRGxTw@gy^(YM@WuAKaxoH-+*x=M{>FpfxKf*=I4A z(=Ma6m4b7z$`y6Z6?M~irfQ1|>l)afl7Fg*Ul+ zN!@A$^Rxc|gtbAx;No0w-D4+cA+`>fjH>GJMd&5(3SV+#LoE23!T5#?%e+J@99 z##AP3T*^Qf%mWAoLvd-2-D(cXaX}SD9)}mnC*Kjo>vxu<8 z^N+Xz1ay&wZel8`Cf~#nS%ZVs!9AsU{YSF;XR-GRj$~2y2gl@?Pewnvba%_Y$Pc^6 z`XxuGfwHi4}Pt#Hn_aXZIMX`vyM{6*|*P=pza@1wXlWWyKTn&s3@l z_#xY#(31R65iJJozvMG$MJzQegHh#< z#}--#xq2i+!ZQsYAgLMh1)J^%Ga>C=BjP4NWpc#wswgUQs^%d8Q+5=4p+doHcX?13 z>SYA@mUKMEhIb6@#I2Z?5DCIKkh$g|Yqzz)#Z}9QY6Z7LasL2Ou*(~1h#~9Bl*d)d zvyjZ%D>%xjQVQ%!)q}z)Ri7JWVgy>2rHufPoUNWA-r^iPh2>n#xFGhB0Rs`}5CCuJ zPCjsa$6b^9jia~@97W7>j4X1FQoq{Qd5IHe2Lu8IFa!y3$;u*2aIqSX^DP@UA3X)N z{B&AdaC^dY_Z;p+x)ju_#4oBKts^Xll=g#pp3?DM--ceI<>KPyqPxXK<(Kab1%Q|c zsY;$%XKbeCMlNICHPaY?C6R6@u@Qxf7coWc71UV!5xbVyqBV9!mkE@zv_7BYLnk{R znb?jjE86dzdWs82nagX$S!`Ibq-&JY;fz4pU@&N-5n@eAwXV?CppEb^;sa9S zGuFh&KywJ3L48L(h7@rH^1=&Cm*z2Y($eZIVPe-p#$36WDa>Cng^DcXmlHrpqb(PR z#w=Nj81WUXu}Noq#$Q=piIX+c{(kF3% zSIpK6*%TMmK?UIHihu_gm%MJI?=lLxF@_)eqQ&=5Dj!5#Kyz?y;^cuFcNT-TU}uCt zxcKY<=7C=XsGIa}<`i&BwU~3OyPS}y+c0l2($qV`*}c(4%3nO7{6KKdaI{nu;;8MS zUSda>v#F`DmXM! z-7@g|L*PTTjY(TgTFyK7m28wmR!lYt>%`6bt|V4atgFX~WdTj83>ct3-A{<;5~9lB z%D4|~Vzothdw;YB3*b$s_F#~R0ir0W$IMuQ_#?M$ ztQ0D^1DM$8vDlZ5{$s93wj3X}S;9WLUINkc?jH$l6MB#SJTdHnmHmh&qq^vQJkAS( z?>x|zMyHA)Flm&`{{Vuq=*GH<+?N(HV)^PfDtvUT(kQX$+A)f_%xU7GW6 zmvC$X!f%JdIg2J3T5e_e4J=5PUojX?IZ?G=EIq(K9$>?XMk`(HPTbL03j$aOD@>c=h>ESY-z=-i7^G`{CE#CWQ115%5?PXUEEA&peIa?0gjGSLFyC}upK?1WogDCAD#PLS`MWq*kc2Go04#@_;oRAQCTVK;E zX6;Qv_uC4JF;FWQ?5K3C(cFOEb!EWzgT!W9g`CBNK!~SrLd{2-BN)vw8jM~lQ*k`a zbYU)HEP$C}BzLF2VSm=3*u&Z*s?=u2#tlN&o%*?VEm-R&EH5(>#24OP*=wT&0;cFL z)yd4YP$lJe3y$Ex%DE|qVS$D><|b1SaeT}~1QTI(8K$MswhD%pmMRob>==NHfM9tq z`v!38ovv4sDpB1W`+ApL#&AOcYT)AW?pds-4zX3G8-_4#tB-(dhyzr!D|n31b2snI zYySXXR#OJNf6xrV7nxrhK*9O=f&i#=x})Zgm?z#?8X5NU$>uT&3dKNvMh?=pN!l7z zt9R7DwlOMazlnnrxX|~@U;kf>hQHIqE+#$gX7=p@|1}=fsn}y1S zi1b;unOs03>1@OlrZ@Ed3RGO=kPWyFph)FEyqIiiN-z1c8OP z$~JzhHIsyuAR42;co|HZ5{42L;l9DcIg=$%(kTEld_JKQw57rvge=}K#M%si3ih}) zmEj3d)1$I$%vI8qt@xHC1*fL&KWxh)hKQX!4@o0V#SGz+=~`FiYXm* zSSK?4^%~riGPR$SZ*C+~B(UQ9%kUt#8;A&2CI0|KoT*m&EIqSg3@`^KNHxquTevrZ z21ClCY4^k6IWTDyssj;(qWoQ;v^bsNbfNrSNse9YOo+>5&|ioOTX8V2%mH@3Wz+PG z0;hFLLX8(OYF7-YSi9mZ$(luC7K8>Wyh5tok~9$vD{YR3hQ1G&4%01Cfm+CO<}1w@ z&>&q5V;$p+;qi02qSUN;mXyGw>yGoA_Y)*a8VhtCqbiKK@h~>`+8+M^)9z<fUM+Thz$-_H4|E%Afh9ZZ>jR^m*zh(Fl}QtZSX@-ZCdRdu?rb+DUXn4 z6~@A;CXyb>V}({~0J9%p%mS_l>NC_9c7_fGnb~k)_JamEQ>mPf zhqwLArnF6wVqI%-XS{qhiDK(g>SfCYCwmc~lbFLBvIVE|8JOVWXfNU$p# z9wvLW#L{9(MVu-5>;g7#)Dr zd*kI6EL_Ew4+L>7Sg;zs6T)5~+jFXk+I}a3_CO6{Si{Nv*>cdq%IrW!Q!d$Z??boA z1`R}nrt8ci@&O(hYn1L~ydPNM3Gqm8Fj2$ZNN>Y49P0;~l5>fvKta!s3cq(MPHNq; zTWQT0_LrrgAM7o1* zM&;B|#A>}x=s1JkDRJX43lWP^Z!P_Yjl@9|iKtaUDmY;@y1wz5E!GgFvD!h?F4zJq zk}hD7@f>50(6hQMTB-rh_rF7JWr=R$XXjkRb0O_18Y2q%ELMZcL;7K2tmx=W!()1 zUI~L9ybqpc6s^EMq9X&KFVugyC32;6+FSZ<8b$vE*Mc@)JDgWnk|S? z`ByF|+6Jrysc?)-`-1nE7ZE{UMa!2jqUO@l^LK0@5-oX`+9)g-c0<%R4NEN^oJ*{OVxv1uVLkdS$@G@4 zA~;mVX@E*Ftf)6b%u{OhhWoxh?M@_sA7AYOg3XFZZiPXk!25(i1|Vi0;1*pJMF%x7 z8HS4Xwii6NXhbvUK&`P`pun%+SFTzx|=D~mAnouIri3eaXON)rH z0c#PZd140Dyv?KHI*W05=*G2JVxyC`qFE=BWa0NNhz>xE7;a#pKnIAihGevi7Y#!E?gPyEnw6ca^l>$y%&hG{KfMwUzu@pFEZn} zxR1Ep&kaY(8+k_@#xaO9ydC3li7F*T&e=?@oW@ z_Zb+7(lB^_AZ8WN{PYGmXz!Fv4NoGzoXU${!(RkqjsTYZ6C5*CtKIrD;S9(2x(}D^ zr3BZhVC)76#q>su`j$mM^@=fLf-#AiO~-P;1X*$MEUn9xso|7sSzF(e%5W0 zy-fRzeMWA6q7hbNE=CFt<|6l%6TES~O2nq-98Mx!7%*W&P*FrdE?t$B_F(LG@%0zXw14DbXKQEZmR*e{SG)9O@c}1ls6D^Xk7-6I zJRN|jKXw~^awD{5hJC%3I6~#exb5d7<&)YxRqSUZ9;|{FWCb%riXvbO0us10QyFz z?}D%O97G>u^{6#BM%k4vpkJVuWsOo-x%6f64L@qgJ-^V8XepA92Y~42BJrP5^oj>p z?p(4jVk}R1v0}xDu!%rL`imCWv1RnT;-Uc_W2-XTZS+}qmvM}|W?w-i^s-ylv(c8z zi*r)t&l#R(df!ERTpJ<|!IU%-rip(O)VRK7$D-z#z+u47Hj@ra_CAU{tI(e+{4c7duqs?s!UO z*{))#muP^if7C{nU5LcXba(#%nE0E+SH1La#4*K{P<3FPfHgeaAat zf{3wV%ZiLq9m|U;Wt)Hzr%-&Nu_DF9e5K2mFU(@)#e`lZ%jtN7E?zSS1ioe24080o zgDNkP)M$A&@gcTLOPddgtBIBDr&+FV4htxhBC)v2(`7ohn@m7I2+&-B1;D;pOF0$ z+E`(j8aNX;z2wX(U1+uJG3mb${bs6z{{Rk;5I}_*PXQ|t$KnaiLcjYjJBt=RU*HTd z+H-eJs9mKl^ZON-A%#l*RV&E=OCF-XJEFHUaa#g`T=&H0X@4{&_N1`$FG z9)SRz+FQ(8ix{i}aiT0(-AWu2aNJY8MdC8+9XAl~ zH8azgvvc1-;U3WFY8^gd8F9`b=FTO{mo8t_Uh@0P%)5gx#J1u z?p(hT<@t+r@h?3VXPBwinO!ZZf8kSz<_sdtJH$l~3>NnggbW8*SynQ9K`{7B0Y5YA=lIEg!@rW^60f#4u`E+=NaN)1Je25MY77oElyUqLRQ6GU3#D6vJN zROJ>fW#^#)z#SL-E?-BLr1`i*rS!I2EtbMuVHspt#*+Dp!r;B}EUd;{9L@B&`iHOZ zzVXkg#2EOO<|}m_P9y#`m>=Njjv|J+f>anJ;OH4_!VU2(41k@x^p@y9DOa(U7%>L5 zQ59EgQr3GqcC0d=2Sn#=#si~)#-hB+`P<8IhQVBWfweL;w-Ey}+AFl^Ix-(L2Hn96|;Vf*YB)E&j=F>F9>W*UY;A0M3TI%psODmJP0_ zr$L6{50n$U7=vno;c2qp>gi!La!dAWIvV;J;vIG+*Q)90-<)tq#% z@_)kL64-x-F(pS(06?D-rx5F1D>AxNtMx06sh*W6iOi{8rDjyAR}=9HT*U1vByWjP zAknxl1QxQvn_Go;gdZ`3Qu-QBax}{UGHM+%5N!yH#j|Sa4p!j82qfrNr%r`Af^pQR zcxBKGTE7y37yJUnSg~W2TuXb!iytv~kG_`{T-QsFOP4NQWy_Zqx)xh5UqD-TZ!!qv(sLkIz8qw>HOgUt|lzNA&nxwCXwhqQv_vv%Zn~9 zC66)ii!NVBqc8se%Hyk#_&b}6Qu^i)m)BheTv$C0u4R*Q@fKXXOT@Wy;`Wbn7Akzj z9%UZBCCiuQUzok($523pbUjO!S#j|>m*P@fc(_2FlCggYS<1fPhM@*s3E?noSho$i`bLL)ZWMY>Lk*nzUj2J>*C#j}iGV5IzHp3}* z>L#G1!zpEl5uIS?qTsyrTo^^W!U$#Q<SiKoffAPA5(dbOgb@`d-tf9LmJKshHZ>{uo(zDYhp^|esoOI(P zV6RO1j0MW6xlY;6zrZkO68bG-B*y20pr%%#8mV!@F!Til1%woZzr+~QX$Ty|fgHk# za^?4fo{N_*UCWo#L|o2lGW!Iv&N>6+-0x)ABE zm^wR!I(qzcSh$RHh$zdlKBGQnYG$(rSt}7h&H^y7PUQK8OR0V(vi#3;12_IQP9>H8 zF8Z7Q02-T&u8m8~yNS@6%(J4!?KJp@Gnb=Jo}M8b2)m3rqPd6eH}tHhNlc;9)oP(r*H5*0nzvqU$!$xqom~M uCoDRg^x8be9UP}5W4?y|A=8!`j-HG Date: Mon, 13 Dec 2021 19:10:02 +0000 Subject: [PATCH 164/226] build: winpack_dldt with dldt 2021.4.2 --- cmake/OpenCVDetectInferenceEngine.cmake | 4 +- ...-dldt-disable-multidevice-autoplugin.patch | 16 ++ ...20210630-dldt-disable-unused-targets.patch | 219 ++++++++++++++++++ .../2021.4.2/20210630-dldt-pdb.patch | 15 ++ .../2021.4.2/20210630-dldt-vs-version.patch | 16 ++ .../winpack_dldt/2021.4.2/build.config.py | 1 + .../winpack_dldt/2021.4.2/patch.config.py | 4 + .../winpack_dldt/2021.4.2/sysroot.config.py | 56 +++++ platforms/winpack_dldt/build_package.py | 2 +- 9 files changed, 330 insertions(+), 3 deletions(-) create mode 100644 platforms/winpack_dldt/2021.4.2/20210630-dldt-disable-multidevice-autoplugin.patch create mode 100644 platforms/winpack_dldt/2021.4.2/20210630-dldt-disable-unused-targets.patch create mode 100644 platforms/winpack_dldt/2021.4.2/20210630-dldt-pdb.patch create mode 100644 platforms/winpack_dldt/2021.4.2/20210630-dldt-vs-version.patch create mode 100644 platforms/winpack_dldt/2021.4.2/build.config.py create mode 100644 platforms/winpack_dldt/2021.4.2/patch.config.py create mode 100644 platforms/winpack_dldt/2021.4.2/sysroot.config.py diff --git a/cmake/OpenCVDetectInferenceEngine.cmake b/cmake/OpenCVDetectInferenceEngine.cmake index 35640fa719..128883b128 100644 --- a/cmake/OpenCVDetectInferenceEngine.cmake +++ b/cmake/OpenCVDetectInferenceEngine.cmake @@ -112,8 +112,8 @@ if(DEFINED InferenceEngine_VERSION) endif() endif() if(NOT INF_ENGINE_RELEASE AND NOT INF_ENGINE_RELEASE_INIT) - message(STATUS "WARNING: InferenceEngine version has not been set, 2021.4.1 will be used by default. Set INF_ENGINE_RELEASE variable if you experience build errors.") - set(INF_ENGINE_RELEASE_INIT "2021040100") + message(STATUS "WARNING: InferenceEngine version has not been set, 2021.4.2 will be used by default. Set INF_ENGINE_RELEASE variable if you experience build errors.") + set(INF_ENGINE_RELEASE_INIT "2021040200") elseif(DEFINED INF_ENGINE_RELEASE) set(INF_ENGINE_RELEASE_INIT "${INF_ENGINE_RELEASE}") endif() diff --git a/platforms/winpack_dldt/2021.4.2/20210630-dldt-disable-multidevice-autoplugin.patch b/platforms/winpack_dldt/2021.4.2/20210630-dldt-disable-multidevice-autoplugin.patch new file mode 100644 index 0000000000..f1e7487442 --- /dev/null +++ b/platforms/winpack_dldt/2021.4.2/20210630-dldt-disable-multidevice-autoplugin.patch @@ -0,0 +1,16 @@ +diff --git a/inference-engine/src/CMakeLists.txt b/inference-engine/src/CMakeLists.txt +index 0ba0dd78..7d34e7cb 100644 +--- a/inference-engine/src/CMakeLists.txt ++++ b/inference-engine/src/CMakeLists.txt +@@ -26,9 +26,9 @@ endif() + + add_subdirectory(hetero_plugin) + +-add_subdirectory(auto_plugin) ++#add_subdirectory(auto_plugin) + +-add_subdirectory(multi_device) ++#add_subdirectory(multi_device) + + add_subdirectory(transformations) + diff --git a/platforms/winpack_dldt/2021.4.2/20210630-dldt-disable-unused-targets.patch b/platforms/winpack_dldt/2021.4.2/20210630-dldt-disable-unused-targets.patch new file mode 100644 index 0000000000..9d44cdadc6 --- /dev/null +++ b/platforms/winpack_dldt/2021.4.2/20210630-dldt-disable-unused-targets.patch @@ -0,0 +1,219 @@ +diff --git a/cmake/developer_package/add_ie_target.cmake b/cmake/developer_package/add_ie_target.cmake +index d49f16a4d..2726ca787 100644 +--- a/cmake/developer_package/add_ie_target.cmake ++++ b/cmake/developer_package/add_ie_target.cmake +@@ -92,7 +92,7 @@ function(addIeTarget) + if (ARG_TYPE STREQUAL EXECUTABLE) + add_executable(${ARG_NAME} ${all_sources}) + elseif(ARG_TYPE STREQUAL STATIC OR ARG_TYPE STREQUAL SHARED) +- add_library(${ARG_NAME} ${ARG_TYPE} ${all_sources}) ++ add_library(${ARG_NAME} ${ARG_TYPE} EXCLUDE_FROM_ALL ${all_sources}) + else() + message(SEND_ERROR "Invalid target type ${ARG_TYPE} specified for target name ${ARG_NAME}") + endif() +diff --git a/inference-engine/CMakeLists.txt b/inference-engine/CMakeLists.txt +index 1ac7fd8bf..df7091e51 100644 +--- a/inference-engine/CMakeLists.txt ++++ b/inference-engine/CMakeLists.txt +@@ -39,7 +39,7 @@ if(ENABLE_TESTS) + add_subdirectory(tests) + endif() + +-add_subdirectory(tools) ++#add_subdirectory(tools) + + function(ie_build_samples) + # samples should be build with the same flags as from OpenVINO package, +@@ -58,7 +58,7 @@ endfunction() + + # gflags and format_reader targets are kept inside of samples directory and + # they must be built even if samples build is disabled (required for tests and tools). +-ie_build_samples() ++#ie_build_samples() + + if(ENABLE_PYTHON) + add_subdirectory(ie_bridges/python) +@@ -142,7 +142,7 @@ endif() + # Developer package + # + +-openvino_developer_export_targets(COMPONENT openvino_common TARGETS format_reader gflags ie_samples_utils) ++#openvino_developer_export_targets(COMPONENT openvino_common TARGETS format_reader gflags ie_samples_utils) + + # for Template plugin + if(NGRAPH_INTERPRETER_ENABLE) +@@ -166,7 +166,7 @@ function(ie_generate_dev_package_config) + @ONLY) + endfunction() + +-ie_generate_dev_package_config() ++#ie_generate_dev_package_config() + + # + # Coverage +diff --git a/inference-engine/src/inference_engine/CMakeLists.txt b/inference-engine/src/inference_engine/CMakeLists.txt +index e8ed1a5c4..1fc9fc3ff 100644 +--- a/inference-engine/src/inference_engine/CMakeLists.txt ++++ b/inference-engine/src/inference_engine/CMakeLists.txt +@@ -110,7 +110,7 @@ add_cpplint_target(${TARGET_NAME}_plugin_api_cpplint FOR_SOURCES ${plugin_api_sr + + # Create object library + +-add_library(${TARGET_NAME}_obj OBJECT ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL + ${LIBRARY_SRC} + ${LIBRARY_HEADERS} + ${PUBLIC_HEADERS}) +@@ -181,7 +181,7 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) + + # Static library used for unit tests which are always built + +-add_library(${TARGET_NAME}_s STATIC ++add_library(${TARGET_NAME}_s STATIC EXCLUDE_FROM_ALL + $ + $ + ${IE_STATIC_DEPENDENT_FILES}) +diff --git a/inference-engine/src/legacy_api/CMakeLists.txt b/inference-engine/src/legacy_api/CMakeLists.txt +index 8eae82bd2..e0e6745b1 100644 +--- a/inference-engine/src/legacy_api/CMakeLists.txt ++++ b/inference-engine/src/legacy_api/CMakeLists.txt +@@ -26,7 +26,7 @@ endif() + + file(TOUCH ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp) + +-add_library(${TARGET_NAME}_obj OBJECT ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL + ${LIBRARY_SRC} + ${PUBLIC_HEADERS}) + +diff --git a/inference-engine/src/mkldnn_plugin/CMakeLists.txt b/inference-engine/src/mkldnn_plugin/CMakeLists.txt +index fe57b29dd..07831e2fb 100644 +--- a/inference-engine/src/mkldnn_plugin/CMakeLists.txt ++++ b/inference-engine/src/mkldnn_plugin/CMakeLists.txt +@@ -67,7 +67,7 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) + + # add test object library + +-add_library(${TARGET_NAME}_obj OBJECT ${SOURCES} ${HEADERS}) ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL ${SOURCES} ${HEADERS}) + target_link_libraries(${TARGET_NAME}_obj PUBLIC mkldnn) + + target_include_directories(${TARGET_NAME}_obj PRIVATE $ +diff --git a/inference-engine/src/preprocessing/CMakeLists.txt b/inference-engine/src/preprocessing/CMakeLists.txt +index f9548339d..ef962145a 100644 +--- a/inference-engine/src/preprocessing/CMakeLists.txt ++++ b/inference-engine/src/preprocessing/CMakeLists.txt +@@ -101,7 +101,7 @@ endif() + + # Create object library + +-add_library(${TARGET_NAME}_obj OBJECT ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL + ${LIBRARY_SRC} + ${LIBRARY_HEADERS}) + +@@ -153,7 +153,7 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) + + # Static library used for unit tests which are always built + +-add_library(${TARGET_NAME}_s STATIC ++add_library(${TARGET_NAME}_s STATIC EXCLUDE_FROM_ALL + $) + + set_ie_threading_interface_for(${TARGET_NAME}_s) +diff --git a/inference-engine/src/vpu/common/CMakeLists.txt b/inference-engine/src/vpu/common/CMakeLists.txt +index 249e47c28..4ddf63049 100644 +--- a/inference-engine/src/vpu/common/CMakeLists.txt ++++ b/inference-engine/src/vpu/common/CMakeLists.txt +@@ -5,7 +5,7 @@ + file(GLOB_RECURSE SOURCES *.cpp *.hpp *.h) + + function(add_common_target TARGET_NAME STATIC_IE) +- add_library(${TARGET_NAME} STATIC ${SOURCES}) ++ add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${SOURCES}) + + ie_faster_build(${TARGET_NAME} + UNITY +@@ -60,7 +60,7 @@ add_common_target("vpu_common_lib" FALSE) + + # Unit tests support for graph transformer + if(WIN32) +- add_common_target("vpu_common_lib_test_static" TRUE) ++ #add_common_target("vpu_common_lib_test_static" TRUE) + else() + add_library("vpu_common_lib_test_static" ALIAS "vpu_common_lib") + endif() +diff --git a/inference-engine/src/vpu/graph_transformer/CMakeLists.txt b/inference-engine/src/vpu/graph_transformer/CMakeLists.txt +index bc73ab5b1..b4c1547fc 100644 +--- a/inference-engine/src/vpu/graph_transformer/CMakeLists.txt ++++ b/inference-engine/src/vpu/graph_transformer/CMakeLists.txt +@@ -5,7 +5,7 @@ + file(GLOB_RECURSE SOURCES *.cpp *.hpp *.h *.inc) + + function(add_graph_transformer_target TARGET_NAME STATIC_IE) +- add_library(${TARGET_NAME} STATIC ${SOURCES}) ++ add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${SOURCES}) + + set_ie_threading_interface_for(${TARGET_NAME}) + +@@ -70,7 +70,7 @@ add_graph_transformer_target("vpu_graph_transformer" FALSE) + + # Unit tests support for graph transformer + if(WIN32) +- add_graph_transformer_target("vpu_graph_transformer_test_static" TRUE) ++ #add_graph_transformer_target("vpu_graph_transformer_test_static" TRUE) + else() + add_library("vpu_graph_transformer_test_static" ALIAS "vpu_graph_transformer") + endif() +diff --git a/inference-engine/thirdparty/pugixml/CMakeLists.txt b/inference-engine/thirdparty/pugixml/CMakeLists.txt +index 8bcb2801a..f7e031c01 100644 +--- a/inference-engine/thirdparty/pugixml/CMakeLists.txt ++++ b/inference-engine/thirdparty/pugixml/CMakeLists.txt +@@ -41,7 +41,7 @@ if(BUILD_SHARED_LIBS) + else() + add_library(pugixml STATIC ${SOURCES}) + if (MSVC) +- add_library(pugixml_mt STATIC ${SOURCES}) ++ #add_library(pugixml_mt STATIC ${SOURCES}) + #if (WIN32) + # set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MT") + # set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd") +diff --git a/ngraph/core/builder/CMakeLists.txt b/ngraph/core/builder/CMakeLists.txt +index ff5c381e7..2797ec9ab 100644 +--- a/ngraph/core/builder/CMakeLists.txt ++++ b/ngraph/core/builder/CMakeLists.txt +@@ -16,7 +16,7 @@ source_group("src" FILES ${LIBRARY_SRC}) + source_group("include" FILES ${PUBLIC_HEADERS}) + + # Create shared library +-add_library(${TARGET_NAME} STATIC ${LIBRARY_SRC} ${PUBLIC_HEADERS}) ++add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${LIBRARY_SRC} ${PUBLIC_HEADERS}) + + if(COMMAND ie_faster_build) + ie_faster_build(${TARGET_NAME} +diff --git a/ngraph/core/reference/CMakeLists.txt b/ngraph/core/reference/CMakeLists.txt +index ef4a764ab..f6d3172e2 100644 +--- a/ngraph/core/reference/CMakeLists.txt ++++ b/ngraph/core/reference/CMakeLists.txt +@@ -16,7 +16,7 @@ source_group("src" FILES ${LIBRARY_SRC}) + source_group("include" FILES ${PUBLIC_HEADERS}) + + # Create shared library +-add_library(${TARGET_NAME} STATIC ${LIBRARY_SRC} ${PUBLIC_HEADERS}) ++add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${LIBRARY_SRC} ${PUBLIC_HEADERS}) + + if(COMMAND ie_faster_build) + ie_faster_build(${TARGET_NAME} +diff --git a/openvino/itt/CMakeLists.txt b/openvino/itt/CMakeLists.txt +index e9f880b8c..c63f4df63 100644 +--- a/openvino/itt/CMakeLists.txt ++++ b/openvino/itt/CMakeLists.txt +@@ -6,7 +6,7 @@ set(TARGET_NAME itt) + + file(GLOB_RECURSE SOURCES "src/*.cpp" "src/*.hpp") + +-add_library(${TARGET_NAME} STATIC ${SOURCES}) ++add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${SOURCES}) + + add_library(openvino::itt ALIAS ${TARGET_NAME}) + diff --git a/platforms/winpack_dldt/2021.4.2/20210630-dldt-pdb.patch b/platforms/winpack_dldt/2021.4.2/20210630-dldt-pdb.patch new file mode 100644 index 0000000000..65e6f84dc8 --- /dev/null +++ b/platforms/winpack_dldt/2021.4.2/20210630-dldt-pdb.patch @@ -0,0 +1,15 @@ +iff --git a/CMakeLists.txt b/CMakeLists.txt +index e0706a72e..9a053b1e4 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -6,6 +6,10 @@ cmake_minimum_required(VERSION 3.13) + + project(OpenVINO) + ++set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Zi /FS") ++set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /DEBUG /OPT:REF /OPT:ICF") ++set(CMAKE_MODULE_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /DEBUG /OPT:REF /OPT:ICF") ++ + set(OpenVINO_MAIN_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + set(IE_MAIN_SOURCE_DIR ${OpenVINO_MAIN_SOURCE_DIR}/inference-engine) + diff --git a/platforms/winpack_dldt/2021.4.2/20210630-dldt-vs-version.patch b/platforms/winpack_dldt/2021.4.2/20210630-dldt-vs-version.patch new file mode 100644 index 0000000000..36b0068775 --- /dev/null +++ b/platforms/winpack_dldt/2021.4.2/20210630-dldt-vs-version.patch @@ -0,0 +1,16 @@ +diff --git a/cmake/developer_package/vs_version/vs_version.cmake b/cmake/developer_package/vs_version/vs_version.cmake +index 14d4c0e1e..6a44f73b9 100644 +--- a/cmake/developer_package/vs_version/vs_version.cmake ++++ b/cmake/developer_package/vs_version/vs_version.cmake +@@ -8,9 +8,9 @@ set(IE_VS_VER_FILEVERSION_STR "${IE_VERSION_MAJOR}.${IE_VERSION_MINOR}.${IE_VERS + + set(IE_VS_VER_COMPANY_NAME_STR "Intel Corporation") + set(IE_VS_VER_PRODUCTVERSION_STR "${CI_BUILD_NUMBER}") +-set(IE_VS_VER_PRODUCTNAME_STR "OpenVINO toolkit") ++set(IE_VS_VER_PRODUCTNAME_STR "OpenVINO toolkit (for OpenCV Windows package)") + set(IE_VS_VER_COPYRIGHT_STR "Copyright (C) 2018-2021, Intel Corporation") +-set(IE_VS_VER_COMMENTS_STR "https://docs.openvinotoolkit.org/") ++set(IE_VS_VER_COMMENTS_STR "https://github.com/opencv/opencv/wiki/Intel%27s-Deep-Learning-Inference-Engine-backend") + + # + # ie_add_vs_version_file(NAME diff --git a/platforms/winpack_dldt/2021.4.2/build.config.py b/platforms/winpack_dldt/2021.4.2/build.config.py new file mode 100644 index 0000000000..708dcdddfb --- /dev/null +++ b/platforms/winpack_dldt/2021.4.2/build.config.py @@ -0,0 +1 @@ +os.environ['CI_BUILD_NUMBER'] = '2021.4.2-opencv_winpack_dldt' diff --git a/platforms/winpack_dldt/2021.4.2/patch.config.py b/platforms/winpack_dldt/2021.4.2/patch.config.py new file mode 100644 index 0000000000..7f8715aae2 --- /dev/null +++ b/platforms/winpack_dldt/2021.4.2/patch.config.py @@ -0,0 +1,4 @@ +applyPatch('20210630-dldt-disable-unused-targets.patch') +applyPatch('20210630-dldt-pdb.patch') +applyPatch('20210630-dldt-disable-multidevice-autoplugin.patch') +applyPatch('20210630-dldt-vs-version.patch') diff --git a/platforms/winpack_dldt/2021.4.2/sysroot.config.py b/platforms/winpack_dldt/2021.4.2/sysroot.config.py new file mode 100644 index 0000000000..fa4281107d --- /dev/null +++ b/platforms/winpack_dldt/2021.4.2/sysroot.config.py @@ -0,0 +1,56 @@ +sysroot_bin_dir = prepare_dir(self.sysrootdir / 'bin') +copytree(self.build_dir / 'install', self.sysrootdir / 'ngraph') +#rm_one(self.sysrootdir / 'ngraph' / 'lib' / 'ngraph.dll') + +build_config = 'Release' if not self.config.build_debug else 'Debug' +build_bin_dir = self.build_dir / 'bin' / 'intel64' / build_config + +def copy_bin(name): + global build_bin_dir, sysroot_bin_dir + copytree(build_bin_dir / name, sysroot_bin_dir / name) + +dll_suffix = 'd' if self.config.build_debug else '' +def copy_dll(name): + global copy_bin, dll_suffix + copy_bin(name + dll_suffix + '.dll') + copy_bin(name + dll_suffix + '.pdb') + +copy_bin('cache.json') +copy_dll('clDNNPlugin') +copy_dll('HeteroPlugin') +copy_dll('inference_engine') +copy_dll('inference_engine_ir_reader') +#copy_dll('inference_engine_ir_v7_reader') +copy_dll('inference_engine_legacy') +copy_dll('inference_engine_transformations') # runtime +copy_dll('inference_engine_lp_transformations') # runtime +#copy_dll('inference_engine_preproc') # runtime +copy_dll('MKLDNNPlugin') # runtime +copy_dll('myriadPlugin') # runtime +#copy_dll('MultiDevicePlugin') # runtime, not used +copy_dll('ngraph') +copy_bin('plugins.xml') +copy_bin('pcie-ma2x8x.elf') +copy_bin('usb-ma2x8x.mvcmd') + +copytree(self.srcdir / 'inference-engine' / 'temp' / 'tbb' / 'bin', sysroot_bin_dir) +copytree(self.srcdir / 'inference-engine' / 'temp' / 'tbb', self.sysrootdir / 'tbb') + +sysroot_ie_dir = prepare_dir(self.sysrootdir / 'deployment_tools' / 'inference_engine') +sysroot_ie_lib_dir = prepare_dir(sysroot_ie_dir / 'lib' / 'intel64') + +copytree(self.srcdir / 'inference-engine' / 'include', sysroot_ie_dir / 'include') +if not self.config.build_debug: + copytree(build_bin_dir / 'ngraph.lib', sysroot_ie_lib_dir / 'ngraph.lib') + copytree(build_bin_dir / 'inference_engine.lib', sysroot_ie_lib_dir / 'inference_engine.lib') + copytree(build_bin_dir / 'inference_engine_ir_reader.lib', sysroot_ie_lib_dir / 'inference_engine_ir_reader.lib') + copytree(build_bin_dir / 'inference_engine_legacy.lib', sysroot_ie_lib_dir / 'inference_engine_legacy.lib') +else: + copytree(build_bin_dir / 'ngraphd.lib', sysroot_ie_lib_dir / 'ngraphd.lib') + copytree(build_bin_dir / 'inference_engined.lib', sysroot_ie_lib_dir / 'inference_engined.lib') + copytree(build_bin_dir / 'inference_engine_ir_readerd.lib', sysroot_ie_lib_dir / 'inference_engine_ir_readerd.lib') + copytree(build_bin_dir / 'inference_engine_legacyd.lib', sysroot_ie_lib_dir / 'inference_engine_legacyd.lib') + +sysroot_license_dir = prepare_dir(self.sysrootdir / 'etc' / 'licenses') +copytree(self.srcdir / 'LICENSE', sysroot_license_dir / 'dldt-LICENSE') +copytree(self.sysrootdir / 'tbb/LICENSE', sysroot_license_dir / 'tbb-LICENSE') diff --git a/platforms/winpack_dldt/build_package.py b/platforms/winpack_dldt/build_package.py index 0292b028b8..7875f11d31 100644 --- a/platforms/winpack_dldt/build_package.py +++ b/platforms/winpack_dldt/build_package.py @@ -469,7 +469,7 @@ class Builder: def main(): dldt_src_url = 'https://github.com/openvinotoolkit/openvino' - dldt_src_commit = '2021.4.1' + dldt_src_commit = '2021.4.2' dldt_config = None dldt_release = None From 3cc83ce024d252aa7306bccaf92fef8aeef2764c Mon Sep 17 00:00:00 2001 From: Zhuo Zhang Date: Tue, 14 Dec 2021 21:06:02 +0800 Subject: [PATCH 165/226] docs: correct normalize factor in gaussian pyramid tutorial --- doc/tutorials/imgproc/pyramids/pyramids.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tutorials/imgproc/pyramids/pyramids.markdown b/doc/tutorials/imgproc/pyramids/pyramids.markdown index 2c9a7ec8d3..94dd36e83d 100644 --- a/doc/tutorials/imgproc/pyramids/pyramids.markdown +++ b/doc/tutorials/imgproc/pyramids/pyramids.markdown @@ -47,7 +47,7 @@ Theory - To produce layer \f$(i+1)\f$ in the Gaussian pyramid, we do the following: - Convolve \f$G_{i}\f$ with a Gaussian kernel: - \f[\frac{1}{16} \begin{bmatrix} 1 & 4 & 6 & 4 & 1 \\ 4 & 16 & 24 & 16 & 4 \\ 6 & 24 & 36 & 24 & 6 \\ 4 & 16 & 24 & 16 & 4 \\ 1 & 4 & 6 & 4 & 1 \end{bmatrix}\f] + \f[\frac{1}{256} \begin{bmatrix} 1 & 4 & 6 & 4 & 1 \\ 4 & 16 & 24 & 16 & 4 \\ 6 & 24 & 36 & 24 & 6 \\ 4 & 16 & 24 & 16 & 4 \\ 1 & 4 & 6 & 4 & 1 \end{bmatrix}\f] - Remove every even-numbered row and column. From 295da7e5f365d0dd32f8447774dd7b0f4ca59d5b Mon Sep 17 00:00:00 2001 From: Trutnev Aleksei Date: Tue, 14 Dec 2021 18:16:59 +0300 Subject: [PATCH 166/226] Merge pull request #21150 from alexgiving:atrutnev/fluid_perf GAPI: Add compare function to perf tests * Add PhasePerfTest and SqrtPerfTest * rebasing * debug tests * remove spaces * Disable DivRCPerfTestGPU * rebase * Applied comments * Correction * Revert parameter changes --- .../gapi/perf/common/gapi_core_perf_tests.hpp | 76 +- .../perf/common/gapi_core_perf_tests_inl.hpp | 867 +++++++++++------- .../perf/cpu/gapi_core_perf_tests_cpu.cpp | 474 +++++----- .../perf/cpu/gapi_core_perf_tests_fluid.cpp | 434 +++++---- .../perf/gpu/gapi_core_perf_tests_gpu.cpp | 438 +++++---- 5 files changed, 1300 insertions(+), 989 deletions(-) diff --git a/modules/gapi/perf/common/gapi_core_perf_tests.hpp b/modules/gapi/perf/common/gapi_core_perf_tests.hpp index f3f251167b..7a1568ad22 100644 --- a/modules/gapi/perf/common/gapi_core_perf_tests.hpp +++ b/modules/gapi/perf/common/gapi_core_perf_tests.hpp @@ -26,64 +26,62 @@ namespace opencv_test }; //------------------------------------------------------------------------------ - - class AddPerfTest : public TestPerfParams> {}; + class PhasePerfTest : public TestPerfParams> {}; + class SqrtPerfTest : public TestPerfParams> {}; + class AddPerfTest : public TestPerfParams> {}; class AddCPerfTest : public TestPerfParams> {}; - class SubPerfTest : public TestPerfParams> {}; + class SubPerfTest : public TestPerfParams> {}; class SubCPerfTest : public TestPerfParams> {}; - class SubRCPerfTest : public TestPerfParams> {}; + class SubRCPerfTest : public TestPerfParams> {}; class MulPerfTest : public TestPerfParams> {}; class MulDoublePerfTest : public TestPerfParams> {}; class MulCPerfTest : public TestPerfParams> {}; class DivPerfTest : public TestPerfParams> {}; - class DivCPerfTest : public TestPerfParams> {}; - class DivRCPerfTest : public TestPerfParams> {}; - class MaskPerfTest : public TestPerfParams> {}; - class MeanPerfTest : public TestPerfParams> {}; + class DivCPerfTest : public TestPerfParams> {}; + class DivRCPerfTest : public TestPerfParams> {}; + class MaskPerfTest : public TestPerfParams> {}; + class MeanPerfTest : public TestPerfParams> {}; class Polar2CartPerfTest : public TestPerfParams> {}; class Cart2PolarPerfTest : public TestPerfParams> {}; - class CmpPerfTest : public TestPerfParams> {}; + class CmpPerfTest : public TestPerfParams> {}; class CmpWithScalarPerfTest : public TestPerfParams> {}; - class BitwisePerfTest : public TestPerfParams> {}; - class BitwiseNotPerfTest : public TestPerfParams> {}; - class SelectPerfTest : public TestPerfParams> {}; - class MinPerfTest : public TestPerfParams> {}; - class MaxPerfTest : public TestPerfParams> {}; - class AbsDiffPerfTest : public TestPerfParams> {}; + class BitwisePerfTest : public TestPerfParams> {}; + class BitwiseNotPerfTest : public TestPerfParams> {}; + class SelectPerfTest : public TestPerfParams> {}; + class MinPerfTest : public TestPerfParams> {}; + class MaxPerfTest : public TestPerfParams> {}; + class AbsDiffPerfTest : public TestPerfParams> {}; class AbsDiffCPerfTest : public TestPerfParams> {}; class SumPerfTest : public TestPerfParams> {}; class CountNonZeroPerfTest : public TestPerfParams> {}; class AddWeightedPerfTest : public TestPerfParams> {}; class NormPerfTest : public TestPerfParams> {}; - class IntegralPerfTest : public TestPerfParams> {}; - class ThresholdPerfTest : public TestPerfParams> {}; - class ThresholdOTPerfTest : public TestPerfParams> {}; - class InRangePerfTest : public TestPerfParams> {}; - class Split3PerfTest : public TestPerfParams> {}; - class Split4PerfTest : public TestPerfParams> {}; - class Merge3PerfTest : public TestPerfParams> {}; - class Merge4PerfTest : public TestPerfParams> {}; - class RemapPerfTest : public TestPerfParams> {}; - class FlipPerfTest : public TestPerfParams> {}; - class CropPerfTest : public TestPerfParams> {}; - class CopyPerfTest : public TestPerfParams> {}; - class ConcatHorPerfTest : public TestPerfParams> {}; - class ConcatHorVecPerfTest : public TestPerfParams> {}; - class ConcatVertPerfTest : public TestPerfParams> {}; - class ConcatVertVecPerfTest : public TestPerfParams> {}; - class LUTPerfTest : public TestPerfParams> {}; + class IntegralPerfTest : public TestPerfParams> {}; + class ThresholdPerfTest : public TestPerfParams> {}; + class ThresholdOTPerfTest : public TestPerfParams> {}; + class InRangePerfTest : public TestPerfParams> {}; + class Split3PerfTest : public TestPerfParams> {}; + class Split4PerfTest : public TestPerfParams> {}; + class Merge3PerfTest : public TestPerfParams> {}; + class Merge4PerfTest : public TestPerfParams> {}; + class RemapPerfTest : public TestPerfParams> {}; + class FlipPerfTest : public TestPerfParams> {}; + class CropPerfTest : public TestPerfParams> {}; + class CopyPerfTest : public TestPerfParams> {}; + class ConcatHorPerfTest : public TestPerfParams> {}; + class ConcatHorVecPerfTest : public TestPerfParams> {}; + class ConcatVertPerfTest : public TestPerfParams> {}; + class ConcatVertVecPerfTest : public TestPerfParams> {}; + class LUTPerfTest : public TestPerfParams> {}; class ConvertToPerfTest : public TestPerfParams> {}; - class KMeansNDPerfTest : public TestPerfParams> {}; - class KMeans2DPerfTest : public TestPerfParams> {}; - class KMeans3DPerfTest : public TestPerfParams> {}; + class KMeansNDPerfTest : public TestPerfParams> {}; + class KMeans2DPerfTest : public TestPerfParams> {}; + class KMeans3DPerfTest : public TestPerfParams> {}; class TransposePerfTest : public TestPerfParams> {}; class ResizePerfTest : public TestPerfParams> {}; class BottleneckKernelsConstInputPerfTest : public TestPerfParams> {}; class ResizeFxFyPerfTest : public TestPerfParams> {}; - class ResizeInSimpleGraphPerfTest : public TestPerfParams> {}; + class ResizeInSimpleGraphPerfTest : public TestPerfParams> {}; class ParseSSDBLPerfTest : public TestPerfParams>, public ParserSSDTest {}; class ParseSSDPerfTest : public TestPerfParams>, public ParserSSDTest {}; class ParseYoloPerfTest : public TestPerfParams>, public ParserYoloTest {}; diff --git a/modules/gapi/perf/common/gapi_core_perf_tests_inl.hpp b/modules/gapi/perf/common/gapi_core_perf_tests_inl.hpp index 96ce369081..d5a8d95f46 100644 --- a/modules/gapi/perf/common/gapi_core_perf_tests_inl.hpp +++ b/modules/gapi/perf/common/gapi_core_perf_tests_inl.hpp @@ -20,12 +20,91 @@ using namespace perf; //------------------------------------------------------------------------------ +PERF_TEST_P_(PhasePerfTest, TestPerformance) +{ + compare_f cmpF; + cv::Size sz; + MatType type = -1; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, compile_args) = GetParam(); + + initMatsRandU(type, sz, type, false); + + // OpenCV code /////////////////////////////////////////////////////////// + + cv::phase(in_mat1, in_mat2, out_mat_ocv); + + // G-API code //////////////////////////////////////////////////////////// + cv::GMat in1, in2; + auto out = cv::gapi::phase(in1, in2); + cv::GComputation c(cv::GIn(in1, in2), cv::GOut(out)); + + // Warm-up graph engine: + auto cc = c.compile(descr_of(gin(in_mat1, in_mat2)), + std::move(compile_args)); + cc(gin(in_mat1, in_mat2), gout(out_mat_gapi)); + + TEST_CYCLE() + { + cc(gin(in_mat1, in_mat2), gout(out_mat_gapi)); + } + + // Comparison //////////////////////////////////////////////////////////// + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } + + SANITY_CHECK_NOTHING(); +} + +//------------------------------------------------------------------------------ + +PERF_TEST_P_(SqrtPerfTest, TestPerformance) +{ + compare_f cmpF; + cv::Size sz; + MatType type = -1; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, compile_args) = GetParam(); + + initMatrixRandU(type, sz, false); + + // OpenCV code /////////////////////////////////////////////////////////// + cv::sqrt(in_mat1, out_mat_ocv); + + // G-API code //////////////////////////////////////////////////////////// + cv::GMat in; + auto out = cv::gapi::sqrt(in); + cv::GComputation c(cv::GIn(in), cv::GOut(out)); + + // Warm-up graph engine: + auto cc = c.compile(descr_of(gin(in_mat1)), + std::move(compile_args)); + cc(gin(in_mat1), gout(out_mat_gapi)); + + TEST_CYCLE() + { + cc(gin(in_mat1), gout(out_mat_gapi)); + } + + // Comparison //////////////////////////////////////////////////////////// + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } + + SANITY_CHECK_NOTHING(); +} + +//------------------------------------------------------------------------------ + PERF_TEST_P_(AddPerfTest, TestPerformance) { - Size sz = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - int dtype = get<2>(GetParam()); - cv::GCompileArgs compile_args = get<3>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + int dtype = -1; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, dtype, compile_args) = GetParam(); initMatsRandU(type, sz, dtype, false); @@ -51,8 +130,9 @@ PERF_TEST_P_(AddPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - // FIXIT unrealiable check: EXPECT_EQ(0, cv::countNonZero(out_mat_gapi != out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -102,10 +182,12 @@ PERF_TEST_P_(AddCPerfTest, TestPerformance) PERF_TEST_P_(SubPerfTest, TestPerformance) { - Size sz = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - int dtype = get<2>(GetParam()); - cv::GCompileArgs compile_args = get<3>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + int dtype = -1; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, dtype, compile_args) = GetParam(); initMatsRandU(type, sz, dtype, false); @@ -128,8 +210,9 @@ PERF_TEST_P_(SubPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - // FIXIT unrealiable check: EXPECT_EQ(0, cv::countNonZero(out_mat_gapi != out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -179,10 +262,12 @@ PERF_TEST_P_(SubCPerfTest, TestPerformance) PERF_TEST_P_(SubRCPerfTest, TestPerformance) { - Size sz = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - int dtype = get<2>(GetParam()); - cv::GCompileArgs compile_args = get<3>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + int dtype = -1; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, dtype, compile_args) = GetParam(); initMatsRandU(type, sz, dtype, false); @@ -206,8 +291,9 @@ PERF_TEST_P_(SubRCPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - // FIXIT unrealiable check: EXPECT_EQ(0, cv::countNonZero(out_mat_gapi != out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -222,7 +308,6 @@ PERF_TEST_P_(MulPerfTest, TestPerformance) int dtype = -1; double scale = 1.0; cv::GCompileArgs compile_args; - std::tie(cmpF, sz, type, dtype, scale, compile_args) = GetParam(); initMatsRandU(type, sz, dtype, false); @@ -263,7 +348,6 @@ PERF_TEST_P_(MulDoublePerfTest, TestPerformance) int dtype = -1; double scale = 1.0; cv::GCompileArgs compile_args; - std::tie(cmpF, sz, type, dtype, compile_args) = GetParam(); auto& rng = cv::theRNG(); @@ -306,7 +390,6 @@ PERF_TEST_P_(MulCPerfTest, TestPerformance) int dtype = -1; double scale = 1.0; cv::GCompileArgs compile_args; - std::tie(cmpF, sz, type, dtype, compile_args) = GetParam(); initMatsRandU(type, sz, dtype, false); @@ -342,12 +425,13 @@ PERF_TEST_P_(MulCPerfTest, TestPerformance) PERF_TEST_P_(DivPerfTest, TestPerformance) { - compare_f cmpF = get<0>(GetParam()); - Size sz = get<1>(GetParam()); - MatType type = get<2>(GetParam()); - int dtype = get<3>(GetParam()); - double scale = get<4>(GetParam()); - cv::GCompileArgs compile_args = get<5>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + int dtype = -1; + double scale = 1.0; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, dtype, scale, compile_args) = GetParam(); // FIXIT Unstable input data for divide initMatsRandU(type, sz, dtype, false); @@ -387,21 +471,24 @@ PERF_TEST_P_(DivPerfTest, TestPerformance) PERF_TEST_P_(DivCPerfTest, TestPerformance) { - Size sz = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - int dtype = get<2>(GetParam()); - cv::GCompileArgs compile_args = get<3>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + int dtype = -1; + double scale = 1.0; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, dtype, scale, compile_args) = GetParam(); // FIXIT Unstable input data for divide initMatsRandU(type, sz, dtype, false); // OpenCV code /////////////////////////////////////////////////////////// - cv::divide(in_mat1, sc, out_mat_ocv, 1.0, dtype); + cv::divide(in_mat1, sc, out_mat_ocv, scale, dtype); // G-API code //////////////////////////////////////////////////////////// cv::GMat in1, out; cv::GScalar sc1; - out = cv::gapi::divC(in1, sc1, 1.0, dtype); + out = cv::gapi::divC(in1, sc1, scale, dtype); cv::GComputation c(GIn(in1, sc1), GOut(out)); // Warm-up graph engine: @@ -415,8 +502,9 @@ PERF_TEST_P_(DivCPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - // FIXIT unrealiable check: EXPECT_EQ(0, cv::countNonZero(out_mat_gapi != out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -425,22 +513,24 @@ PERF_TEST_P_(DivCPerfTest, TestPerformance) PERF_TEST_P_(DivRCPerfTest, TestPerformance) { - compare_f cmpF = get<0>(GetParam()); - Size sz = get<1>(GetParam()); - MatType type = get<2>(GetParam()); - int dtype = get<3>(GetParam()); - cv::GCompileArgs compile_args = get<4>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + int dtype = -1; + double scale = 1.0; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, dtype, scale, compile_args) = GetParam(); // FIXIT Unstable input data for divide initMatsRandU(type, sz, dtype, false); // OpenCV code /////////////////////////////////////////////////////////// - cv::divide(sc, in_mat1, out_mat_ocv, 1.0, dtype); + cv::divide(sc, in_mat1, out_mat_ocv, scale, dtype); // G-API code //////////////////////////////////////////////////////////// cv::GMat in1, out; cv::GScalar sc1; - out = cv::gapi::divRC(sc1, in1, 1.0, dtype); + out = cv::gapi::divRC(sc1, in1, scale, dtype); cv::GComputation c(GIn(in1, sc1), GOut(out)); // Warm-up graph engine: @@ -454,8 +544,9 @@ PERF_TEST_P_(DivRCPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - // FIXIT unrealiable check: EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -464,12 +555,14 @@ PERF_TEST_P_(DivRCPerfTest, TestPerformance) PERF_TEST_P_(MaskPerfTest, TestPerformance) { - Size sz_in = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - cv::GCompileArgs compile_args = get<2>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, compile_args) = GetParam(); - initMatrixRandU(type, sz_in, type, false); - in_mat2 = cv::Mat(sz_in, CV_8UC1); + initMatrixRandU(type, sz, type, false); + in_mat2 = cv::Mat(sz, CV_8UC1); cv::randu(in_mat2, cv::Scalar::all(0), cv::Scalar::all(255)); in_mat2 = in_mat2 > 128; @@ -493,7 +586,9 @@ PERF_TEST_P_(MaskPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - EXPECT_EQ(0, cv::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -502,11 +597,13 @@ PERF_TEST_P_(MaskPerfTest, TestPerformance) PERF_TEST_P_(MeanPerfTest, TestPerformance) { - Size sz_in = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - cv::GCompileArgs compile_args = get<2>(GetParam()); + compare_scalar_f cmpF; + cv::Size sz; + MatType type = -1; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, compile_args) = GetParam(); - initMatrixRandU(type, sz_in, false); + initMatrixRandU(type, sz, false); cv::Scalar out_norm; cv::Scalar out_norm_ocv; @@ -529,7 +626,9 @@ PERF_TEST_P_(MeanPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - // FIXIT unrealiable check: EXPECT_EQ(out_norm[0], out_norm_ocv[0]); + { + EXPECT_TRUE(cmpF(out_norm, out_norm_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -538,11 +637,12 @@ PERF_TEST_P_(MeanPerfTest, TestPerformance) PERF_TEST_P_(Polar2CartPerfTest, TestPerformance) { - compare_f cmpF = get<0>(GetParam()); - Size sz_in = get<1>(GetParam()); - cv::GCompileArgs compile_args = get<2>(GetParam()); + compare_f cmpF; + cv::Size sz; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, compile_args) = GetParam(); - initMatsRandU(CV_32FC1, sz_in, CV_32FC1, false); + initMatsRandU(CV_32FC1, sz, CV_32FC1, false); cv::Mat out_mat2; cv::Mat out_mat_ocv2; @@ -564,9 +664,10 @@ PERF_TEST_P_(Polar2CartPerfTest, TestPerformance) cc(gin(in_mat1, in_mat2), gout(out_mat_gapi, out_mat2)); } // Comparison //////////////////////////////////////////////////////////// - EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_TRUE(cmpF(out_mat_ocv2, out_mat2)); - EXPECT_EQ(out_mat_gapi.size(), sz_in); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + EXPECT_TRUE(cmpF(out_mat2, out_mat_ocv2)); + } SANITY_CHECK_NOTHING(); } @@ -575,13 +676,14 @@ PERF_TEST_P_(Polar2CartPerfTest, TestPerformance) PERF_TEST_P_(Cart2PolarPerfTest, TestPerformance) { - compare_f cmpF = get<0>(GetParam()); - Size sz_in = get<1>(GetParam()); - cv::GCompileArgs compile_args = get<2>(GetParam()); + compare_f cmpF; + cv::Size sz; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, compile_args) = GetParam(); - initMatsRandU(CV_32FC1, sz_in, CV_32FC1, false); - cv::Mat out_mat2(sz_in, CV_32FC1); - cv::Mat out_mat_ocv2(sz_in, CV_32FC1); + initMatsRandU(CV_32FC1, sz, CV_32FC1, false); + cv::Mat out_mat2(sz, CV_32FC1); + cv::Mat out_mat_ocv2(sz, CV_32FC1); // OpenCV code /////////////////////////////////////////////////////////// cv::cartToPolar(in_mat1, in_mat2, out_mat_ocv, out_mat_ocv2); @@ -602,9 +704,10 @@ PERF_TEST_P_(Cart2PolarPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_TRUE(cmpF(out_mat_ocv2, out_mat2)); - EXPECT_EQ(out_mat_gapi.size(), sz_in); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + EXPECT_TRUE(cmpF(out_mat2, out_mat_ocv2)); + } SANITY_CHECK_NOTHING(); } @@ -613,10 +716,12 @@ PERF_TEST_P_(Cart2PolarPerfTest, TestPerformance) PERF_TEST_P_(CmpPerfTest, TestPerformance) { - CmpTypes opType = get<0>(GetParam()); - cv::Size sz = get<1>(GetParam()); - MatType type = get<2>(GetParam()); - cv::GCompileArgs compile_args = get<3>(GetParam()); + compare_f cmpF; + CmpTypes opType = CMP_EQ; + cv::Size sz; + MatType type = -1; + cv::GCompileArgs compile_args; + std::tie(cmpF, opType, sz, type, compile_args) = GetParam(); initMatsRandU(type, sz, CV_8U, false); @@ -648,8 +753,9 @@ PERF_TEST_P_(CmpPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - EXPECT_EQ(0, cvtest::norm(out_mat_gapi, out_mat_ocv, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -658,10 +764,10 @@ PERF_TEST_P_(CmpPerfTest, TestPerformance) PERF_TEST_P_(CmpWithScalarPerfTest, TestPerformance) { - MatType type = -1; + compare_f cmpF; CmpTypes opType = CMP_EQ; cv::Size sz; - compare_f cmpF; + MatType type = -1; cv::GCompileArgs compile_args; std::tie(cmpF, opType, sz, type, compile_args) = GetParam(); @@ -696,8 +802,9 @@ PERF_TEST_P_(CmpWithScalarPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - EXPECT_EQ(out_mat_gapi.size(), sz); - EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -706,13 +813,13 @@ PERF_TEST_P_(CmpWithScalarPerfTest, TestPerformance) PERF_TEST_P_(BitwisePerfTest, TestPerformance) { - MatType type = -1; - bitwiseOp opType = AND; - bool testWithScalar = false; + compare_f cmpF; + bitwiseOp opType = AND; + bool testWithScalar = false; cv::Size sz; + MatType type = -1; cv::GCompileArgs compile_args; - - std::tie(opType, testWithScalar, sz, type, compile_args) = GetParam(); + std::tie(cmpF, opType, testWithScalar, sz, type, compile_args) = GetParam(); initMatsRandU(type, sz, type, false); @@ -779,8 +886,9 @@ PERF_TEST_P_(BitwisePerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - EXPECT_EQ(0, cv::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -789,11 +897,13 @@ PERF_TEST_P_(BitwisePerfTest, TestPerformance) PERF_TEST_P_(BitwiseNotPerfTest, TestPerformance) { - cv::Size sz_in = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - cv::GCompileArgs compile_args = get<2>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, compile_args) = GetParam(); - initMatrixRandU(type, sz_in, type, false); + initMatrixRandU(type, sz, type, false); // OpenCV code /////////////////////////////////////////////////////////// cv::bitwise_not(in_mat1, out_mat_ocv); @@ -814,8 +924,9 @@ PERF_TEST_P_(BitwiseNotPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - EXPECT_EQ(0, cv::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz_in); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -824,12 +935,14 @@ PERF_TEST_P_(BitwiseNotPerfTest, TestPerformance) PERF_TEST_P_(SelectPerfTest, TestPerformance) { - cv::Size sz_in = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - cv::GCompileArgs compile_args = get<2>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, compile_args) = GetParam(); - initMatsRandU(type, sz_in, type, false); - cv::Mat in_mask(sz_in, CV_8UC1); + initMatsRandU(type, sz, type, false); + cv::Mat in_mask(sz, CV_8UC1); cv::randu(in_mask, cv::Scalar::all(0), cv::Scalar::all(255)); // OpenCV code /////////////////////////////////////////////////////////// @@ -852,8 +965,9 @@ PERF_TEST_P_(SelectPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - EXPECT_EQ(0, cv::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz_in); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -862,12 +976,13 @@ PERF_TEST_P_(SelectPerfTest, TestPerformance) PERF_TEST_P_(MinPerfTest, TestPerformance) { - cv::Size sz_in = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - cv::GCompileArgs compile_args = get<2>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, compile_args) = GetParam(); - - initMatsRandU(type, sz_in, type, false); + initMatsRandU(type, sz, type, false); // OpenCV code /////////////////////////////////////////////////////////// cv::min(in_mat1, in_mat2, out_mat_ocv); @@ -888,8 +1003,9 @@ PERF_TEST_P_(MinPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - EXPECT_EQ(0, cv::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz_in); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -898,12 +1014,13 @@ PERF_TEST_P_(MinPerfTest, TestPerformance) PERF_TEST_P_(MaxPerfTest, TestPerformance) { - cv::Size sz_in = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - cv::GCompileArgs compile_args = get<2>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, compile_args) = GetParam(); - - initMatsRandU(type, sz_in, type, false); + initMatsRandU(type, sz, type, false); // OpenCV code /////////////////////////////////////////////////////////// cv::max(in_mat1, in_mat2, out_mat_ocv); @@ -924,8 +1041,9 @@ PERF_TEST_P_(MaxPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - EXPECT_EQ(0, cv::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz_in); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -934,12 +1052,13 @@ PERF_TEST_P_(MaxPerfTest, TestPerformance) PERF_TEST_P_(AbsDiffPerfTest, TestPerformance) { - cv::Size sz_in = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - cv::GCompileArgs compile_args = get<2>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, compile_args) = GetParam(); - - initMatsRandU(type, sz_in, type, false); + initMatsRandU(type, sz, type, false); // OpenCV code /////////////////////////////////////////////////////////// cv::absdiff(in_mat1, in_mat2, out_mat_ocv); @@ -960,8 +1079,9 @@ PERF_TEST_P_(AbsDiffPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - // FIXIT unrealiable check: EXPECT_EQ(0, cv::countNonZero(out_mat_gapi != out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz_in); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -970,13 +1090,13 @@ PERF_TEST_P_(AbsDiffPerfTest, TestPerformance) PERF_TEST_P_(AbsDiffCPerfTest, TestPerformance) { - compare_f cmpF = get<0>(GetParam()); - cv::Size sz_in = get<1>(GetParam()); - MatType type = get<2>(GetParam()); - cv::GCompileArgs compile_args = get<3>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, compile_args) = GetParam(); - - initMatsRandU(type, sz_in, type, false); + initMatsRandU(type, sz, type, false); // OpenCV code /////////////////////////////////////////////////////////// cv::absdiff(in_mat1, sc, out_mat_ocv); @@ -1009,13 +1129,13 @@ PERF_TEST_P_(AbsDiffCPerfTest, TestPerformance) PERF_TEST_P_(SumPerfTest, TestPerformance) { - compare_scalar_f cmpF = get<0>(GetParam()); - cv::Size sz_in = get<1>(GetParam()); - MatType type = get<2>(GetParam()); - cv::GCompileArgs compile_args = get<3>(GetParam()); + compare_scalar_f cmpF; + cv::Size sz; + MatType type = -1; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, compile_args) = GetParam(); - - initMatrixRandU(type, sz_in, type, false); + initMatrixRandU(type, sz, type, false); cv::Scalar out_sum; cv::Scalar out_sum_ocv; @@ -1051,12 +1171,12 @@ PERF_TEST_P_(SumPerfTest, TestPerformance) PERF_TEST_P_(CountNonZeroPerfTest, TestPerformance) { compare_scalar_f cmpF; - cv::Size sz_in; + cv::Size sz; MatType type = -1; cv::GCompileArgs compile_args; - std::tie(cmpF, sz_in, type, compile_args) = GetParam(); + std::tie(cmpF, sz, type, compile_args) = GetParam(); - initMatrixRandU(type, sz_in, type, false); + initMatrixRandU(type, sz, type, false); int out_cnz_gapi, out_cnz_ocv; // OpenCV code /////////////////////////////////////////////////////////// @@ -1087,17 +1207,18 @@ PERF_TEST_P_(CountNonZeroPerfTest, TestPerformance) PERF_TEST_P_(AddWeightedPerfTest, TestPerformance) { - compare_f cmpF = get<0>(GetParam()); - cv::Size sz_in = get<1>(GetParam()); - MatType type = get<2>(GetParam()); - int dtype = get<3>(GetParam()); - cv::GCompileArgs compile_args = get<4>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + int dtype = -1; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, dtype, compile_args) = GetParam(); auto& rng = cv::theRNG(); double alpha = rng.uniform(0.0, 1.0); double beta = rng.uniform(0.0, 1.0); double gamma = rng.uniform(0.0, 1.0); - initMatsRandU(type, sz_in, dtype, false); + initMatsRandU(type, sz, dtype, false); // OpenCV code /////////////////////////////////////////////////////////// cv::addWeighted(in_mat1, alpha, in_mat2, beta, gamma, out_mat_ocv, dtype); @@ -1118,9 +1239,9 @@ PERF_TEST_P_(AddWeightedPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz_in); - + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -1129,12 +1250,12 @@ PERF_TEST_P_(AddWeightedPerfTest, TestPerformance) PERF_TEST_P_(NormPerfTest, TestPerformance) { - compare_scalar_f cmpF = get<0>(GetParam()); - NormTypes opType = get<1>(GetParam()); - cv::Size sz = get<2>(GetParam()); - MatType type = get<3>(GetParam()); - cv::GCompileArgs compile_args = get<4>(GetParam()); - + compare_scalar_f cmpF; + NormTypes opType = NORM_INF; + cv::Size sz; + MatType type = -1; + cv::GCompileArgs compile_args; + std::tie(cmpF, opType, sz, type, compile_args) = GetParam(); initMatrixRandU(type, sz, type, false); cv::Scalar out_norm; @@ -1177,18 +1298,18 @@ PERF_TEST_P_(NormPerfTest, TestPerformance) PERF_TEST_P_(IntegralPerfTest, TestPerformance) { - cv::Size sz_in = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - cv::GCompileArgs compile_args = get<2>(GetParam()); - + compare_f cmpF; + cv::Size sz; + MatType type = -1; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, compile_args) = GetParam(); MatType type_out = (type == CV_8U) ? CV_32SC1 : CV_64FC1; - - in_mat1 = cv::Mat(sz_in, type); + in_mat1 = cv::Mat(sz, type); cv::randu(in_mat1, cv::Scalar::all(0), cv::Scalar::all(255)); - cv::Size sz_out = cv::Size(sz_in.width + 1, sz_in.height + 1); + cv::Size sz_out = cv::Size(sz.width + 1, sz.height + 1); cv::Mat out_mat1(sz_out, type_out); cv::Mat out_mat_ocv1(sz_out, type_out); @@ -1214,8 +1335,10 @@ PERF_TEST_P_(IntegralPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - // FIXIT unrealiable check: EXPECT_EQ(0, cv::countNonZero(out_mat_ocv1 != out_mat1)); - // FIXIT unrealiable check: EXPECT_EQ(0, cv::countNonZero(out_mat_ocv2 != out_mat2)); + { + EXPECT_TRUE(cmpF(out_mat1, out_mat_ocv1)); + EXPECT_TRUE(cmpF(out_mat2, out_mat_ocv2)); + } SANITY_CHECK_NOTHING(); } @@ -1224,14 +1347,16 @@ PERF_TEST_P_(IntegralPerfTest, TestPerformance) PERF_TEST_P_(ThresholdPerfTest, TestPerformance) { - cv::Size sz_in = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - int tt = get<2>(GetParam()); - cv::GCompileArgs compile_args = get<3>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + int tt = 0; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, tt, compile_args) = GetParam(); cv::Scalar thr = initScalarRandU(50); cv::Scalar maxval = initScalarRandU(50) + cv::Scalar(50, 50, 50, 50); - initMatrixRandU(type, sz_in, type, false); + initMatrixRandU(type, sz, type, false); cv::Scalar out_scalar; // OpenCV code /////////////////////////////////////////////////////////// @@ -1254,8 +1379,9 @@ PERF_TEST_P_(ThresholdPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - EXPECT_EQ(0, cv::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz_in); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -1264,13 +1390,15 @@ PERF_TEST_P_(ThresholdPerfTest, TestPerformance) PERF_TEST_P_(ThresholdOTPerfTest, TestPerformance) { - cv::Size sz_in = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - int tt = get<2>(GetParam()); - cv::GCompileArgs compile_args = get<3>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + int tt = 0; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, tt, compile_args) = GetParam(); cv::Scalar maxval = initScalarRandU(50) + cv::Scalar(50, 50, 50, 50); - initMatrixRandU(type, sz_in, type, false); + initMatrixRandU(type, sz, type, false); cv::Scalar out_gapi_scalar; double ocv_res; @@ -1294,9 +1422,10 @@ PERF_TEST_P_(ThresholdOTPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - // FIXIT unrealiable check: EXPECT_EQ(0, cv::countNonZero(out_mat_ocv != out_mat_gapi)); - EXPECT_EQ(out_mat_gapi.size(), sz_in); - EXPECT_EQ(ocv_res, out_gapi_scalar.val[0]); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + EXPECT_EQ(ocv_res, out_gapi_scalar.val[0]); + } SANITY_CHECK_NOTHING(); } @@ -1305,13 +1434,15 @@ PERF_TEST_P_(ThresholdOTPerfTest, TestPerformance) PERF_TEST_P_(InRangePerfTest, TestPerformance) { - cv::Size sz_in = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - cv::GCompileArgs compile_args = get<2>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, compile_args) = GetParam(); cv::Scalar thrLow = initScalarRandU(100); cv::Scalar thrUp = initScalarRandU(100) + cv::Scalar(100, 100, 100, 100); - initMatrixRandU(type, sz_in, type, false); + initMatrixRandU(type, sz, type, false); // OpenCV code /////////////////////////////////////////////////////////// cv::inRange(in_mat1, thrLow, thrUp, out_mat_ocv); @@ -1333,8 +1464,9 @@ PERF_TEST_P_(InRangePerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - EXPECT_EQ(0, cv::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz_in); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -1343,15 +1475,16 @@ PERF_TEST_P_(InRangePerfTest, TestPerformance) PERF_TEST_P_(Split3PerfTest, TestPerformance) { - Size sz_in = get<0>(GetParam()); - cv::GCompileArgs compile_args = get<1>(GetParam()); + compare_f cmpF; + cv::Size sz; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, compile_args) = GetParam(); - - initMatrixRandU(CV_8UC3, sz_in, CV_8UC1); - cv::Mat out_mat2 = cv::Mat(sz_in, CV_8UC1); - cv::Mat out_mat3 = cv::Mat(sz_in, CV_8UC1); - cv::Mat out_mat_ocv2 = cv::Mat(sz_in, CV_8UC1); - cv::Mat out_mat_ocv3 = cv::Mat(sz_in, CV_8UC1); + initMatrixRandU(CV_8UC3, sz, CV_8UC1); + cv::Mat out_mat2 = cv::Mat(sz, CV_8UC1); + cv::Mat out_mat3 = cv::Mat(sz, CV_8UC1); + cv::Mat out_mat_ocv2 = cv::Mat(sz, CV_8UC1); + cv::Mat out_mat_ocv3 = cv::Mat(sz, CV_8UC1); // OpenCV code /////////////////////////////////////////////////////////// std::vector out_mats_ocv = { out_mat_ocv, out_mat_ocv2, out_mat_ocv3 }; @@ -1373,9 +1506,11 @@ PERF_TEST_P_(Split3PerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - EXPECT_EQ(0, cv::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); - EXPECT_EQ(0, cv::norm(out_mat_ocv2, out_mat2, NORM_INF)); - EXPECT_EQ(0, cv::norm(out_mat_ocv3, out_mat3, NORM_INF)); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + EXPECT_TRUE(cmpF(out_mat2, out_mat_ocv2)); + EXPECT_TRUE(cmpF(out_mat3, out_mat_ocv3)); + } SANITY_CHECK_NOTHING(); } @@ -1384,16 +1519,18 @@ PERF_TEST_P_(Split3PerfTest, TestPerformance) PERF_TEST_P_(Split4PerfTest, TestPerformance) { - Size sz_in = get<0>(GetParam()); - cv::GCompileArgs compile_args = get<1>(GetParam()); + compare_f cmpF; + cv::Size sz; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, compile_args) = GetParam(); - initMatrixRandU(CV_8UC4, sz_in, CV_8UC1); - cv::Mat out_mat2 = cv::Mat(sz_in, CV_8UC1); - cv::Mat out_mat3 = cv::Mat(sz_in, CV_8UC1); - cv::Mat out_mat4 = cv::Mat(sz_in, CV_8UC1); - cv::Mat out_mat_ocv2 = cv::Mat(sz_in, CV_8UC1); - cv::Mat out_mat_ocv3 = cv::Mat(sz_in, CV_8UC1); - cv::Mat out_mat_ocv4 = cv::Mat(sz_in, CV_8UC1); + initMatrixRandU(CV_8UC4, sz, CV_8UC1); + cv::Mat out_mat2 = cv::Mat(sz, CV_8UC1); + cv::Mat out_mat3 = cv::Mat(sz, CV_8UC1); + cv::Mat out_mat4 = cv::Mat(sz, CV_8UC1); + cv::Mat out_mat_ocv2 = cv::Mat(sz, CV_8UC1); + cv::Mat out_mat_ocv3 = cv::Mat(sz, CV_8UC1); + cv::Mat out_mat_ocv4 = cv::Mat(sz, CV_8UC1); // OpenCV code /////////////////////////////////////////////////////////// std::vector out_mats_ocv = { out_mat_ocv, out_mat_ocv2, out_mat_ocv3, out_mat_ocv4 }; @@ -1415,10 +1552,12 @@ PERF_TEST_P_(Split4PerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - EXPECT_EQ(0, cv::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); - EXPECT_EQ(0, cv::norm(out_mat_ocv2, out_mat2, NORM_INF)); - EXPECT_EQ(0, cv::norm(out_mat_ocv3, out_mat3, NORM_INF)); - EXPECT_EQ(0, cv::norm(out_mat_ocv4, out_mat4, NORM_INF)); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + EXPECT_TRUE(cmpF(out_mat2, out_mat_ocv2)); + EXPECT_TRUE(cmpF(out_mat3, out_mat_ocv3)); + EXPECT_TRUE(cmpF(out_mat4, out_mat_ocv4)); + } SANITY_CHECK_NOTHING(); } @@ -1427,11 +1566,13 @@ PERF_TEST_P_(Split4PerfTest, TestPerformance) PERF_TEST_P_(Merge3PerfTest, TestPerformance) { - Size sz_in = get<0>(GetParam()); - cv::GCompileArgs compile_args = get<1>(GetParam()); + compare_f cmpF; + cv::Size sz; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, compile_args) = GetParam(); - initMatsRandU(CV_8UC1, sz_in, CV_8UC3); - cv::Mat in_mat3(sz_in, CV_8UC1); + initMatsRandU(CV_8UC1, sz, CV_8UC3); + cv::Mat in_mat3(sz, CV_8UC1); cv::Scalar mean = cv::Scalar::all(127); cv::Scalar stddev = cv::Scalar::all(40.f); cv::randn(in_mat3, mean, stddev); @@ -1456,7 +1597,9 @@ PERF_TEST_P_(Merge3PerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - EXPECT_EQ(0, cv::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -1465,12 +1608,14 @@ PERF_TEST_P_(Merge3PerfTest, TestPerformance) PERF_TEST_P_(Merge4PerfTest, TestPerformance) { - Size sz_in = get<0>(GetParam()); - cv::GCompileArgs compile_args = get<1>(GetParam()); + compare_f cmpF; + cv::Size sz; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, compile_args) = GetParam(); - initMatsRandU(CV_8UC1, sz_in, CV_8UC3); - cv::Mat in_mat3(sz_in, CV_8UC1); - cv::Mat in_mat4(sz_in, CV_8UC1); + initMatsRandU(CV_8UC1, sz, CV_8UC3); + cv::Mat in_mat3(sz, CV_8UC1); + cv::Mat in_mat4(sz, CV_8UC1); cv::Scalar mean = cv::Scalar::all(127); cv::Scalar stddev = cv::Scalar::all(40.f); cv::randn(in_mat3, mean, stddev); @@ -1496,7 +1641,9 @@ PERF_TEST_P_(Merge4PerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - EXPECT_EQ(0, cv::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -1505,12 +1652,14 @@ PERF_TEST_P_(Merge4PerfTest, TestPerformance) PERF_TEST_P_(RemapPerfTest, TestPerformance) { - cv::Size sz_in = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - cv::GCompileArgs compile_args = get<2>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, compile_args) = GetParam(); - initMatrixRandU(type, sz_in, type, false); - cv::Mat in_map1(sz_in, CV_16SC2); + initMatrixRandU(type, sz, type, false); + cv::Mat in_map1(sz, CV_16SC2); cv::Mat in_map2 = cv::Mat(); cv::randu(in_map1, cv::Scalar::all(0), cv::Scalar::all(255)); cv::Scalar bv = cv::Scalar(); @@ -1534,8 +1683,9 @@ PERF_TEST_P_(RemapPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - // FIXIT unrealiable check: EXPECT_EQ(0, cv::countNonZero(out_mat_ocv != out_mat_gapi)); - EXPECT_EQ(out_mat_gapi.size(), sz_in); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -1544,12 +1694,14 @@ PERF_TEST_P_(RemapPerfTest, TestPerformance) PERF_TEST_P_(FlipPerfTest, TestPerformance) { - cv::Size sz_in = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - int flipCode = get<2>(GetParam()); - cv::GCompileArgs compile_args = get<3>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + int flipCode = -1; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, flipCode, compile_args) = GetParam(); - initMatrixRandU(type, sz_in, type, false); + initMatrixRandU(type, sz, type, false); // OpenCV code /////////////////////////////////////////////////////////// cv::flip(in_mat1, out_mat_ocv, flipCode); @@ -1570,8 +1722,9 @@ PERF_TEST_P_(FlipPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - EXPECT_EQ(0, cv::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz_in); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -1580,13 +1733,14 @@ PERF_TEST_P_(FlipPerfTest, TestPerformance) PERF_TEST_P_(CropPerfTest, TestPerformance) { - cv::Size sz_in = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - cv::Rect rect_to = get<2>(GetParam()); - cv::GCompileArgs compile_args = get<3>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + cv::Rect rect_to; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, rect_to, compile_args) = GetParam(); - initMatrixRandU(type, sz_in, type, false); - cv::Size sz_out = cv::Size(rect_to.width, rect_to.height); + initMatrixRandU(type, sz, type, false); // OpenCV code /////////////////////////////////////////////////////////// cv::Mat(in_mat1, rect_to).copyTo(out_mat_ocv); @@ -1607,8 +1761,9 @@ PERF_TEST_P_(CropPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - EXPECT_EQ(0, cv::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz_out); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -1617,12 +1772,13 @@ PERF_TEST_P_(CropPerfTest, TestPerformance) PERF_TEST_P_(CopyPerfTest, TestPerformance) { - cv::Size sz_in = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - cv::GCompileArgs compile_args = get<2>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, compile_args) = GetParam(); - initMatrixRandU(type, sz_in, type, false); - cv::Size sz_out = sz_in; + initMatrixRandU(type, sz, type, false); // OpenCV code /////////////////////////////////////////////////////////// cv::Mat(in_mat1).copyTo(out_mat_ocv); @@ -1643,8 +1799,9 @@ PERF_TEST_P_(CopyPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - EXPECT_EQ(0, cv::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz_out); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -1653,14 +1810,16 @@ PERF_TEST_P_(CopyPerfTest, TestPerformance) PERF_TEST_P_(ConcatHorPerfTest, TestPerformance) { - cv::Size sz_out = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - cv::GCompileArgs compile_args = get<2>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, compile_args) = GetParam(); - int wpart = sz_out.width / 4; + int wpart = sz.width / 4; - cv::Size sz_in1 = cv::Size(wpart, sz_out.height); - cv::Size sz_in2 = cv::Size(sz_out.width - wpart, sz_out.height); + cv::Size sz_in1 = cv::Size(wpart, sz.height); + cv::Size sz_in2 = cv::Size(sz.width - wpart, sz.height); in_mat1 = cv::Mat(sz_in1, type); in_mat2 = cv::Mat(sz_in2, type); @@ -1671,8 +1830,8 @@ PERF_TEST_P_(ConcatHorPerfTest, TestPerformance) cv::randn(in_mat1, mean, stddev); cv::randn(in_mat2, mean, stddev); - out_mat_gapi = cv::Mat(sz_out, type); - out_mat_ocv = cv::Mat(sz_out, type); + out_mat_gapi = cv::Mat(sz, type); + out_mat_ocv = cv::Mat(sz, type); // OpenCV code /////////////////////////////////////////////////////////// cv::hconcat(in_mat1, in_mat2, out_mat_ocv); @@ -1693,7 +1852,9 @@ PERF_TEST_P_(ConcatHorPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - EXPECT_EQ(0, cv::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -1702,16 +1863,18 @@ PERF_TEST_P_(ConcatHorPerfTest, TestPerformance) PERF_TEST_P_(ConcatHorVecPerfTest, TestPerformance) { - cv::Size sz_out = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - cv::GCompileArgs compile_args = get<2>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, compile_args) = GetParam(); - int wpart1 = sz_out.width / 3; - int wpart2 = sz_out.width / 2; + int wpart1 = sz.width / 3; + int wpart2 = sz.width / 2; - cv::Size sz_in1 = cv::Size(wpart1, sz_out.height); - cv::Size sz_in2 = cv::Size(wpart2, sz_out.height); - cv::Size sz_in3 = cv::Size(sz_out.width - wpart1 - wpart2, sz_out.height); + cv::Size sz_in1 = cv::Size(wpart1, sz.height); + cv::Size sz_in2 = cv::Size(wpart2, sz.height); + cv::Size sz_in3 = cv::Size(sz.width - wpart1 - wpart2, sz.height); in_mat1 = cv::Mat(sz_in1, type); in_mat2 = cv::Mat(sz_in2, type); @@ -1724,8 +1887,8 @@ PERF_TEST_P_(ConcatHorVecPerfTest, TestPerformance) cv::randn(in_mat2, mean, stddev); cv::randn(in_mat3, mean, stddev); - out_mat_gapi = cv::Mat(sz_out, type); - out_mat_ocv = cv::Mat(sz_out, type); + out_mat_gapi = cv::Mat(sz, type); + out_mat_ocv = cv::Mat(sz, type); std::vector cvmats = { in_mat1, in_mat2, in_mat3 }; @@ -1748,7 +1911,9 @@ PERF_TEST_P_(ConcatHorVecPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - EXPECT_EQ(0, cv::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -1757,14 +1922,16 @@ PERF_TEST_P_(ConcatHorVecPerfTest, TestPerformance) PERF_TEST_P_(ConcatVertPerfTest, TestPerformance) { - cv::Size sz_out = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - cv::GCompileArgs compile_args = get<2>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, compile_args) = GetParam(); - int hpart = sz_out.height * 2 / 3; + int hpart = sz.height * 2 / 3; - cv::Size sz_in1 = cv::Size(sz_out.width, hpart); - cv::Size sz_in2 = cv::Size(sz_out.width, sz_out.height - hpart); + cv::Size sz_in1 = cv::Size(sz.width, hpart); + cv::Size sz_in2 = cv::Size(sz.width, sz.height - hpart); in_mat1 = cv::Mat(sz_in1, type); in_mat2 = cv::Mat(sz_in2, type); @@ -1775,8 +1942,8 @@ PERF_TEST_P_(ConcatVertPerfTest, TestPerformance) cv::randn(in_mat1, mean, stddev); cv::randn(in_mat2, mean, stddev); - out_mat_gapi = cv::Mat(sz_out, type); - out_mat_ocv = cv::Mat(sz_out, type); + out_mat_gapi = cv::Mat(sz, type); + out_mat_ocv = cv::Mat(sz, type); // OpenCV code /////////////////////////////////////////////////////////// cv::vconcat(in_mat1, in_mat2, out_mat_ocv); @@ -1797,7 +1964,9 @@ PERF_TEST_P_(ConcatVertPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - EXPECT_EQ(0, cv::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -1806,16 +1975,18 @@ PERF_TEST_P_(ConcatVertPerfTest, TestPerformance) PERF_TEST_P_(ConcatVertVecPerfTest, TestPerformance) { - cv::Size sz_out = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - cv::GCompileArgs compile_args = get<2>(GetParam()); + compare_f cmpF; + cv::Size sz; + MatType type = -1; + cv::GCompileArgs compile_args; + std::tie(cmpF, sz, type, compile_args) = GetParam(); - int hpart1 = sz_out.height * 2 / 5; - int hpart2 = sz_out.height / 5; + int hpart1 = sz.height * 2 / 5; + int hpart2 = sz.height / 5; - cv::Size sz_in1 = cv::Size(sz_out.width, hpart1); - cv::Size sz_in2 = cv::Size(sz_out.width, hpart2); - cv::Size sz_in3 = cv::Size(sz_out.width, sz_out.height - hpart1 - hpart2); + cv::Size sz_in1 = cv::Size(sz.width, hpart1); + cv::Size sz_in2 = cv::Size(sz.width, hpart2); + cv::Size sz_in3 = cv::Size(sz.width, sz.height - hpart1 - hpart2); in_mat1 = cv::Mat(sz_in1, type); in_mat2 = cv::Mat(sz_in2, type); @@ -1828,8 +1999,8 @@ PERF_TEST_P_(ConcatVertVecPerfTest, TestPerformance) cv::randn(in_mat2, mean, stddev); cv::randn(in_mat3, mean, stddev); - out_mat_gapi = cv::Mat(sz_out, type); - out_mat_ocv = cv::Mat(sz_out, type); + out_mat_gapi = cv::Mat(sz, type); + out_mat_ocv = cv::Mat(sz, type); std::vector cvmats = { in_mat1, in_mat2, in_mat3 }; @@ -1852,7 +2023,9 @@ PERF_TEST_P_(ConcatVertVecPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - EXPECT_EQ(0, cv::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -1861,13 +2034,16 @@ PERF_TEST_P_(ConcatVertVecPerfTest, TestPerformance) PERF_TEST_P_(LUTPerfTest, TestPerformance) { - MatType type_mat = get<0>(GetParam()); - MatType type_lut = get<1>(GetParam()); - MatType type_out = CV_MAKETYPE(CV_MAT_DEPTH(type_lut), CV_MAT_CN(type_mat)); - cv::Size sz_in = get<2>(GetParam()); - cv::GCompileArgs compile_args = get<3>(GetParam()); + compare_f cmpF; + MatType type_mat = -1; + MatType type_lut = -1; + cv::Size sz; + cv::GCompileArgs compile_args; + std::tie(cmpF, type_mat, type_lut, sz, compile_args) = GetParam(); - initMatrixRandU(type_mat, sz_in, type_out); + MatType type_out = CV_MAKETYPE(CV_MAT_DEPTH(type_lut), CV_MAT_CN(type_mat)); + + initMatrixRandU(type_mat, sz, type_out); cv::Size sz_lut = cv::Size(1, 256); cv::Mat in_lut(sz_lut, type_lut); cv::randu(in_lut, cv::Scalar::all(0), cv::Scalar::all(255)); @@ -1891,8 +2067,9 @@ PERF_TEST_P_(LUTPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - EXPECT_EQ(0, cv::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz_in); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -1904,13 +2081,13 @@ PERF_TEST_P_(ConvertToPerfTest, TestPerformance) int depth_to = -1; MatType type_mat = -1; double alpha = 0., beta = 0.; - cv::Size sz_in; + cv::Size sz; compare_f cmpF; cv::GCompileArgs compile_args; - std::tie(cmpF, type_mat, depth_to, sz_in, alpha, beta, compile_args) = GetParam(); + std::tie(cmpF, type_mat, depth_to, sz, alpha, beta, compile_args) = GetParam(); MatType type_out = CV_MAKETYPE(depth_to, CV_MAT_CN(type_mat)); - initMatrixRandU(type_mat, sz_in, type_out); + initMatrixRandU(type_mat, sz, type_out); // OpenCV code /////////////////////////////////////////////////////////// in_mat1.convertTo(out_mat_ocv, depth_to, alpha, beta); @@ -1931,8 +2108,9 @@ PERF_TEST_P_(ConvertToPerfTest, TestPerformance) } // Comparison //////////////////////////////////////////////////////////// - EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz_in); + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + } SANITY_CHECK_NOTHING(); } @@ -2071,12 +2249,12 @@ PERF_TEST_P_(KMeans3DPerfTest, TestPerformance) PERF_TEST_P_(TransposePerfTest, TestPerformance) { compare_f cmpF; - cv::Size sz_in; + cv::Size sz; MatType type = -1; cv::GCompileArgs compile_args; - std::tie(cmpF, sz_in, type, compile_args) = GetParam(); + std::tie(cmpF, sz, type, compile_args) = GetParam(); - initMatrixRandU(type, sz_in, type, false); + initMatrixRandU(type, sz, type, false); // OpenCV code /////////////////////////////////////////////////////////// cv::transpose(in_mat1, out_mat_ocv); @@ -2106,14 +2284,15 @@ PERF_TEST_P_(TransposePerfTest, TestPerformance) PERF_TEST_P_(ResizePerfTest, TestPerformance) { - compare_f cmpF = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - int interp = get<2>(GetParam()); - cv::Size sz_in = get<3>(GetParam()); - cv::Size sz_out = get<4>(GetParam()); - cv::GCompileArgs compile_args = get<5>(GetParam()); + compare_f cmpF; + MatType type = -1; + int interp = 1; + cv::Size sz; + cv::Size sz_out; + cv::GCompileArgs compile_args; + std::tie(cmpF, type, interp, sz, sz_out, compile_args) = GetParam(); - in_mat1 = cv::Mat(sz_in, type); + in_mat1 = cv::Mat(sz, type); cv::Scalar mean = cv::Scalar::all(127); cv::Scalar stddev = cv::Scalar::all(40.f); cv::randn(in_mat1, mean, stddev); @@ -2150,19 +2329,20 @@ PERF_TEST_P_(ResizePerfTest, TestPerformance) PERF_TEST_P_(ResizeFxFyPerfTest, TestPerformance) { - compare_f cmpF = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - int interp = get<2>(GetParam()); - cv::Size sz_in = get<3>(GetParam()); - double fx = get<4>(GetParam()); - double fy = get<5>(GetParam()); - cv::GCompileArgs compile_args = get<6>(GetParam()); + compare_f cmpF; + MatType type = -1; + int interp = 1; + cv::Size sz; + double fx = 0.0; + double fy = 0.0; + cv::GCompileArgs compile_args; + std::tie(cmpF, type, interp, sz, fx, fy, compile_args) = GetParam(); - in_mat1 = cv::Mat(sz_in, type); + in_mat1 = cv::Mat(sz, type); cv::Scalar mean = cv::Scalar::all(127); cv::Scalar stddev = cv::Scalar::all(40.f); cv::randn(in_mat1, mean, stddev); - cv::Size sz_out = cv::Size(saturate_cast(sz_in.width *fx), saturate_cast(sz_in.height*fy)); + cv::Size sz_out = cv::Size(saturate_cast(sz.width *fx), saturate_cast(sz.height*fy)); out_mat_gapi = cv::Mat(sz_out, type); out_mat_ocv = cv::Mat(sz_out, type); @@ -2198,9 +2378,12 @@ PERF_TEST_P_(ResizeFxFyPerfTest, TestPerformance) PERF_TEST_P_(BottleneckKernelsConstInputPerfTest, TestPerformance) { - compare_f cmpF = get<0>(GetParam()); - std::string fileName = get<1>(GetParam()); - cv::GCompileArgs compile_args = get<2>(GetParam()); + compare_f cmpF; + std::string fileName = ""; + cv::GCompileArgs compile_args; + double fx = 0.5; + double fy = 0.5; + std::tie(cmpF, fileName, compile_args) = GetParam(); in_mat1 = cv::imread(findDataFile(fileName)); @@ -2208,13 +2391,13 @@ PERF_TEST_P_(BottleneckKernelsConstInputPerfTest, TestPerformance) cv::Mat cvgray; cv::Mat cvblurred; - cv::resize(in_mat1, cvvga, cv::Size(), 0.5, 0.5); + cv::resize(in_mat1, cvvga, cv::Size(), fx, fy); cv::cvtColor(cvvga, cvgray, cv::COLOR_BGR2GRAY); cv::blur(cvgray, cvblurred, cv::Size(3, 3)); cv::Canny(cvblurred, out_mat_ocv, 32, 128, 3); cv::GMat in; - cv::GMat vga = cv::gapi::resize(in, cv::Size(), 0.5, 0.5, INTER_LINEAR); + cv::GMat vga = cv::gapi::resize(in, cv::Size(), fx, fy, INTER_LINEAR); cv::GMat gray = cv::gapi::BGR2Gray(vga); cv::GMat blurred = cv::gapi::blur(gray, cv::Size(3, 3)); cv::GMat out = cv::gapi::Canny(blurred, 32, 128, 3); @@ -2241,21 +2424,24 @@ PERF_TEST_P_(BottleneckKernelsConstInputPerfTest, TestPerformance) PERF_TEST_P_(ResizeInSimpleGraphPerfTest, TestPerformance) { - compare_f cmpF = get<0>(GetParam()); - MatType type = get<1>(GetParam()); - cv::Size sz_in = get<2>(GetParam()); - cv::GCompileArgs compile_args = get<3>(GetParam()); + compare_f cmpF; + MatType type = -1; + cv::Size sz; + double fx = 0.5; + double fy = 0.5; + cv::GCompileArgs compile_args; + std::tie(cmpF, type, sz, fx, fy, compile_args) = GetParam(); - initMatsRandU(type, sz_in, type, false); + initMatsRandU(type, sz, type, false); cv::Mat add_res_ocv; cv::add(in_mat1, in_mat2, add_res_ocv); - cv::resize(add_res_ocv, out_mat_ocv, cv::Size(), 0.5, 0.5); + cv::resize(add_res_ocv, out_mat_ocv, cv::Size(), fx, fy); cv::GMat in1, in2; cv::GMat add_res_gapi = cv::gapi::add(in1, in2); - cv::GMat out = cv::gapi::resize(add_res_gapi, cv::Size(), 0.5, 0.5, INTER_LINEAR); + cv::GMat out = cv::gapi::resize(add_res_gapi, cv::Size(), fx, fy, INTER_LINEAR); cv::GComputation ac(GIn(in1, in2), GOut(out)); auto cc = ac.compile(descr_of(gin(in_mat1, in_mat2)), @@ -2284,6 +2470,7 @@ PERF_TEST_P_(ParseSSDBLPerfTest, TestPerformance) int filter_label = 0; cv::GCompileArgs compile_args; std::tie(sz, confidence_threshold, filter_label, compile_args) = GetParam(); + cv::Mat in_mat = generateSSDoutput(sz); std::vector boxes_gapi, boxes_ref; std::vector labels_gapi, labels_ref; @@ -2324,6 +2511,7 @@ PERF_TEST_P_(ParseSSDPerfTest, TestPerformance) bool alignment_to_square = false, filter_out_of_bounds = false; cv::GCompileArgs compile_args; std::tie(sz, confidence_threshold, alignment_to_square, filter_out_of_bounds, compile_args) = GetParam(); + cv::Mat in_mat = generateSSDoutput(sz); std::vector boxes_gapi, boxes_ref; @@ -2362,6 +2550,7 @@ PERF_TEST_P_(ParseYoloPerfTest, TestPerformance) int num_classes = 0; cv::GCompileArgs compile_args; std::tie(sz, confidence_threshold, nms_threshold, num_classes, compile_args) = GetParam(); + cv::Mat in_mat = generateYoloOutput(num_classes); auto anchors = cv::gapi::nn::parsers::GParseYolo::defaultAnchors(); std::vector boxes_gapi, boxes_ref; @@ -2398,7 +2587,7 @@ PERF_TEST_P_(ParseYoloPerfTest, TestPerformance) PERF_TEST_P_(SizePerfTest, TestPerformance) { - MatType type; + MatType type = -1; cv::Size sz; cv::GCompileArgs compile_args; std::tie(type, sz, compile_args) = GetParam(); @@ -2408,20 +2597,20 @@ PERF_TEST_P_(SizePerfTest, TestPerformance) cv::GMat in; auto out = cv::gapi::streaming::size(in); cv::GComputation c(cv::GIn(in), cv::GOut(out)); - cv::Size out_sz; + cv::Size sz_out; // Warm-up graph engine: auto cc = c.compile(descr_of(in_mat1), std::move(compile_args)); - cc(cv::gin(in_mat1), cv::gout(out_sz)); + cc(cv::gin(in_mat1), cv::gout(sz_out)); TEST_CYCLE() { - cc(cv::gin(in_mat1), cv::gout(out_sz)); + cc(cv::gin(in_mat1), cv::gout(sz_out)); } // Comparison //////////////////////////////////////////////////////////// { - EXPECT_EQ(out_sz, sz); + EXPECT_EQ(sz_out, sz); } SANITY_CHECK_NOTHING(); @@ -2440,20 +2629,20 @@ PERF_TEST_P_(SizeRPerfTest, TestPerformance) cv::GOpaque op_rect; auto out = cv::gapi::streaming::size(op_rect); cv::GComputation c(cv::GIn(op_rect), cv::GOut(out)); - cv::Size out_sz; + cv::Size sz_out; // Warm-up graph engine: auto cc = c.compile(descr_of(rect), std::move(compile_args)); - cc(cv::gin(rect), cv::gout(out_sz)); + cc(cv::gin(rect), cv::gout(sz_out)); TEST_CYCLE() { - cc(cv::gin(rect), cv::gout(out_sz)); + cc(cv::gin(rect), cv::gout(sz_out)); } // Comparison //////////////////////////////////////////////////////////// { - EXPECT_EQ(out_sz, sz); + EXPECT_EQ(sz_out, sz); } SANITY_CHECK_NOTHING(); diff --git a/modules/gapi/perf/cpu/gapi_core_perf_tests_cpu.cpp b/modules/gapi/perf/cpu/gapi_core_perf_tests_cpu.cpp index c110de4fdd..5323ea8f08 100644 --- a/modules/gapi/perf/cpu/gapi_core_perf_tests_cpu.cpp +++ b/modules/gapi/perf/cpu/gapi_core_perf_tests_cpu.cpp @@ -14,46 +14,60 @@ namespace opencv_test { +INSTANTIATE_TEST_CASE_P(PhasePerfTestFluid, PhasePerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_32FC1, CV_64FC1), + Values(cv::compile_args(CORE_CPU)))); + +INSTANTIATE_TEST_CASE_P(SqrtPerfTestFluid, SqrtPerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_32FC1, CV_64FC1), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(AddPerfTestCPU, AddPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(-1, CV_8U, CV_16U, CV_32F), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(-1, CV_8U, CV_16U, CV_32F), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(AddCPerfTestCPU, AddCPerfTest, Combine(Values(AbsExact().to_compare_f()), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(-1, CV_8U, CV_16U, CV_32F), - Values(cv::compile_args(CORE_CPU)))); + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(-1, CV_8U, CV_16U, CV_32F), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(SubPerfTestCPU, SubPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(-1, CV_8U, CV_16U, CV_32F), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(-1, CV_8U, CV_16U, CV_32F), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(SubCPerfTestCPU, SubCPerfTest, Combine(Values(AbsExact().to_compare_f()), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(-1, CV_8U, CV_16U, CV_32F), - Values(cv::compile_args(CORE_CPU)))); + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(-1, CV_8U, CV_16U, CV_32F), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(SubRCPerfTestCPU, SubRCPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(-1, CV_8U, CV_16U, CV_32F), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(-1, CV_8U, CV_16U, CV_32F), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(MulPerfTestCPU, MulPerfTest, Combine(Values(AbsExact().to_compare_f()), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(-1, CV_8U, CV_16U, CV_32F), - Values(2.0), - Values(cv::compile_args(CORE_CPU)))); + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(-1, CV_8U, CV_16U, CV_32F), + Values(2.0), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(MulDoublePerfTestCPU, MulDoublePerfTest, Combine(Values(AbsExact().to_compare_f()), @@ -64,57 +78,63 @@ INSTANTIATE_TEST_CASE_P(MulDoublePerfTestCPU, MulDoublePerfTest, INSTANTIATE_TEST_CASE_P(MulCPerfTestCPU, MulCPerfTest, Combine(Values(AbsExact().to_compare_f()), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(-1, CV_8U, CV_16U, CV_32F), - Values(cv::compile_args(CORE_CPU)))); + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(-1, CV_8U, CV_16U, CV_32F), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(DivPerfTestCPU, DivPerfTest, Combine(Values(AbsExact().to_compare_f()), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(-1, CV_8U, CV_16U, CV_16S, CV_32F), - Values(2.3), - Values(cv::compile_args(CORE_CPU)))); + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(-1, CV_8U, CV_16U, CV_16S, CV_32F), + Values(2.3), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(DivCPerfTestCPU, DivCPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(-1, CV_8U, CV_16U, CV_32F), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(-1, CV_8U, CV_16U, CV_32F), + Values(1.0), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(DivRCPerfTestCPU, DivRCPerfTest, Combine(Values(AbsExact().to_compare_f()), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(-1, CV_8U, CV_16U, CV_32F), - Values(cv::compile_args(CORE_CPU)))); + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(-1, CV_8U, CV_16U, CV_32F), + Values(1.0), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(MaskPerfTestCPU, MaskPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_16UC1, CV_16SC1), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_16UC1, CV_16SC1), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(MeanPerfTestCPU, MeanPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsToleranceScalar(0.0).to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(Polar2CartPerfTestCPU, Polar2CartPerfTest, Combine(Values(AbsExact().to_compare_f()), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(cv::compile_args(CORE_CPU)))); + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(Cart2PolarPerfTestCPU, Cart2PolarPerfTest, Combine(Values(AbsExact().to_compare_f()), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(cv::compile_args(CORE_CPU)))); + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(CmpPerfTestCPU, CmpPerfTest, - Combine(Values(CMP_EQ, CMP_GE, CMP_NE, CMP_GT, CMP_LT, CMP_LE), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(CMP_EQ, CMP_GE, CMP_NE, CMP_GT, CMP_LT, CMP_LE), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(CmpWithScalarPerfTestCPU, CmpWithScalarPerfTest, Combine(Values(AbsExact().to_compare_f()), @@ -124,162 +144,184 @@ INSTANTIATE_TEST_CASE_P(CmpWithScalarPerfTestCPU, CmpWithScalarPerfTest, Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(BitwisePerfTestCPU, BitwisePerfTest, - Combine(Values(AND, OR, XOR), + Combine(Values(AbsExact().to_compare_f()), + Values(AND, OR, XOR), testing::Bool(), Values(szSmall128, szVGA, sz720p, sz1080p), Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1), Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(BitwiseNotPerfTestCPU, BitwiseNotPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(SelectPerfTestCPU, SelectPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(MinPerfTestCPU, MinPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(MaxPerfTestCPU, MaxPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(AbsDiffPerfTestCPU, AbsDiffPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(AbsDiffCPerfTestCPU, AbsDiffCPerfTest, Combine(Values(AbsExact().to_compare_f()), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(cv::compile_args(CORE_CPU)))); + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(SumPerfTestCPU, SumPerfTest, Combine(Values(AbsToleranceScalar(0.0).to_compare_f()), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - //Values(0.0), - Values(cv::compile_args(CORE_CPU)))); + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(CountNonZeroPerfTestCPU, CountNonZeroPerfTest, - Combine(Values(AbsToleranceScalar(0.0).to_compare_f()), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_16UC1, CV_16SC1, CV_32FC1), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsToleranceScalar(0.0).to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(AddWeightedPerfTestCPU, AddWeightedPerfTest, - Combine(Values(Tolerance_FloatRel_IntAbs(1e-6, 1).to_compare_f()), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(-1, CV_8U, CV_16U, CV_32F), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(-1, CV_8U, CV_16U, CV_32F), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(NormPerfTestCPU, NormPerfTest, Combine(Values(AbsToleranceScalar(0.0).to_compare_f()), - Values(NORM_INF, NORM_L1, NORM_L2), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(cv::compile_args(CORE_CPU)))); + Values(NORM_INF, NORM_L1, NORM_L2), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(IntegralPerfTestCPU, IntegralPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_16UC1, CV_16SC1, CV_32FC1), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(ThresholdPerfTestCPU, ThresholdPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(cv::THRESH_BINARY, cv::THRESH_BINARY_INV, cv::THRESH_TRUNC, cv::THRESH_TOZERO, cv::THRESH_TOZERO_INV), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::THRESH_BINARY, cv::THRESH_BINARY_INV, cv::THRESH_TRUNC, cv::THRESH_TOZERO, cv::THRESH_TOZERO_INV), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(ThresholdPerfTestCPU, ThresholdOTPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1), - Values(cv::THRESH_OTSU, cv::THRESH_TRIANGLE), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1), + Values(cv::THRESH_OTSU, cv::THRESH_TRIANGLE), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(InRangePerfTestCPU, InRangePerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(Split3PerfTestCPU, Split3PerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(Split4PerfTestCPU, Split4PerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(Merge3PerfTestCPU, Merge3PerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(Merge4PerfTestCPU, Merge4PerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(RemapPerfTestCPU, RemapPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(FlipPerfTestCPU, FlipPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(0, 1, -1), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(0, 1, -1), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(CropPerfTestCPU, CropPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(cv::Rect(10, 8, 20, 35), cv::Rect(4, 10, 37, 50)), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::Rect(10, 8, 20, 35), cv::Rect(4, 10, 37, 50)), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(CopyPerfTestCPU, CopyPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(ConcatHorPerfTestCPU, ConcatHorPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(ConcatHorVecPerfTestCPU, ConcatHorVecPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(ConcatVertPerfTestCPU, ConcatVertPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(ConcatVertVecPerfTestCPU, ConcatVertVecPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(LUTPerfTestCPU, LUTPerfTest, - Combine(Values(CV_8UC1, CV_8UC3), - Values(CV_8UC1), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(CV_8UC1, CV_8UC3), + Values(CV_8UC1), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(LUTPerfTestCustomCPU, LUTPerfTest, - Combine(Values(CV_8UC3), - Values(CV_8UC3), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(cv::compile_args(CORE_CPU)))); - + Combine(Values(AbsExact().to_compare_f()), + Values(CV_8UC3), + Values(CV_8UC3), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(ConvertToPerfTestCPU, ConvertToPerfTest, Combine(Values(AbsExact().to_compare_f()), @@ -291,98 +333,100 @@ INSTANTIATE_TEST_CASE_P(ConvertToPerfTestCPU, ConvertToPerfTest, Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(KMeansNDPerfTestCPU, KMeansNDPerfTest, - Combine(Values(cv::Size(1, 20), - cv::Size(16, 4096)), - Values(AbsTolerance(0.01).to_compare_obj()), - Values(5, 15), - Values(cv::KMEANS_RANDOM_CENTERS, - cv::KMEANS_PP_CENTERS, - cv::KMEANS_RANDOM_CENTERS | cv::KMEANS_USE_INITIAL_LABELS, - cv::KMEANS_PP_CENTERS | cv::KMEANS_USE_INITIAL_LABELS), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(cv::Size(1, 20), + cv::Size(16, 4096)), + Values(AbsTolerance(0.01).to_compare_obj()), + Values(5, 15), + Values(cv::KMEANS_RANDOM_CENTERS, + cv::KMEANS_PP_CENTERS, + cv::KMEANS_RANDOM_CENTERS | cv::KMEANS_USE_INITIAL_LABELS, + cv::KMEANS_PP_CENTERS | cv::KMEANS_USE_INITIAL_LABELS), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(KMeans2DPerfTestCPU, KMeans2DPerfTest, - Combine(Values(20, 4096), - Values(5, 15), - Values(cv::KMEANS_RANDOM_CENTERS, - cv::KMEANS_PP_CENTERS, - cv::KMEANS_RANDOM_CENTERS | cv::KMEANS_USE_INITIAL_LABELS, - cv::KMEANS_PP_CENTERS | cv::KMEANS_USE_INITIAL_LABELS), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(20, 4096), + Values(5, 15), + Values(cv::KMEANS_RANDOM_CENTERS, + cv::KMEANS_PP_CENTERS, + cv::KMEANS_RANDOM_CENTERS | cv::KMEANS_USE_INITIAL_LABELS, + cv::KMEANS_PP_CENTERS | cv::KMEANS_USE_INITIAL_LABELS), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(KMeans3DPerfTestCPU, KMeans3DPerfTest, - Combine(Values(20, 4096), - Values(5, 15), - Values(cv::KMEANS_RANDOM_CENTERS, - cv::KMEANS_PP_CENTERS, - cv::KMEANS_RANDOM_CENTERS | cv::KMEANS_USE_INITIAL_LABELS, - cv::KMEANS_PP_CENTERS | cv::KMEANS_USE_INITIAL_LABELS), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(20, 4096), + Values(5, 15), + Values(cv::KMEANS_RANDOM_CENTERS, + cv::KMEANS_PP_CENTERS, + cv::KMEANS_RANDOM_CENTERS | cv::KMEANS_USE_INITIAL_LABELS, + cv::KMEANS_PP_CENTERS | cv::KMEANS_USE_INITIAL_LABELS), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(TransposePerfTestCPU, TransposePerfTest, - Combine(Values(AbsExact().to_compare_f()), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_16UC1, CV_16SC1, CV_32FC1, - CV_8UC2, CV_16UC2, CV_16SC2, CV_32FC2, - CV_8UC3, CV_16UC3, CV_16SC3, CV_32FC3), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_16UC1, CV_16SC1, CV_32FC1, + CV_8UC2, CV_16UC2, CV_16SC2, CV_32FC2, + CV_8UC3, CV_16UC3, CV_16SC3, CV_32FC3), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(ResizePerfTestCPU, ResizePerfTest, Combine(Values(AbsExact().to_compare_f()), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1), - Values(cv::INTER_NEAREST, cv::INTER_LINEAR, cv::INTER_AREA), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(cv::Size(64, 64), - cv::Size(32, 32)), - Values(cv::compile_args(CORE_CPU)))); + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1), + Values(cv::INTER_NEAREST, cv::INTER_LINEAR, cv::INTER_AREA), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values( cv::Size(64, 64), + cv::Size(32, 32)), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(BottleneckKernelsPerfTestCPU, BottleneckKernelsConstInputPerfTest, Combine(Values(AbsExact().to_compare_f()), - Values("cv/optflow/frames/1080p_00.png", "cv/optflow/frames/720p_00.png", - "cv/optflow/frames/VGA_00.png", "cv/dnn_face/recognition/Aaron_Tippin_0001.jpg"), - Values(cv::compile_args(CORE_CPU)))); + Values( "cv/optflow/frames/1080p_00.png", "cv/optflow/frames/720p_00.png", + "cv/optflow/frames/VGA_00.png", "cv/dnn_face/recognition/Aaron_Tippin_0001.jpg"), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(ResizeInSimpleGraphPerfTestCPU, ResizeInSimpleGraphPerfTest, Combine(Values(AbsExact().to_compare_f()), - Values(CV_8UC3), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(cv::compile_args(CORE_CPU)))); + Values(CV_8UC3), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(0.5), + Values(0.5), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(ResizeFxFyPerfTestCPU, ResizeFxFyPerfTest, Combine(Values(AbsExact().to_compare_f()), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1), - Values(cv::INTER_NEAREST, cv::INTER_LINEAR, cv::INTER_AREA), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(0.5, 0.1), - Values(0.5, 0.1), - Values(cv::compile_args(CORE_CPU)))); + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1), + Values(cv::INTER_NEAREST, cv::INTER_LINEAR, cv::INTER_AREA), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(0.5, 0.1), + Values(0.5, 0.1), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(ParseSSDBLPerfTestCPU, ParseSSDBLPerfTest, - Combine(Values(sz720p, sz1080p), - Values(0.3f, 0.7f), - Values(0, 1), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(sz720p, sz1080p), + Values(0.3f, 0.7f), + Values(0, 1), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(ParseSSDPerfTestCPU, ParseSSDPerfTest, - Combine(Values(sz720p, sz1080p), - Values(0.3f, 0.7f), - testing::Bool(), - testing::Bool(), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(sz720p, sz1080p), + Values(0.3f, 0.7f), + testing::Bool(), + testing::Bool(), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(ParseYoloPerfTestCPU, ParseYoloPerfTest, - Combine(Values(sz720p, sz1080p), - Values(0.3f, 0.7f), - Values(0.5), - Values(7, 80), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(sz720p, sz1080p), + Values(0.3f, 0.7f), + Values(0.5), + Values(7, 80), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(SizePerfTestCPU, SizePerfTest, - Combine(Values(CV_8UC1, CV_8UC3, CV_32FC1), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(CV_8UC1, CV_8UC3, CV_32FC1), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(cv::compile_args(CORE_CPU)))); INSTANTIATE_TEST_CASE_P(SizeRPerfTestCPU, SizeRPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(cv::compile_args(CORE_CPU)))); + Combine(Values(szSmall128, szVGA, sz720p, sz1080p), + Values(cv::compile_args(CORE_CPU)))); } // opencv_test diff --git a/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp b/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp index 442d9efa7a..e25029b835 100644 --- a/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp +++ b/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp @@ -12,106 +12,128 @@ namespace opencv_test { + +INSTANTIATE_TEST_CASE_P(PhasePerfTestFluid, PhasePerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_32FC1, CV_64FC1), + Values(cv::compile_args(CORE_FLUID)))); + +INSTANTIATE_TEST_CASE_P(SqrtPerfTestFluid, SqrtPerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_32FC1, CV_64FC1), + Values(cv::compile_args(CORE_FLUID)))); + INSTANTIATE_TEST_CASE_P(AddPerfTestFluid, AddPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), + Combine(Values(Tolerance_FloatRel_IntAbs(1e-6, 1).to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), Values(CV_8UC1, CV_8UC3, CV_16SC1, CV_32FC1), Values(-1, CV_8U, CV_32F), Values(cv::compile_args(CORE_FLUID)))); - INSTANTIATE_TEST_CASE_P(AddCPerfTestFluid, AddCPerfTest, - Combine(Values(Tolerance_FloatRel_IntAbs(1e-6, 1).to_compare_f()), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(-1, CV_8U, CV_16U, CV_16S, CV_32F), - Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(AddCPerfTestFluid, AddCPerfTest, + Combine(Values(Tolerance_FloatRel_IntAbs(1e-6, 1).to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(-1, CV_8U, CV_16U, CV_16S, CV_32F), + Values(cv::compile_args(CORE_FLUID)))); INSTANTIATE_TEST_CASE_P(SubPerfTestFluid, SubPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), Values(CV_8UC1, CV_8UC3, CV_16SC1, CV_32FC1), Values(-1, CV_8U, CV_32F), Values(cv::compile_args(CORE_FLUID)))); - INSTANTIATE_TEST_CASE_P(SubCPerfTestFluid, SubCPerfTest, - Combine(Values(Tolerance_FloatRel_IntAbs(1e-6, 1).to_compare_f()), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(-1, CV_8U, CV_16U, CV_16S, CV_32F), - Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(SubCPerfTestFluid, SubCPerfTest, + Combine(Values(Tolerance_FloatRel_IntAbs(1e-6, 1).to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16SC1, CV_32FC1), + Values(-1, CV_8U, CV_32F), + Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(SubRCPerfTestFluid, SubRCPerfTest, -// Combine(Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), -// Values(-1, CV_8U, CV_16U, CV_32F), -// Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(SubRCPerfTestFluid, SubRCPerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16SC1, CV_32FC1), + Values(-1, CV_8U, CV_32F), + Values(cv::compile_args(CORE_FLUID)))); - INSTANTIATE_TEST_CASE_P(MulPerfTestFluid, MulPerfTest, - Combine(Values(AbsExact().to_compare_f()), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(-1, CV_8U, CV_16U, CV_16S, CV_32F), - Values(2.0), - Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(MulPerfTestFluid, MulPerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(-1, CV_8U, CV_16U, CV_16S, CV_32F), + Values(2.0), + Values(cv::compile_args(CORE_FLUID)))); - INSTANTIATE_TEST_CASE_P(MulDoublePerfTestFluid, MulDoublePerfTest, - Combine(Values(Tolerance_FloatRel_IntAbs(1e-6, 1).to_compare_f()), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16SC1, CV_32FC1), - Values(-1, CV_8U, CV_32F), - Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(MulDoublePerfTestFluid, MulDoublePerfTest, + Combine(Values(Tolerance_FloatRel_IntAbs(1e-6, 1).to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16SC1, CV_32FC1), + Values(-1, CV_8U, CV_32F), + Values(cv::compile_args(CORE_FLUID)))); - INSTANTIATE_TEST_CASE_P(MulCPerfTestFluid, MulCPerfTest, - Combine(Values(Tolerance_FloatRel_IntAbs(1e-6, 1).to_compare_f()), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(-1, CV_8U, CV_16U, CV_16S, CV_32F), - Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(MulCPerfTestFluid, MulCPerfTest, + Combine(Values(Tolerance_FloatRel_IntAbs(1e-6, 1).to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16SC1, CV_32FC1), + Values(-1, CV_8U, CV_32F), + Values(cv::compile_args(CORE_FLUID)))); - INSTANTIATE_TEST_CASE_P(DivPerfTestFluid, DivPerfTest, - Combine(Values(AbsExact().to_compare_f()), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(-1, CV_8U, CV_16U, CV_16S, CV_32F), - Values(2.3), - Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(DivPerfTestFluid, DivPerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(-1, CV_8U, CV_16U, CV_16S, CV_32F), + Values(2.3), + Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(DivCPerfTestFluid, DivCPerfTest, -// Combine(Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), -// Values(-1, CV_8U, CV_16U, CV_32F), -// Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(DivCPerfTestFluid, DivCPerfTest, + Combine(Values(Tolerance_FloatRel_IntAbs(1e-6, 1).to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16SC1, CV_32FC1), + Values(-1, CV_8U, CV_32F), + Values(1.0), + Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(DivRCPerfTestFluid, DivRCPerfTest, -// Combine(Values(AbsExact().to_compare_f()), -// Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), -// Values(-1, CV_8U, CV_16U, CV_32F), -// Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(DivRCPerfTestFluid, DivRCPerfTest, + Combine(Values(Tolerance_FloatRel_IntAbs(1e-5, 1).to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_32FC1), + Values(-1, CV_8U, CV_32F), + Values(1.0), + Values(cv::compile_args(CORE_FLUID)))); INSTANTIATE_TEST_CASE_P(MaskPerfTestFluid, MaskPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), Values(CV_8UC1, CV_16UC1, CV_16SC1), Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(MeanPerfTestFluid, MeanPerfTest, -// Combine(Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), -// Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(MeanPerfTestFluid, MeanPerfTest, + Combine(Values(AbsToleranceScalar(0.0).to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(Polar2CartPerfTestFluid, Polar2CartPerfTest, -// Combine(Values(AbsExact().to_compare_f()), -// Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(Polar2CartPerfTestFluid, Polar2CartPerfTest, + Combine(Values(Tolerance_FloatRel_IntAbs(1e-6, 1).to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(Cart2PolarPerfTestFluid, Cart2PolarPerfTest, -// Combine(Values(AbsExact().to_compare_f()), -// Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(Cart2PolarPerfTestFluid, Cart2PolarPerfTest, + Combine(Values(Tolerance_FloatRel_IntAbs(1e-04, 1).to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(CmpPerfTestFluid, CmpPerfTest, -// Combine(Values(CMP_EQ, CMP_GE, CMP_NE, CMP_GT, CMP_LT, CMP_LE), -// Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), -// Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(CmpPerfTestFluid, CmpPerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(CMP_EQ, CMP_GE, CMP_NE, CMP_GT, CMP_LT, CMP_LE), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_FLUID)))); INSTANTIATE_TEST_CASE_P(CmpWithScalarPerfTestFluid, CmpWithScalarPerfTest, Combine(Values(AbsSimilarPoints(1, 0.01).to_compare_f()), @@ -121,34 +143,40 @@ INSTANTIATE_TEST_CASE_P(CmpWithScalarPerfTestFluid, CmpWithScalarPerfTest, Values(cv::compile_args(CORE_FLUID)))); INSTANTIATE_TEST_CASE_P(BitwisePerfTestFluid, BitwisePerfTest, - Combine(Values(AND, OR, XOR), + Combine(Values(AbsExact().to_compare_f()), + Values(AND, OR, XOR), testing::Bool(), Values(szSmall128, szVGA, sz720p, sz1080p), Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1), Values(cv::compile_args(CORE_FLUID)))); INSTANTIATE_TEST_CASE_P(BitwiseNotPerfTestFluid, BitwiseNotPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1), Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(SelectPerfTestFluid, SelectPerfTest, -// Combine(Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), -// Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(SelectPerfTestFluid, SelectPerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1), + Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(MinPerfTestFluid, MinPerfTest, -// Combine(Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), -// Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(MinPerfTestFluid, MinPerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(MaxPerfTestFluid, MaxPerfTest, -// Combine(Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), -// Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(MaxPerfTestFluid, MaxPerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_FLUID)))); INSTANTIATE_TEST_CASE_P(AbsDiffPerfTestFluid, AbsDiffPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), Values(cv::compile_args(CORE_FLUID)))); @@ -158,12 +186,11 @@ INSTANTIATE_TEST_CASE_P(AbsDiffCPerfTestFluid, AbsDiffCPerfTest, Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(SumPerfTestFluid, SumPerfTest, -// Combine(Values(AbsToleranceScalar(0.0).to_compare_f()), -// Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), -// //Values(0.0), -// Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(SumPerfTestFluid, SumPerfTest, + Combine(Values(AbsToleranceScalar(0.0).to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_FLUID)))); INSTANTIATE_TEST_CASE_P(AddWeightedPerfTestFluid, AddWeightedPerfTest, Combine(Values(Tolerance_FloatRel_IntAbs(1e-6, 1).to_compare_f()), @@ -179,102 +206,121 @@ INSTANTIATE_TEST_CASE_P(AddWeightedPerfTestFluid_short, AddWeightedPerfTest, Values(-1), Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(NormPerfTestFluid, NormPerfTest, -// Combine(Values(AbsToleranceScalar(0.0).to_compare_f()), -// Values(NORM_INF, NORM_L1, NORM_L2), -// Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), -// Values(cv::compile_args(CORE_FLUID)))); - -// INSTANTIATE_TEST_CASE_P(IntegralPerfTestFluid, IntegralPerfTest, -// Combine(Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(CV_8UC1, CV_16UC1, CV_16SC1, CV_32FC1), -// Values(cv::compile_args(CORE_FLUID)))); - -// INSTANTIATE_TEST_CASE_P(ThresholdPerfTestFluid, ThresholdPerfTest, -// Combine(Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), -// Values(cv::THRESH_BINARY, cv::THRESH_BINARY_INV, cv::THRESH_TRUNC, cv::THRESH_TOZERO, cv::THRESH_TOZERO_INV), -// Values(cv::compile_args(CORE_FLUID)))); - -// INSTANTIATE_TEST_CASE_P(ThresholdPerfTestFluid, ThresholdOTPerfTest, -// Combine(Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(CV_8UC1), -// Values(cv::THRESH_OTSU, cv::THRESH_TRIANGLE), -// Values(cv::compile_args(CORE_FLUID)))); - -// INSTANTIATE_TEST_CASE_P(InRangePerfTestFluid, InRangePerfTest, -// Combine(Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(CV_8UC1), -// Values(cv::compile_args(CORE_FLUID)))); - -INSTANTIATE_TEST_CASE_P(Split3PerfTestFluid, Split3PerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), +INSTANTIATE_TEST_CASE_P(NormPerfTestFluid, NormPerfTest, + Combine(Values(AbsToleranceScalar(0.0).to_compare_f()), + Values(NORM_INF, NORM_L1, NORM_L2), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(Split4PerfTestFluid, Split4PerfTest, -// Combine(Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(IntegralPerfTestFluid, IntegralPerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(Merge3PerfTestFluid, Merge3PerfTest, -// Combine(Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(ThresholdPerfTestFluid, ThresholdPerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1), + Values(cv::THRESH_BINARY, cv::THRESH_BINARY_INV, cv::THRESH_TRUNC, + cv::THRESH_TOZERO, cv::THRESH_TOZERO_INV), + Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(Merge4PerfTestFluid, Merge4PerfTest, -// Combine(Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(ThresholdPerfTestFluid, ThresholdOTPerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1), + Values(cv::THRESH_OTSU, cv::THRESH_TRIANGLE), + Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(RemapPerfTestFluid, RemapPerfTest, -// Combine(Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), -// Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(InRangePerfTestFluid, InRangePerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1), + Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(FlipPerfTestFluid, FlipPerfTest, -// Combine(Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), -// Values(0, 1, -1), -// Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(Split3PerfTestFluid, Split3PerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(CropPerfTestFluid, CropPerfTest, -// Combine(Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), -// Values(cv::Rect(10, 8, 20, 35), cv::Rect(4, 10, 37, 50)), -// Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(Split4PerfTestFluid, Split4PerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(ConcatHorPerfTestFluid, ConcatHorPerfTest, -// Combine(Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), -// Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(Merge3PerfTestFluid, Merge3PerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(ConcatHorVecPerfTestFluid, ConcatHorVecPerfTest, -// Combine(Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), -// Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(Merge4PerfTestFluid, Merge4PerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(ConcatVertPerfTestFluid, ConcatVertPerfTest, -// Combine(Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), -// Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(RemapPerfTestFluid, RemapPerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(ConcatVertVecPerfTestFluid, ConcatVertVecPerfTest, -// Combine(Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), -// Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(FlipPerfTestFluid, FlipPerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(0, 1, -1), + Values(cv::compile_args(CORE_FLUID)))); -// INSTANTIATE_TEST_CASE_P(LUTPerfTestFluid, LUTPerfTest, -// Combine(Values(CV_8UC1, CV_8UC3), -// Values(CV_8UC1), -// Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(CropPerfTestFluid, CropPerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::Rect(10, 8, 20, 35), cv::Rect(4, 10, 37, 50)), + Values(cv::compile_args(CORE_FLUID)))); +INSTANTIATE_TEST_CASE_P(ConcatHorPerfTestFluid, ConcatHorPerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_FLUID)))); + +INSTANTIATE_TEST_CASE_P(ConcatHorVecPerfTestFluid, ConcatHorVecPerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_FLUID)))); + +INSTANTIATE_TEST_CASE_P(ConcatVertPerfTestFluid, ConcatVertPerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_FLUID)))); + +INSTANTIATE_TEST_CASE_P(ConcatVertVecPerfTestFluid, ConcatVertVecPerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_FLUID)))); + +INSTANTIATE_TEST_CASE_P(LUTPerfTestFluid, LUTPerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(CV_8UC1, CV_8UC3), + Values(CV_8UC1), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(cv::compile_args(CORE_FLUID)))); + +// FIXIT: This test case doesn't work [3030] // INSTANTIATE_TEST_CASE_P(LUTPerfTestCustomFluid, LUTPerfTest, -// Combine(Values(CV_8UC3), -// Values(CV_8UC3), -// Values(szSmall128, szVGA, sz720p, sz1080p), -// Values(cv::compile_args(CORE_FLUID)))); +// Combine(Values(AbsExact().to_compare_f()), +// Values(CV_8UC3), +// Values(CV_8UC3), +// Values(szSmall128, szVGA, sz720p, sz1080p), +// Values(cv::compile_args(CORE_FLUID)))); INSTANTIATE_TEST_CASE_P(ConvertToPerfTestFluid, ConvertToPerfTest, - Combine(Values(Tolerance_FloatRel_IntAbs(1e-5, 2).to_compare_f()), + Combine(Values(Tolerance_FloatRel_IntAbs(1e-5, 1).to_compare_f()), Values(CV_8UC3, CV_8UC1, CV_16UC1, CV_16SC1, CV_32FC1), Values(CV_8U, CV_16U, CV_16S, CV_32F), Values(szSmall128, szVGA, sz720p, sz1080p), @@ -284,32 +330,34 @@ INSTANTIATE_TEST_CASE_P(ConvertToPerfTestFluid, ConvertToPerfTest, INSTANTIATE_TEST_CASE_P(ResizePerfTestFluid, ResizePerfTest, Combine(Values(Tolerance_FloatRel_IntAbs(1e-5, 1).to_compare_f()), - Values(CV_8UC3), - Values(cv::INTER_LINEAR), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(cv::Size(64, 64), - cv::Size(30, 30)), - Values(cv::compile_args(CORE_FLUID)))); + Values(CV_8UC3), + Values(cv::INTER_LINEAR), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(cv::Size(64, 64), + cv::Size(30, 30)), + Values(cv::compile_args(CORE_FLUID)))); #define IMGPROC_FLUID cv::gapi::imgproc::fluid::kernels() INSTANTIATE_TEST_CASE_P(BottleneckKernelsPerfTestFluid, BottleneckKernelsConstInputPerfTest, Combine(Values(AbsSimilarPoints(0, 1).to_compare_f()), - Values("cv/optflow/frames/1080p_00.png", "cv/optflow/frames/720p_00.png", - "cv/optflow/frames/VGA_00.png", "cv/dnn_face/recognition/Aaron_Tippin_0001.jpg"), - Values(cv::compile_args(CORE_FLUID, IMGPROC_FLUID)))); + Values("cv/optflow/frames/1080p_00.png", "cv/optflow/frames/720p_00.png", + "cv/optflow/frames/VGA_00.png", "cv/dnn_face/recognition/Aaron_Tippin_0001.jpg"), + Values(cv::compile_args(CORE_FLUID, IMGPROC_FLUID)))); INSTANTIATE_TEST_CASE_P(ResizeInSimpleGraphPerfTestFluid, ResizeInSimpleGraphPerfTest, Combine(Values(Tolerance_FloatRel_IntAbs(1e-5, 1).to_compare_f()), - Values(CV_8UC3), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(cv::compile_args(CORE_FLUID, IMGPROC_FLUID)))); + Values(CV_8UC3), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(0.5), + Values(0.5), + Values(cv::compile_args(CORE_FLUID, IMGPROC_FLUID)))); INSTANTIATE_TEST_CASE_P(ResizeFxFyPerfTestFluid, ResizeFxFyPerfTest, Combine(Values(Tolerance_FloatRel_IntAbs(1e-5, 1).to_compare_f()), - Values(CV_8UC3), - Values(cv::INTER_LINEAR), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(0.5, 0.1), - Values(0.5, 0.1), - Values(cv::compile_args(CORE_FLUID)))); + Values(CV_8UC3), + Values(cv::INTER_LINEAR), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(0.5, 0.1), + Values(0.5, 0.1), + Values(cv::compile_args(CORE_FLUID)))); } // opencv_test diff --git a/modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp b/modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp index fd00f1caea..6aaec4d79a 100644 --- a/modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp +++ b/modules/gapi/perf/gpu/gapi_core_perf_tests_gpu.cpp @@ -14,299 +14,331 @@ namespace opencv_test { INSTANTIATE_TEST_CASE_P(AddPerfTestGPU, AddPerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values( -1, CV_8U, CV_16U, CV_32F ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values( -1, CV_8U, CV_16U, CV_32F ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(AddCPerfTestGPU, AddCPerfTest, - Combine(Values(AbsExact().to_compare_f()), - Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values( -1, CV_8U, CV_16U, CV_32F ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values( -1, CV_8U, CV_16U, CV_32F ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(SubPerfTestGPU, SubPerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values( -1, CV_8U, CV_16U, CV_32F ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values( -1, CV_8U, CV_16U, CV_32F ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(SubCPerfTestGPU, SubCPerfTest, - Combine(Values(AbsExact().to_compare_f()), - Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values( -1, CV_8U, CV_16U, CV_32F ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values( -1, CV_8U, CV_16U, CV_32F ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(SubRCPerfTestGPU, SubRCPerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values( -1, CV_8U, CV_16U, CV_32F ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values( -1, CV_8U, CV_16U, CV_32F ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(MulPerfTestGPU, MulPerfTest, - Combine(Values(AbsExact().to_compare_f()), - Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values( -1, CV_8U, CV_16U, CV_32F ), - Values(2.0), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values( -1, CV_8U, CV_16U, CV_32F ), + Values(2.0), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(MulDoublePerfTestGPU, MulDoublePerfTest, - Combine(Values(Tolerance_FloatRel_IntAbs(1e-5, 1).to_compare_f()), - Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values( -1, CV_8U, CV_16U, CV_32F ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(Tolerance_FloatRel_IntAbs(1e-5, 1).to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values( -1, CV_8U, CV_16U, CV_32F ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(MulCPerfTestGPU, MulCPerfTest, - Combine(Values(Tolerance_FloatRel_IntAbs(1e-5, 1).to_compare_f()), - Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values( -1, CV_8U, CV_16U, CV_32F ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(Tolerance_FloatRel_IntAbs(1e-5, 1).to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values( -1, CV_8U, CV_16U, CV_32F ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(DivPerfTestGPU, DivPerfTest, - Combine(Values(AbsTolerance(2).to_compare_f()), - Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values( -1, CV_8U, CV_16U, CV_32F ), - Values(2.3), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsTolerance(2).to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values( -1, CV_8U, CV_16U, CV_32F ), + Values(2.3), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(DivCPerfTestGPU, DivCPerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values( -1, CV_8U, CV_16U, CV_32F ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values( -1, CV_8U, CV_16U, CV_32F ), + Values(1.0), + Values(cv::compile_args(CORE_GPU)))); +// FIXIT: CV_16SC1 test case doesn't work with OpenCL [3031] INSTANTIATE_TEST_CASE_P(DivRCPerfTestGPU, DivRCPerfTest, - Combine(Values(Tolerance_FloatRel_IntAbs(1e-5, 2).to_compare_f()), - Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values( -1, CV_8U, CV_16U, CV_32F ), - Values(cv::compile_args(CORE_GPU)))); -//TODO: mask test doesn't work + Combine(Values(Tolerance_FloatRel_IntAbs(1e-6, 1).to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, /*CV_16SC1,*/ CV_32FC1 ), + Values( -1, CV_8U, CV_16U, CV_32F ), + Values(1.0), + Values(cv::compile_args(CORE_GPU)))); + +// FIXIT: mask test on GPU doesn't work [3032] INSTANTIATE_TEST_CASE_P(DISABLED_MaskPerfTestGPU, MaskPerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_16UC1, CV_16SC1), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_16UC1, CV_16SC1), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(MeanPerfTestGPU, MeanPerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsToleranceScalar(0.0).to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(Polar2CartPerfTestGPU, Polar2CartPerfTest, - Combine(Values(Tolerance_FloatRel_IntAbs(1e-5, 2).to_compare_f()), - Values( szSmall128, szVGA, sz720p, sz1080p ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(Tolerance_FloatRel_IntAbs(1e-5, 2).to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(Cart2PolarPerfTestGPU, Cart2PolarPerfTest, - Combine(Values(Tolerance_FloatRel_IntAbs(1e-2, 2).to_compare_f()), - Values( szSmall128, szVGA, sz720p, sz1080p ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(Tolerance_FloatRel_IntAbs(1e-2, 2).to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(CmpPerfTestGPU, CmpPerfTest, - Combine(Values(CMP_EQ, CMP_GE, CMP_NE, CMP_GT, CMP_LT, CMP_LE), - Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(CMP_EQ, CMP_GE, CMP_NE, CMP_GT, CMP_LT, CMP_LE), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(CmpWithScalarPerfTestGPU, CmpWithScalarPerfTest, - Combine(Values(AbsExact().to_compare_f()), - Values(CMP_EQ, CMP_GE, CMP_NE, CMP_GT, CMP_LT, CMP_LE), - Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(CMP_EQ, CMP_GE, CMP_NE, CMP_GT, CMP_LT, CMP_LE), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(BitwisePerfTestGPU, BitwisePerfTest, - Combine(Values(AND, OR, XOR), - testing::Bool(), - Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(AND, OR, XOR), + testing::Bool(), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(BitwiseNotPerfTestGPU, BitwiseNotPerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(SelectPerfTestGPU, SelectPerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(MinPerfTestGPU, MinPerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(MaxPerfTestGPU, MaxPerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(AbsDiffPerfTestGPU, AbsDiffPerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(AbsDiffCPerfTestGPU, AbsDiffCPerfTest, - Combine(Values(AbsExact().to_compare_f()), - Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(SumPerfTestGPU, SumPerfTest, - Combine(Values(AbsToleranceScalar(1e-5).to_compare_f()), - Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsToleranceScalar(1e-5).to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(CountNonZeroPerfTestGPU, CountNonZeroPerfTest, - Combine(Values(AbsToleranceScalar(0.0).to_compare_f()), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_16UC1, CV_16SC1, CV_32FC1), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsToleranceScalar(0.0).to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(AddWeightedPerfTestGPU, AddWeightedPerfTest, - Combine(Values(Tolerance_FloatRel_IntAbs(1e-6, 1).to_compare_f()), - Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values( -1, CV_8U, CV_16U, CV_32F ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(Tolerance_FloatRel_IntAbs(1e-6, 1).to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values( -1, CV_8U, CV_16U, CV_32F ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(NormPerfTestGPU, NormPerfTest, - Combine(Values(AbsToleranceScalar(1e-5).to_compare_f()), - Values(NORM_INF, NORM_L1, NORM_L2), - Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsToleranceScalar(1e-5).to_compare_f()), + Values(NORM_INF, NORM_L1, NORM_L2), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(IntegralPerfTestGPU, IntegralPerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(ThresholdPerfTestGPU, ThresholdPerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values(cv::THRESH_BINARY, cv::THRESH_BINARY_INV, cv::THRESH_TRUNC, cv::THRESH_TOZERO, cv::THRESH_TOZERO_INV), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values(cv::THRESH_BINARY, cv::THRESH_BINARY_INV, + cv::THRESH_TRUNC, cv::THRESH_TOZERO, cv::THRESH_TOZERO_INV), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(ThresholdPerfTestGPU, ThresholdOTPerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1 ), - Values(cv::THRESH_OTSU, cv::THRESH_TRIANGLE), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1 ), + Values(cv::THRESH_OTSU, cv::THRESH_TRIANGLE), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(InRangePerfTestGPU, InRangePerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1 ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1 ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(Split3PerfTestGPU, Split3PerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(Split4PerfTestGPU, Split4PerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(Merge3PerfTestGPU, Merge3PerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(Merge4PerfTestGPU, Merge4PerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(RemapPerfTestGPU, RemapPerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(FlipPerfTestGPU, FlipPerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values(0,1,-1), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values(0,1,-1), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(CropPerfTestGPU, CropPerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values(cv::Rect(10, 8, 20, 35), cv::Rect(4, 10, 37, 50)), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values(cv::Rect(10, 8, 20, 35), cv::Rect(4, 10, 37, 50)), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(ConcatHorPerfTestGPU, ConcatHorPerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(ConcatVertPerfTestGPU, ConcatVertPerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values(cv::compile_args(CORE_GPU)))); -//TODO: fix this backend to allow ConcatVertVec ConcatHorVec -INSTANTIATE_TEST_CASE_P(DISABLED_ConcatHorVecPerfTestGPU, ConcatHorVecPerfTest, - Combine(Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), - Values(cv::compile_args(CORE_GPU)))); +INSTANTIATE_TEST_CASE_P(ConcatHorVecPerfTestGPU, ConcatHorVecPerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1), + Values(cv::compile_args(CORE_GPU)))); - -INSTANTIATE_TEST_CASE_P(DISABLED_ConcatVertVecPerfTestGPU, ConcatVertVecPerfTest, - Combine(Values( szSmall128, szVGA, sz720p, sz1080p ), - Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), - Values(cv::compile_args(CORE_GPU)))); +INSTANTIATE_TEST_CASE_P(ConcatVertVecPerfTestGPU, ConcatVertVecPerfTest, + Combine(Values(AbsExact().to_compare_f()), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values( CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1, CV_32FC1 ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(LUTPerfTestGPU, LUTPerfTest, - Combine(Values(CV_8UC1, CV_8UC3), - Values(CV_8UC1), - Values( szSmall128, szVGA, sz720p, sz1080p ), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(CV_8UC1, CV_8UC3), + Values(CV_8UC1), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(LUTPerfTestCustomGPU, LUTPerfTest, - Combine(Values(CV_8UC3), - Values(CV_8UC3), - Values( szSmall128, szVGA, sz720p, sz1080p ), - Values(cv::compile_args(CORE_GPU)))); - + Combine(Values(AbsExact().to_compare_f()), + Values(CV_8UC3), + Values(CV_8UC3), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(ConvertToPerfTestGPU, ConvertToPerfTest, - Combine(Values(AbsExact().to_compare_f()), - Values(CV_8UC3, CV_8UC1, CV_16UC1, CV_32FC1), - Values(CV_8U, CV_16U, CV_16S, CV_32F), - Values( szSmall128, szVGA, sz720p, sz1080p ), - Values(2.5, 1.0), - Values(0.0), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(CV_8UC3, CV_8UC1, CV_16UC1, CV_32FC1), + Values(CV_8U, CV_16U, CV_16S, CV_32F), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values(2.5, 1.0), + Values(0.0), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(TransposePerfTestGPU, TransposePerfTest, - Combine(Values(AbsExact().to_compare_f()), - Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_16UC1, CV_16SC1, CV_32FC1, - CV_8UC2, CV_16UC2, CV_16SC2, CV_32FC2, - CV_8UC3, CV_16UC3, CV_16SC3, CV_32FC3), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsExact().to_compare_f()), + Values(szSmall128, szVGA, sz720p, sz1080p), + Values(CV_8UC1, CV_16UC1, CV_16SC1, CV_32FC1, + CV_8UC2, CV_16UC2, CV_16SC2, CV_32FC2, + CV_8UC3, CV_16UC3, CV_16SC3, CV_32FC3), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(ResizePerfTestGPU, ResizePerfTest, - Combine(Values(AbsSimilarPoints(2, 0.05).to_compare_f()), - Values(CV_8UC1, CV_16UC1, CV_16SC1), - Values(cv::INTER_NEAREST, cv::INTER_LINEAR, cv::INTER_AREA), - Values( szSmall128, szVGA, sz720p, sz1080p ), - Values(cv::Size(64,64), - cv::Size(30,30)), - Values(cv::compile_args(CORE_GPU)))); + Combine(Values(AbsSimilarPoints(2, 0.05).to_compare_f()), + Values(CV_8UC1, CV_16UC1, CV_16SC1), + Values(cv::INTER_NEAREST, cv::INTER_LINEAR, cv::INTER_AREA), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values(cv::Size(64,64), + cv::Size(30,30)), + Values(cv::compile_args(CORE_GPU)))); INSTANTIATE_TEST_CASE_P(ResizeFxFyPerfTestGPU, ResizeFxFyPerfTest, - Combine(Values(AbsSimilarPoints(2, 0.05).to_compare_f()), - Values(CV_8UC1, CV_16UC1, CV_16SC1), - Values(cv::INTER_NEAREST, cv::INTER_LINEAR, cv::INTER_AREA), - Values( szSmall128, szVGA, sz720p, sz1080p ), - Values(0.5, 0.1), - Values(0.5, 0.1), - Values(cv::compile_args(CORE_GPU)))); -} + Combine(Values(AbsSimilarPoints(2, 0.05).to_compare_f()), + Values(CV_8UC1, CV_16UC1, CV_16SC1), + Values(cv::INTER_NEAREST, cv::INTER_LINEAR, cv::INTER_AREA), + Values( szSmall128, szVGA, sz720p, sz1080p ), + Values(0.5, 0.1), + Values(0.5, 0.1), + Values(cv::compile_args(CORE_GPU)))); +} // opencv_test From 72c55b56f25422d00cbb43baebe3a204d64b019d Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 11 Dec 2021 17:28:21 +0000 Subject: [PATCH 167/226] imgproc: catch NaNs in clip(), use table index debug check - no NaN propagation guarantee --- modules/imgproc/src/color_lab.cpp | 7 ++++++- modules/imgproc/test/test_color.cpp | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/modules/imgproc/src/color_lab.cpp b/modules/imgproc/src/color_lab.cpp index 6b03be6195..14df6473f4 100644 --- a/modules/imgproc/src/color_lab.cpp +++ b/modules/imgproc/src/color_lab.cpp @@ -978,8 +978,9 @@ static struct LUVLUT_T { const long long int *LvToVpl_b; } LUVLUT = {0, 0, 0}; +/* NB: no NaN propagation guarantee */ #define clip(value) \ - value < 0.0f ? 0.0f : value > 1.0f ? 1.0f : value; + value < 0.0f ? 0.0f : value <= 1.0f ? value : 1.0f; //all constants should be presented through integers to keep bit-exactness static const softdouble gammaThreshold = softdouble(809)/softdouble(20000); // 0.04045 @@ -1330,6 +1331,10 @@ static inline void trilinearInterpolate(int cx, int cy, int cz, const int16_t* L int ty = cy >> (lab_base_shift - lab_lut_shift); int tz = cz >> (lab_base_shift - lab_lut_shift); + CV_DbgCheck(tx, tx >= 0 && tx < LAB_LUT_DIM, ""); + CV_DbgCheck(ty, ty >= 0 && ty < LAB_LUT_DIM, ""); + CV_DbgCheck(tz, tz >= 0 && tz < LAB_LUT_DIM, ""); + const int16_t* baseLUT = &LUT[3*8*tx + (3*8*LAB_LUT_DIM)*ty + (3*8*LAB_LUT_DIM*LAB_LUT_DIM)*tz]; int aa[8], bb[8], cc[8]; for(int i = 0; i < 8; i++) diff --git a/modules/imgproc/test/test_color.cpp b/modules/imgproc/test/test_color.cpp index f828dac18a..0c4fdfb2ca 100644 --- a/modules/imgproc/test/test_color.cpp +++ b/modules/imgproc/test/test_color.cpp @@ -3117,5 +3117,25 @@ TEST(ImgProc_cvtColorTwoPlane, y_plane_padding_differs_from_uv_plane_padding_170 EXPECT_DOUBLE_EQ(cvtest::norm(rgb_reference_mat, rgb_uv_padded_mat, NORM_INF), .0); } +TEST(ImgProc_RGB2Lab, NaN_21111) +{ + const float kNaN = std::numeric_limits::quiet_NaN(); + cv::Mat3f src(1, 111, Vec3f::all(kNaN)), dst; + // Make some entries with only one NaN. + src(0, 0) = src(0, 27) = src(0, 81) = src(0, 108) = cv::Vec3f(0, 0, kNaN); + src(0, 1) = src(0, 28) = src(0, 82) = src(0, 109) = cv::Vec3f(0, kNaN, 0); + src(0, 2) = src(0, 29) = src(0, 83) = src(0, 110) = cv::Vec3f(kNaN, 0, 0); + EXPECT_NO_THROW(cvtColor(src, dst, COLOR_RGB2Lab)); + +#if 0 // no NaN propagation guarantee + for (int i = 0; i < 20; ++i) + { + for (int j = 0; j < 3; ++j) + { + EXPECT_TRUE(cvIsNaN(dst(0, i)[j])); + } + } +#endif +} }} // namespace From 4827fe86bbfd657a3a536ab2905b63a5e5172718 Mon Sep 17 00:00:00 2001 From: rogday Date: Tue, 14 Dec 2021 19:58:06 +0300 Subject: [PATCH 168/226] Merge pull request #21088 from rogday:onnx_tests Onnx conformance tests * Add ONNX conformance tests * dnn(test): add filters for ONNX conformance tests * add filter lists for OCV backend * address review comments * move test_clip_inbounds to all_denylist * address clip issue * avoid empty lists Co-authored-by: Alexander Alekhin --- modules/dnn/src/onnx/onnx_importer.cpp | 1 + modules/dnn/test/test_common.hpp | 6 +- modules/dnn/test/test_common.impl.hpp | 12 +- modules/dnn/test/test_onnx_conformance.cpp | 1250 +++++++++++++++++ ...e_layer_filter_opencv_all_denylist.inl.hpp | 23 + ...e_layer_filter_opencv_cpu_denylist.inl.hpp | 0 ...er_filter_opencv_ocl_fp16_denylist.inl.hpp | 21 + ...er_filter_opencv_ocl_fp32_denylist.inl.hpp | 2 + ..._conformance_layer_parser_denylist.inl.hpp | 717 ++++++++++ modules/ts/misc/testlog_parser.py | 5 +- 10 files changed, 2034 insertions(+), 3 deletions(-) create mode 100644 modules/dnn/test/test_onnx_conformance.cpp create mode 100644 modules/dnn/test/test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp create mode 100644 modules/dnn/test/test_onnx_conformance_layer_filter_opencv_cpu_denylist.inl.hpp create mode 100644 modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp16_denylist.inl.hpp create mode 100644 modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp32_denylist.inl.hpp create mode 100644 modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index f776bdc5da..8ff91f5e44 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -1205,6 +1205,7 @@ void ONNXImporter::parseImageScaler(LayerParams& layerParams, const opencv_onnx: void ONNXImporter::parseClip(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { + CV_CheckEQ(node_proto.input_size(), 1, ""); layerParams.type = "ReLU6"; layerParams.set("min_value", layerParams.get("min", -FLT_MAX)); layerParams.set("max_value", layerParams.get("max", FLT_MAX)); diff --git a/modules/dnn/test/test_common.hpp b/modules/dnn/test/test_common.hpp index 02a676da36..f2573f7562 100644 --- a/modules/dnn/test/test_common.hpp +++ b/modules/dnn/test/test_common.hpp @@ -18,8 +18,9 @@ #define INF_ENGINE_VER_MAJOR_LE(ver) (((INF_ENGINE_RELEASE) / 10000) <= ((ver) / 10000)) #define INF_ENGINE_VER_MAJOR_EQ(ver) (((INF_ENGINE_RELEASE) / 10000) == ((ver) / 10000)) - +#define CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND "dnn_skip_opencv_backend" #define CV_TEST_TAG_DNN_SKIP_HALIDE "dnn_skip_halide" +#define CV_TEST_TAG_DNN_SKIP_CPU "dnn_skip_cpu" #define CV_TEST_TAG_DNN_SKIP_OPENCL "dnn_skip_ocl" #define CV_TEST_TAG_DNN_SKIP_OPENCL_FP16 "dnn_skip_ocl_fp16" #define CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER "dnn_skip_ie_nn_builder" @@ -38,6 +39,9 @@ #define CV_TEST_TAG_DNN_SKIP_IE_MYRIAD CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2, CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X #define CV_TEST_TAG_DNN_SKIP_IE_ARM_CPU "dnn_skip_ie_arm_cpu" +#define CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE "dnn_skip_onnx_conformance" +#define CV_TEST_TAG_DNN_SKIP_PARSER "dnn_skip_parser" + #ifdef HAVE_INF_ENGINE #if INF_ENGINE_VER_MAJOR_EQ(2018050000) diff --git a/modules/dnn/test/test_common.impl.hpp b/modules/dnn/test/test_common.impl.hpp index a11c6641b2..a55a8f788c 100644 --- a/modules/dnn/test/test_common.impl.hpp +++ b/modules/dnn/test/test_common.impl.hpp @@ -361,9 +361,15 @@ void initDNNTests() cvtest::addDataSearchPath(extraTestDataPath); registerGlobalSkipTag( - CV_TEST_TAG_DNN_SKIP_HALIDE, + CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, + CV_TEST_TAG_DNN_SKIP_CPU, CV_TEST_TAG_DNN_SKIP_OPENCL, CV_TEST_TAG_DNN_SKIP_OPENCL_FP16 ); +#if defined(HAVE_HALIDE) + registerGlobalSkipTag( + CV_TEST_TAG_DNN_SKIP_HALIDE + ) +#endif #if defined(INF_ENGINE_RELEASE) registerGlobalSkipTag( CV_TEST_TAG_DNN_SKIP_IE, @@ -392,6 +398,10 @@ void initDNNTests() CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16 ); #endif + registerGlobalSkipTag( + CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE, + CV_TEST_TAG_DNN_SKIP_PARSER + ); } } // namespace diff --git a/modules/dnn/test/test_onnx_conformance.cpp b/modules/dnn/test/test_onnx_conformance.cpp new file mode 100644 index 0000000000..a1d83bd614 --- /dev/null +++ b/modules/dnn/test/test_onnx_conformance.cpp @@ -0,0 +1,1250 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + + +#include "test_precomp.hpp" +#include +#include +#include "npy_blob.hpp" +#include + +#if defined(_MSC_VER) // workaround for 32-bit MSVC compiler +#pragma optimize("", off) +#endif + + +#define CV_TEST_TAG_DNN_ERROR_PARSER "dnn_error_parser" +#define CV_TEST_TAG_DNN_ERROR_NET_SETUP "dnn_error_net_setup" +#define CV_TEST_TAG_DNN_ERROR_FORWARD "dnn_error_forward" +#define CV_TEST_TAG_DNN_LAYER_FALLBACK "dnn_layer_fallback" +#define CV_TEST_TAG_DNN_NO_ACCURACY_CHECK "dnn_no_accuracy_check" + + +namespace opencv_test { + +struct TestCase +{ + const char* name; + uint32_t inputs; + uint32_t outputs; +}; + +static const TestCase testConformanceConfig[] = { + {"test_abs", 1, 1}, + {"test_acos", 1, 1}, + {"test_acos_example", 1, 1}, + {"test_acosh", 1, 1}, + {"test_acosh_example", 1, 1}, + {"test_adagrad", 5, 2}, + {"test_adagrad_multiple", 8, 4}, + {"test_adam", 6, 3}, + {"test_adam_multiple", 10, 6}, + {"test_add", 2, 1}, + {"test_add_bcast", 2, 1}, + {"test_add_uint8", 2, 1}, + {"test_and2d", 2, 1}, + {"test_and3d", 2, 1}, + {"test_and4d", 2, 1}, + {"test_and_bcast3v1d", 2, 1}, + {"test_and_bcast3v2d", 2, 1}, + {"test_and_bcast4v2d", 2, 1}, + {"test_and_bcast4v3d", 2, 1}, + {"test_and_bcast4v4d", 2, 1}, + {"test_argmax_default_axis_example", 1, 1}, + {"test_argmax_default_axis_example_select_last_index", 1, 1}, + {"test_argmax_default_axis_random", 1, 1}, + {"test_argmax_default_axis_random_select_last_index", 1, 1}, + {"test_argmax_keepdims_example", 1, 1}, + {"test_argmax_keepdims_example_select_last_index", 1, 1}, + {"test_argmax_keepdims_random", 1, 1}, + {"test_argmax_keepdims_random_select_last_index", 1, 1}, + {"test_argmax_negative_axis_keepdims_example", 1, 1}, + {"test_argmax_negative_axis_keepdims_example_select_last_index", 1, 1}, + {"test_argmax_negative_axis_keepdims_random", 1, 1}, + {"test_argmax_negative_axis_keepdims_random_select_last_index", 1, 1}, + {"test_argmax_no_keepdims_example", 1, 1}, + {"test_argmax_no_keepdims_example_select_last_index", 1, 1}, + {"test_argmax_no_keepdims_random", 1, 1}, + {"test_argmax_no_keepdims_random_select_last_index", 1, 1}, + {"test_argmin_default_axis_example", 1, 1}, + {"test_argmin_default_axis_example_select_last_index", 1, 1}, + {"test_argmin_default_axis_random", 1, 1}, + {"test_argmin_default_axis_random_select_last_index", 1, 1}, + {"test_argmin_keepdims_example", 1, 1}, + {"test_argmin_keepdims_example_select_last_index", 1, 1}, + {"test_argmin_keepdims_random", 1, 1}, + {"test_argmin_keepdims_random_select_last_index", 1, 1}, + {"test_argmin_negative_axis_keepdims_example", 1, 1}, + {"test_argmin_negative_axis_keepdims_example_select_last_index", 1, 1}, + {"test_argmin_negative_axis_keepdims_random", 1, 1}, + {"test_argmin_negative_axis_keepdims_random_select_last_index", 1, 1}, + {"test_argmin_no_keepdims_example", 1, 1}, + {"test_argmin_no_keepdims_example_select_last_index", 1, 1}, + {"test_argmin_no_keepdims_random", 1, 1}, + {"test_argmin_no_keepdims_random_select_last_index", 1, 1}, + {"test_asin", 1, 1}, + {"test_asin_example", 1, 1}, + {"test_asinh", 1, 1}, + {"test_asinh_example", 1, 1}, + {"test_atan", 1, 1}, + {"test_atan_example", 1, 1}, + {"test_atanh", 1, 1}, + {"test_atanh_example", 1, 1}, + {"test_averagepool_1d_default", 1, 1}, + {"test_averagepool_2d_ceil", 1, 1}, + {"test_averagepool_2d_default", 1, 1}, + {"test_averagepool_2d_pads", 1, 1}, + {"test_averagepool_2d_pads_count_include_pad", 1, 1}, + {"test_averagepool_2d_precomputed_pads", 1, 1}, + {"test_averagepool_2d_precomputed_pads_count_include_pad", 1, 1}, + {"test_averagepool_2d_precomputed_same_upper", 1, 1}, + {"test_averagepool_2d_precomputed_strides", 1, 1}, + {"test_averagepool_2d_same_lower", 1, 1}, + {"test_averagepool_2d_same_upper", 1, 1}, + {"test_averagepool_2d_strides", 1, 1}, + {"test_averagepool_3d_default", 1, 1}, + {"test_basic_conv_with_padding", 2, 1}, + {"test_basic_conv_without_padding", 2, 1}, + {"test_basic_convinteger", 3, 1}, + {"test_batchnorm_epsilon", 5, 1}, + {"test_batchnorm_epsilon_training_mode", 5, 3}, + {"test_batchnorm_example", 5, 1}, + {"test_batchnorm_example_training_mode", 5, 3}, + {"test_bernoulli", 1, 1}, + {"test_bernoulli_double", 1, 1}, + {"test_bernoulli_double_expanded", 1, 1}, + {"test_bernoulli_expanded", 1, 1}, + {"test_bernoulli_seed", 1, 1}, + {"test_bernoulli_seed_expanded", 1, 1}, + {"test_bitshift_left_uint16", 2, 1}, + {"test_bitshift_left_uint32", 2, 1}, + {"test_bitshift_left_uint64", 2, 1}, + {"test_bitshift_left_uint8", 2, 1}, + {"test_bitshift_right_uint16", 2, 1}, + {"test_bitshift_right_uint32", 2, 1}, + {"test_bitshift_right_uint64", 2, 1}, + {"test_bitshift_right_uint8", 2, 1}, + {"test_cast_BFLOAT16_to_FLOAT", 1, 1}, + {"test_cast_DOUBLE_to_FLOAT", 1, 1}, + {"test_cast_DOUBLE_to_FLOAT16", 1, 1}, + {"test_cast_FLOAT16_to_DOUBLE", 1, 1}, + {"test_cast_FLOAT16_to_FLOAT", 1, 1}, + {"test_cast_FLOAT_to_BFLOAT16", 1, 1}, + {"test_cast_FLOAT_to_DOUBLE", 1, 1}, + {"test_cast_FLOAT_to_FLOAT16", 1, 1}, + {"test_cast_FLOAT_to_STRING", 1, 1}, + {"test_cast_STRING_to_FLOAT", 1, 1}, + {"test_castlike_BFLOAT16_to_FLOAT", 2, 1}, + {"test_castlike_BFLOAT16_to_FLOAT_expanded", 2, 1}, + {"test_castlike_DOUBLE_to_FLOAT", 2, 1}, + {"test_castlike_DOUBLE_to_FLOAT16", 2, 1}, + {"test_castlike_DOUBLE_to_FLOAT16_expanded", 2, 1}, + {"test_castlike_DOUBLE_to_FLOAT_expanded", 2, 1}, + {"test_castlike_FLOAT16_to_DOUBLE", 2, 1}, + {"test_castlike_FLOAT16_to_DOUBLE_expanded", 2, 1}, + {"test_castlike_FLOAT16_to_FLOAT", 2, 1}, + {"test_castlike_FLOAT16_to_FLOAT_expanded", 2, 1}, + {"test_castlike_FLOAT_to_BFLOAT16", 2, 1}, + {"test_castlike_FLOAT_to_BFLOAT16_expanded", 2, 1}, + {"test_castlike_FLOAT_to_DOUBLE", 2, 1}, + {"test_castlike_FLOAT_to_DOUBLE_expanded", 2, 1}, + {"test_castlike_FLOAT_to_FLOAT16", 2, 1}, + {"test_castlike_FLOAT_to_FLOAT16_expanded", 2, 1}, + {"test_castlike_FLOAT_to_STRING", 2, 1}, + {"test_castlike_FLOAT_to_STRING_expanded", 2, 1}, + {"test_castlike_STRING_to_FLOAT", 2, 1}, + {"test_castlike_STRING_to_FLOAT_expanded", 2, 1}, + {"test_ceil", 1, 1}, + {"test_ceil_example", 1, 1}, + {"test_celu", 1, 1}, + {"test_celu_expanded", 1, 1}, + {"test_clip", 3, 1}, + {"test_clip_default_inbounds", 1, 1}, + {"test_clip_default_int8_inbounds", 1, 1}, + {"test_clip_default_int8_max", 2, 1}, + {"test_clip_default_int8_min", 2, 1}, + {"test_clip_default_max", 2, 1}, + {"test_clip_default_min", 2, 1}, + {"test_clip_example", 3, 1}, + {"test_clip_inbounds", 3, 1}, + {"test_clip_outbounds", 3, 1}, + {"test_clip_splitbounds", 3, 1}, + {"test_compress_0", 2, 1}, + {"test_compress_1", 2, 1}, + {"test_compress_default_axis", 2, 1}, + {"test_compress_negative_axis", 2, 1}, + {"test_concat_1d_axis_0", 2, 1}, + {"test_concat_1d_axis_negative_1", 2, 1}, + {"test_concat_2d_axis_0", 2, 1}, + {"test_concat_2d_axis_1", 2, 1}, + {"test_concat_2d_axis_negative_1", 2, 1}, + {"test_concat_2d_axis_negative_2", 2, 1}, + {"test_concat_3d_axis_0", 2, 1}, + {"test_concat_3d_axis_1", 2, 1}, + {"test_concat_3d_axis_2", 2, 1}, + {"test_concat_3d_axis_negative_1", 2, 1}, + {"test_concat_3d_axis_negative_2", 2, 1}, + {"test_concat_3d_axis_negative_3", 2, 1}, + {"test_constant", 0, 1}, + {"test_constant_pad", 3, 1}, + {"test_constantofshape_float_ones", 1, 1}, + {"test_constantofshape_int_shape_zero", 1, 1}, + {"test_constantofshape_int_zeros", 1, 1}, + {"test_conv_with_autopad_same", 2, 1}, + {"test_conv_with_strides_and_asymmetric_padding", 2, 1}, + {"test_conv_with_strides_no_padding", 2, 1}, + {"test_conv_with_strides_padding", 2, 1}, + {"test_convinteger_with_padding", 3, 1}, + {"test_convinteger_without_padding", 3, 1}, + {"test_convtranspose", 2, 1}, + {"test_convtranspose_1d", 2, 1}, + {"test_convtranspose_3d", 2, 1}, + {"test_convtranspose_autopad_same", 2, 1}, + {"test_convtranspose_dilations", 2, 1}, + {"test_convtranspose_kernel_shape", 2, 1}, + {"test_convtranspose_output_shape", 2, 1}, + {"test_convtranspose_pad", 2, 1}, + {"test_convtranspose_pads", 2, 1}, + {"test_convtranspose_with_kernel", 2, 1}, + {"test_cos", 1, 1}, + {"test_cos_example", 1, 1}, + {"test_cosh", 1, 1}, + {"test_cosh_example", 1, 1}, + {"test_cumsum_1d", 2, 1}, + {"test_cumsum_1d_exclusive", 2, 1}, + {"test_cumsum_1d_reverse", 2, 1}, + {"test_cumsum_1d_reverse_exclusive", 2, 1}, + {"test_cumsum_2d_axis_0", 2, 1}, + {"test_cumsum_2d_axis_1", 2, 1}, + {"test_cumsum_2d_negative_axis", 2, 1}, + {"test_depthtospace_crd_mode", 1, 1}, + {"test_depthtospace_crd_mode_example", 1, 1}, + {"test_depthtospace_dcr_mode", 1, 1}, + {"test_depthtospace_example", 1, 1}, + {"test_dequantizelinear", 3, 1}, + {"test_dequantizelinear_axis", 3, 1}, + {"test_det_2d", 1, 1}, + {"test_det_nd", 1, 1}, + {"test_div", 2, 1}, + {"test_div_bcast", 2, 1}, + {"test_div_example", 2, 1}, + {"test_div_uint8", 2, 1}, + {"test_dropout_default", 1, 1}, + {"test_dropout_default_mask", 1, 2}, + {"test_dropout_default_mask_ratio", 2, 2}, + {"test_dropout_default_old", 1, 1}, + {"test_dropout_default_ratio", 2, 1}, + {"test_dropout_random_old", 1, 1}, + {"test_dynamicquantizelinear", 1, 3}, + {"test_dynamicquantizelinear_expanded", 1, 3}, + {"test_dynamicquantizelinear_max_adjusted", 1, 3}, + {"test_dynamicquantizelinear_max_adjusted_expanded", 1, 3}, + {"test_dynamicquantizelinear_min_adjusted", 1, 3}, + {"test_dynamicquantizelinear_min_adjusted_expanded", 1, 3}, + {"test_edge_pad", 2, 1}, + {"test_einsum_batch_diagonal", 1, 1}, + {"test_einsum_batch_matmul", 2, 1}, + {"test_einsum_inner_prod", 2, 1}, + {"test_einsum_sum", 1, 1}, + {"test_einsum_transpose", 1, 1}, + {"test_elu", 1, 1}, + {"test_elu_default", 1, 1}, + {"test_elu_example", 1, 1}, + {"test_equal", 2, 1}, + {"test_equal_bcast", 2, 1}, + {"test_erf", 1, 1}, + {"test_exp", 1, 1}, + {"test_exp_example", 1, 1}, + {"test_expand_dim_changed", 2, 1}, + {"test_expand_dim_unchanged", 2, 1}, + {"test_eyelike_populate_off_main_diagonal", 1, 1}, + {"test_eyelike_with_dtype", 1, 1}, + {"test_eyelike_without_dtype", 1, 1}, + {"test_flatten_axis0", 1, 1}, + {"test_flatten_axis1", 1, 1}, + {"test_flatten_axis2", 1, 1}, + {"test_flatten_axis3", 1, 1}, + {"test_flatten_default_axis", 1, 1}, + {"test_flatten_negative_axis1", 1, 1}, + {"test_flatten_negative_axis2", 1, 1}, + {"test_flatten_negative_axis3", 1, 1}, + {"test_flatten_negative_axis4", 1, 1}, + {"test_floor", 1, 1}, + {"test_floor_example", 1, 1}, + {"test_gather_0", 2, 1}, + {"test_gather_1", 2, 1}, + {"test_gather_2d_indices", 2, 1}, + {"test_gather_elements_0", 2, 1}, + {"test_gather_elements_1", 2, 1}, + {"test_gather_elements_negative_indices", 2, 1}, + {"test_gather_negative_indices", 2, 1}, + {"test_gathernd_example_float32", 2, 1}, + {"test_gathernd_example_int32", 2, 1}, + {"test_gathernd_example_int32_batch_dim1", 2, 1}, + {"test_gemm_all_attributes", 3, 1}, + {"test_gemm_alpha", 3, 1}, + {"test_gemm_beta", 3, 1}, + {"test_gemm_default_matrix_bias", 3, 1}, + {"test_gemm_default_no_bias", 2, 1}, + {"test_gemm_default_scalar_bias", 3, 1}, + {"test_gemm_default_single_elem_vector_bias", 3, 1}, + {"test_gemm_default_vector_bias", 3, 1}, + {"test_gemm_default_zero_bias", 3, 1}, + {"test_gemm_transposeA", 3, 1}, + {"test_gemm_transposeB", 3, 1}, + {"test_globalaveragepool", 1, 1}, + {"test_globalaveragepool_precomputed", 1, 1}, + {"test_globalmaxpool", 1, 1}, + {"test_globalmaxpool_precomputed", 1, 1}, + {"test_greater", 2, 1}, + {"test_greater_bcast", 2, 1}, + {"test_greater_equal", 2, 1}, + {"test_greater_equal_bcast", 2, 1}, + {"test_greater_equal_bcast_expanded", 2, 1}, + {"test_greater_equal_expanded", 2, 1}, + {"test_gridsample", 2, 1}, + {"test_gridsample_aligncorners_true", 2, 1}, + {"test_gridsample_bicubic", 2, 1}, + {"test_gridsample_bilinear", 2, 1}, + {"test_gridsample_border_padding", 2, 1}, + {"test_gridsample_nearest", 2, 1}, + {"test_gridsample_reflection_padding", 2, 1}, + {"test_gridsample_zeros_padding", 2, 1}, + {"test_gru_batchwise", 3, 2}, + {"test_gru_defaults", 3, 1}, + {"test_gru_seq_length", 4, 1}, + {"test_gru_with_initial_bias", 4, 1}, + {"test_hardmax_axis_0", 1, 1}, + {"test_hardmax_axis_1", 1, 1}, + {"test_hardmax_axis_2", 1, 1}, + {"test_hardmax_default_axis", 1, 1}, + {"test_hardmax_example", 1, 1}, + {"test_hardmax_negative_axis", 1, 1}, + {"test_hardmax_one_hot", 1, 1}, + {"test_hardsigmoid", 1, 1}, + {"test_hardsigmoid_default", 1, 1}, + {"test_hardsigmoid_example", 1, 1}, + {"test_hardswish", 1, 1}, + {"test_hardswish_expanded", 1, 1}, + {"test_identity", 1, 1}, + {"test_identity_opt", 1, 1}, + {"test_identity_sequence", 1, 1}, + {"test_if", 1, 1}, + {"test_if_opt", 1, 1}, + {"test_if_seq", 1, 1}, + {"test_instancenorm_epsilon", 3, 1}, + {"test_instancenorm_example", 3, 1}, + {"test_isinf", 1, 1}, + {"test_isinf_negative", 1, 1}, + {"test_isinf_positive", 1, 1}, + {"test_isnan", 1, 1}, + {"test_leakyrelu", 1, 1}, + {"test_leakyrelu_default", 1, 1}, + {"test_leakyrelu_example", 1, 1}, + {"test_less", 2, 1}, + {"test_less_bcast", 2, 1}, + {"test_less_equal", 2, 1}, + {"test_less_equal_bcast", 2, 1}, + {"test_less_equal_bcast_expanded", 2, 1}, + {"test_less_equal_expanded", 2, 1}, + {"test_log", 1, 1}, + {"test_log_example", 1, 1}, + {"test_logsoftmax_axis_0", 1, 1}, + {"test_logsoftmax_axis_0_expanded", 1, 1}, + {"test_logsoftmax_axis_1", 1, 1}, + {"test_logsoftmax_axis_1_expanded", 1, 1}, + {"test_logsoftmax_axis_2", 1, 1}, + {"test_logsoftmax_axis_2_expanded", 1, 1}, + {"test_logsoftmax_default_axis", 1, 1}, + {"test_logsoftmax_default_axis_expanded", 1, 1}, + {"test_logsoftmax_example_1", 1, 1}, + {"test_logsoftmax_example_1_expanded", 1, 1}, + {"test_logsoftmax_large_number", 1, 1}, + {"test_logsoftmax_large_number_expanded", 1, 1}, + {"test_logsoftmax_negative_axis", 1, 1}, + {"test_logsoftmax_negative_axis_expanded", 1, 1}, + {"test_loop11", 3, 2}, + {"test_loop13_seq", 3, 1}, + {"test_loop16_seq_none", 3, 1}, + {"test_lrn", 1, 1}, + {"test_lrn_default", 1, 1}, + {"test_lstm_batchwise", 3, 2}, + {"test_lstm_defaults", 3, 1}, + {"test_lstm_with_initial_bias", 4, 1}, + {"test_lstm_with_peepholes", 8, 1}, + {"test_matmul_2d", 2, 1}, + {"test_matmul_3d", 2, 1}, + {"test_matmul_4d", 2, 1}, + {"test_matmulinteger", 4, 1}, + {"test_max_example", 3, 1}, + {"test_max_float16", 2, 1}, + {"test_max_float32", 2, 1}, + {"test_max_float64", 2, 1}, + {"test_max_int16", 2, 1}, + {"test_max_int32", 2, 1}, + {"test_max_int64", 2, 1}, + {"test_max_int8", 2, 1}, + {"test_max_one_input", 1, 1}, + {"test_max_two_inputs", 2, 1}, + {"test_max_uint16", 2, 1}, + {"test_max_uint32", 2, 1}, + {"test_max_uint64", 2, 1}, + {"test_max_uint8", 2, 1}, + {"test_maxpool_1d_default", 1, 1}, + {"test_maxpool_2d_ceil", 1, 1}, + {"test_maxpool_2d_default", 1, 1}, + {"test_maxpool_2d_dilations", 1, 1}, + {"test_maxpool_2d_pads", 1, 1}, + {"test_maxpool_2d_precomputed_pads", 1, 1}, + {"test_maxpool_2d_precomputed_same_upper", 1, 1}, + {"test_maxpool_2d_precomputed_strides", 1, 1}, + {"test_maxpool_2d_same_lower", 1, 1}, + {"test_maxpool_2d_same_upper", 1, 1}, + {"test_maxpool_2d_strides", 1, 1}, + {"test_maxpool_2d_uint8", 1, 1}, + {"test_maxpool_3d_default", 1, 1}, + {"test_maxpool_with_argmax_2d_precomputed_pads", 1, 2}, + {"test_maxpool_with_argmax_2d_precomputed_strides", 1, 2}, + {"test_maxunpool_export_with_output_shape", 3, 1}, + {"test_maxunpool_export_without_output_shape", 2, 1}, + {"test_mean_example", 3, 1}, + {"test_mean_one_input", 1, 1}, + {"test_mean_two_inputs", 2, 1}, + {"test_min_example", 3, 1}, + {"test_min_float16", 2, 1}, + {"test_min_float32", 2, 1}, + {"test_min_float64", 2, 1}, + {"test_min_int16", 2, 1}, + {"test_min_int32", 2, 1}, + {"test_min_int64", 2, 1}, + {"test_min_int8", 2, 1}, + {"test_min_one_input", 1, 1}, + {"test_min_two_inputs", 2, 1}, + {"test_min_uint16", 2, 1}, + {"test_min_uint32", 2, 1}, + {"test_min_uint64", 2, 1}, + {"test_min_uint8", 2, 1}, + {"test_mod_broadcast", 2, 1}, + {"test_mod_int64_fmod", 2, 1}, + {"test_mod_mixed_sign_float16", 2, 1}, + {"test_mod_mixed_sign_float32", 2, 1}, + {"test_mod_mixed_sign_float64", 2, 1}, + {"test_mod_mixed_sign_int16", 2, 1}, + {"test_mod_mixed_sign_int32", 2, 1}, + {"test_mod_mixed_sign_int64", 2, 1}, + {"test_mod_mixed_sign_int8", 2, 1}, + {"test_mod_uint16", 2, 1}, + {"test_mod_uint32", 2, 1}, + {"test_mod_uint64", 2, 1}, + {"test_mod_uint8", 2, 1}, + {"test_momentum", 5, 2}, + {"test_momentum_multiple", 8, 4}, + {"test_mul", 2, 1}, + {"test_mul_bcast", 2, 1}, + {"test_mul_example", 2, 1}, + {"test_mul_uint8", 2, 1}, + {"test_mvn", 1, 1}, + {"test_mvn_expanded", 1, 1}, + {"test_neg", 1, 1}, + {"test_neg_example", 1, 1}, + {"test_nesterov_momentum", 5, 2}, + {"test_nllloss_NC", 2, 1}, + {"test_nllloss_NC_expanded", 2, 1}, + {"test_nllloss_NCd1", 2, 1}, + {"test_nllloss_NCd1_expanded", 2, 1}, + {"test_nllloss_NCd1_ii", 2, 1}, + {"test_nllloss_NCd1_ii_expanded", 2, 1}, + {"test_nllloss_NCd1_mean_weight_negative_ii", 3, 1}, + {"test_nllloss_NCd1_mean_weight_negative_ii_expanded", 3, 1}, + {"test_nllloss_NCd1_weight", 3, 1}, + {"test_nllloss_NCd1_weight_expanded", 3, 1}, + {"test_nllloss_NCd1_weight_ii", 3, 1}, + {"test_nllloss_NCd1_weight_ii_expanded", 3, 1}, + {"test_nllloss_NCd1d2", 2, 1}, + {"test_nllloss_NCd1d2_expanded", 2, 1}, + {"test_nllloss_NCd1d2_no_weight_reduction_mean_ii", 2, 1}, + {"test_nllloss_NCd1d2_no_weight_reduction_mean_ii_expanded", 2, 1}, + {"test_nllloss_NCd1d2_reduction_mean", 2, 1}, + {"test_nllloss_NCd1d2_reduction_mean_expanded", 2, 1}, + {"test_nllloss_NCd1d2_reduction_sum", 2, 1}, + {"test_nllloss_NCd1d2_reduction_sum_expanded", 2, 1}, + {"test_nllloss_NCd1d2_with_weight", 3, 1}, + {"test_nllloss_NCd1d2_with_weight_expanded", 3, 1}, + {"test_nllloss_NCd1d2_with_weight_reduction_mean", 3, 1}, + {"test_nllloss_NCd1d2_with_weight_reduction_mean_expanded", 3, 1}, + {"test_nllloss_NCd1d2_with_weight_reduction_sum", 3, 1}, + {"test_nllloss_NCd1d2_with_weight_reduction_sum_expanded", 3, 1}, + {"test_nllloss_NCd1d2_with_weight_reduction_sum_ii", 3, 1}, + {"test_nllloss_NCd1d2_with_weight_reduction_sum_ii_expanded", 3, 1}, + {"test_nllloss_NCd1d2d3_none_no_weight_negative_ii", 2, 1}, + {"test_nllloss_NCd1d2d3_none_no_weight_negative_ii_expanded", 2, 1}, + {"test_nllloss_NCd1d2d3_sum_weight_high_ii", 3, 1}, + {"test_nllloss_NCd1d2d3_sum_weight_high_ii_expanded", 3, 1}, + {"test_nllloss_NCd1d2d3d4d5_mean_weight", 3, 1}, + {"test_nllloss_NCd1d2d3d4d5_mean_weight_expanded", 3, 1}, + {"test_nllloss_NCd1d2d3d4d5_none_no_weight", 2, 1}, + {"test_nllloss_NCd1d2d3d4d5_none_no_weight_expanded", 2, 1}, + {"test_nonmaxsuppression_center_point_box_format", 5, 1}, + {"test_nonmaxsuppression_flipped_coordinates", 5, 1}, + {"test_nonmaxsuppression_identical_boxes", 5, 1}, + {"test_nonmaxsuppression_limit_output_size", 5, 1}, + {"test_nonmaxsuppression_single_box", 5, 1}, + {"test_nonmaxsuppression_suppress_by_IOU", 5, 1}, + {"test_nonmaxsuppression_suppress_by_IOU_and_scores", 5, 1}, + {"test_nonmaxsuppression_two_batches", 5, 1}, + {"test_nonmaxsuppression_two_classes", 5, 1}, + {"test_nonzero_example", 1, 1}, + {"test_not_2d", 1, 1}, + {"test_not_3d", 1, 1}, + {"test_not_4d", 1, 1}, + {"test_onehot_negative_indices", 3, 1}, + {"test_onehot_with_axis", 3, 1}, + {"test_onehot_with_negative_axis", 3, 1}, + {"test_onehot_without_axis", 3, 1}, + {"test_optional_get_element", 1, 1}, + {"test_optional_get_element_sequence", 1, 1}, + {"test_optional_has_element", 1, 1}, + {"test_optional_has_element_empty", 1, 1}, + {"test_or2d", 2, 1}, + {"test_or3d", 2, 1}, + {"test_or4d", 2, 1}, + {"test_or_bcast3v1d", 2, 1}, + {"test_or_bcast3v2d", 2, 1}, + {"test_or_bcast4v2d", 2, 1}, + {"test_or_bcast4v3d", 2, 1}, + {"test_or_bcast4v4d", 2, 1}, + {"test_pow", 2, 1}, + {"test_pow_bcast_array", 2, 1}, + {"test_pow_bcast_scalar", 2, 1}, + {"test_pow_example", 2, 1}, + {"test_pow_types_float", 2, 1}, + {"test_pow_types_float32_int32", 2, 1}, + {"test_pow_types_float32_int64", 2, 1}, + {"test_pow_types_float32_uint32", 2, 1}, + {"test_pow_types_float32_uint64", 2, 1}, + {"test_pow_types_int", 2, 1}, + {"test_pow_types_int32_float32", 2, 1}, + {"test_pow_types_int32_int32", 2, 1}, + {"test_pow_types_int64_float32", 2, 1}, + {"test_pow_types_int64_int64", 2, 1}, + {"test_prelu_broadcast", 2, 1}, + {"test_prelu_example", 2, 1}, + {"test_qlinearconv", 8, 1}, + {"test_qlinearmatmul_2D", 8, 1}, + {"test_qlinearmatmul_3D", 8, 1}, + {"test_quantizelinear", 3, 1}, + {"test_quantizelinear_axis", 3, 1}, + {"test_range_float_type_positive_delta", 3, 1}, + {"test_range_float_type_positive_delta_expanded", 3, 1}, + {"test_range_int32_type_negative_delta", 3, 1}, + {"test_range_int32_type_negative_delta_expanded", 3, 1}, + {"test_reciprocal", 1, 1}, + {"test_reciprocal_example", 1, 1}, + {"test_reduce_l1_default_axes_keepdims_example", 1, 1}, + {"test_reduce_l1_default_axes_keepdims_random", 1, 1}, + {"test_reduce_l1_do_not_keepdims_example", 1, 1}, + {"test_reduce_l1_do_not_keepdims_random", 1, 1}, + {"test_reduce_l1_keep_dims_example", 1, 1}, + {"test_reduce_l1_keep_dims_random", 1, 1}, + {"test_reduce_l1_negative_axes_keep_dims_example", 1, 1}, + {"test_reduce_l1_negative_axes_keep_dims_random", 1, 1}, + {"test_reduce_l2_default_axes_keepdims_example", 1, 1}, + {"test_reduce_l2_default_axes_keepdims_random", 1, 1}, + {"test_reduce_l2_do_not_keepdims_example", 1, 1}, + {"test_reduce_l2_do_not_keepdims_random", 1, 1}, + {"test_reduce_l2_keep_dims_example", 1, 1}, + {"test_reduce_l2_keep_dims_random", 1, 1}, + {"test_reduce_l2_negative_axes_keep_dims_example", 1, 1}, + {"test_reduce_l2_negative_axes_keep_dims_random", 1, 1}, + {"test_reduce_log_sum", 1, 1}, + {"test_reduce_log_sum_asc_axes", 1, 1}, + {"test_reduce_log_sum_default", 1, 1}, + {"test_reduce_log_sum_desc_axes", 1, 1}, + {"test_reduce_log_sum_exp_default_axes_keepdims_example", 1, 1}, + {"test_reduce_log_sum_exp_default_axes_keepdims_random", 1, 1}, + {"test_reduce_log_sum_exp_do_not_keepdims_example", 1, 1}, + {"test_reduce_log_sum_exp_do_not_keepdims_random", 1, 1}, + {"test_reduce_log_sum_exp_keepdims_example", 1, 1}, + {"test_reduce_log_sum_exp_keepdims_random", 1, 1}, + {"test_reduce_log_sum_exp_negative_axes_keepdims_example", 1, 1}, + {"test_reduce_log_sum_exp_negative_axes_keepdims_random", 1, 1}, + {"test_reduce_log_sum_negative_axes", 1, 1}, + {"test_reduce_max_default_axes_keepdim_example", 1, 1}, + {"test_reduce_max_default_axes_keepdims_random", 1, 1}, + {"test_reduce_max_do_not_keepdims_example", 1, 1}, + {"test_reduce_max_do_not_keepdims_random", 1, 1}, + {"test_reduce_max_keepdims_example", 1, 1}, + {"test_reduce_max_keepdims_random", 1, 1}, + {"test_reduce_max_negative_axes_keepdims_example", 1, 1}, + {"test_reduce_max_negative_axes_keepdims_random", 1, 1}, + {"test_reduce_mean_default_axes_keepdims_example", 1, 1}, + {"test_reduce_mean_default_axes_keepdims_random", 1, 1}, + {"test_reduce_mean_do_not_keepdims_example", 1, 1}, + {"test_reduce_mean_do_not_keepdims_random", 1, 1}, + {"test_reduce_mean_keepdims_example", 1, 1}, + {"test_reduce_mean_keepdims_random", 1, 1}, + {"test_reduce_mean_negative_axes_keepdims_example", 1, 1}, + {"test_reduce_mean_negative_axes_keepdims_random", 1, 1}, + {"test_reduce_min_default_axes_keepdims_example", 1, 1}, + {"test_reduce_min_default_axes_keepdims_random", 1, 1}, + {"test_reduce_min_do_not_keepdims_example", 1, 1}, + {"test_reduce_min_do_not_keepdims_random", 1, 1}, + {"test_reduce_min_keepdims_example", 1, 1}, + {"test_reduce_min_keepdims_random", 1, 1}, + {"test_reduce_min_negative_axes_keepdims_example", 1, 1}, + {"test_reduce_min_negative_axes_keepdims_random", 1, 1}, + {"test_reduce_prod_default_axes_keepdims_example", 1, 1}, + {"test_reduce_prod_default_axes_keepdims_random", 1, 1}, + {"test_reduce_prod_do_not_keepdims_example", 1, 1}, + {"test_reduce_prod_do_not_keepdims_random", 1, 1}, + {"test_reduce_prod_keepdims_example", 1, 1}, + {"test_reduce_prod_keepdims_random", 1, 1}, + {"test_reduce_prod_negative_axes_keepdims_example", 1, 1}, + {"test_reduce_prod_negative_axes_keepdims_random", 1, 1}, + {"test_reduce_sum_default_axes_keepdims_example", 2, 1}, + {"test_reduce_sum_default_axes_keepdims_random", 2, 1}, + {"test_reduce_sum_do_not_keepdims_example", 2, 1}, + {"test_reduce_sum_do_not_keepdims_random", 2, 1}, + {"test_reduce_sum_empty_axes_input_noop_example", 2, 1}, + {"test_reduce_sum_empty_axes_input_noop_random", 2, 1}, + {"test_reduce_sum_keepdims_example", 2, 1}, + {"test_reduce_sum_keepdims_random", 2, 1}, + {"test_reduce_sum_negative_axes_keepdims_example", 2, 1}, + {"test_reduce_sum_negative_axes_keepdims_random", 2, 1}, + {"test_reduce_sum_square_default_axes_keepdims_example", 1, 1}, + {"test_reduce_sum_square_default_axes_keepdims_random", 1, 1}, + {"test_reduce_sum_square_do_not_keepdims_example", 1, 1}, + {"test_reduce_sum_square_do_not_keepdims_random", 1, 1}, + {"test_reduce_sum_square_keepdims_example", 1, 1}, + {"test_reduce_sum_square_keepdims_random", 1, 1}, + {"test_reduce_sum_square_negative_axes_keepdims_example", 1, 1}, + {"test_reduce_sum_square_negative_axes_keepdims_random", 1, 1}, + {"test_reflect_pad", 2, 1}, + {"test_relu", 1, 1}, + {"test_reshape_allowzero_reordered", 2, 1}, + {"test_reshape_extended_dims", 2, 1}, + {"test_reshape_negative_dim", 2, 1}, + {"test_reshape_negative_extended_dims", 2, 1}, + {"test_reshape_one_dim", 2, 1}, + {"test_reshape_reduced_dims", 2, 1}, + {"test_reshape_reordered_all_dims", 2, 1}, + {"test_reshape_reordered_last_dims", 2, 1}, + {"test_reshape_zero_and_negative_dim", 2, 1}, + {"test_reshape_zero_dim", 2, 1}, + {"test_resize_downsample_scales_cubic", 2, 1}, + {"test_resize_downsample_scales_cubic_A_n0p5_exclude_outside", 2, 1}, + {"test_resize_downsample_scales_cubic_align_corners", 2, 1}, + {"test_resize_downsample_scales_linear", 2, 1}, + {"test_resize_downsample_scales_linear_align_corners", 2, 1}, + {"test_resize_downsample_scales_nearest", 2, 1}, + {"test_resize_downsample_sizes_cubic", 2, 1}, + {"test_resize_downsample_sizes_linear_pytorch_half_pixel", 2, 1}, + {"test_resize_downsample_sizes_nearest", 2, 1}, + {"test_resize_downsample_sizes_nearest_tf_half_pixel_for_nn", 2, 1}, + {"test_resize_tf_crop_and_resize", 3, 1}, + {"test_resize_upsample_scales_cubic", 2, 1}, + {"test_resize_upsample_scales_cubic_A_n0p5_exclude_outside", 2, 1}, + {"test_resize_upsample_scales_cubic_align_corners", 2, 1}, + {"test_resize_upsample_scales_cubic_asymmetric", 2, 1}, + {"test_resize_upsample_scales_linear", 2, 1}, + {"test_resize_upsample_scales_linear_align_corners", 2, 1}, + {"test_resize_upsample_scales_nearest", 2, 1}, + {"test_resize_upsample_sizes_cubic", 2, 1}, + {"test_resize_upsample_sizes_nearest", 2, 1}, + {"test_resize_upsample_sizes_nearest_ceil_half_pixel", 2, 1}, + {"test_resize_upsample_sizes_nearest_floor_align_corners", 2, 1}, + {"test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric", 2, 1}, + {"test_reversesequence_batch", 2, 1}, + {"test_reversesequence_time", 2, 1}, + {"test_rnn_seq_length", 4, 1}, + {"test_roialign_aligned_false", 3, 1}, + {"test_roialign_aligned_true", 3, 1}, + {"test_round", 1, 1}, + {"test_scan9_sum", 2, 2}, + {"test_scan_sum", 2, 2}, + {"test_scatter_elements_with_axis", 3, 1}, + {"test_scatter_elements_with_duplicate_indices", 3, 1}, + {"test_scatter_elements_with_negative_indices", 3, 1}, + {"test_scatter_elements_without_axis", 3, 1}, + {"test_scatter_with_axis", 3, 1}, + {"test_scatter_without_axis", 3, 1}, + {"test_scatternd", 3, 1}, + {"test_scatternd_add", 3, 1}, + {"test_scatternd_multiply", 3, 1}, + {"test_sce_NCd1_mean_weight_negative_ii", 3, 1}, + {"test_sce_NCd1_mean_weight_negative_ii_expanded", 3, 1}, + {"test_sce_NCd1_mean_weight_negative_ii_log_prob", 3, 2}, + {"test_sce_NCd1_mean_weight_negative_ii_log_prob_expanded", 3, 2}, + {"test_sce_NCd1d2d3_none_no_weight_negative_ii", 2, 1}, + {"test_sce_NCd1d2d3_none_no_weight_negative_ii_expanded", 2, 1}, + {"test_sce_NCd1d2d3_none_no_weight_negative_ii_log_prob", 2, 2}, + {"test_sce_NCd1d2d3_none_no_weight_negative_ii_log_prob_expanded", 2, 2}, + {"test_sce_NCd1d2d3_sum_weight_high_ii", 3, 1}, + {"test_sce_NCd1d2d3_sum_weight_high_ii_expanded", 3, 1}, + {"test_sce_NCd1d2d3_sum_weight_high_ii_log_prob", 3, 2}, + {"test_sce_NCd1d2d3_sum_weight_high_ii_log_prob_expanded", 3, 2}, + {"test_sce_NCd1d2d3d4d5_mean_weight", 3, 1}, + {"test_sce_NCd1d2d3d4d5_mean_weight_expanded", 3, 1}, + {"test_sce_NCd1d2d3d4d5_mean_weight_log_prob", 3, 2}, + {"test_sce_NCd1d2d3d4d5_mean_weight_log_prob_expanded", 3, 2}, + {"test_sce_NCd1d2d3d4d5_none_no_weight", 2, 1}, + {"test_sce_NCd1d2d3d4d5_none_no_weight_expanded", 2, 1}, + {"test_sce_NCd1d2d3d4d5_none_no_weight_log_prob", 2, 2}, + {"test_sce_NCd1d2d3d4d5_none_no_weight_log_prob_expanded", 2, 2}, + {"test_sce_mean", 2, 1}, + {"test_sce_mean_3d", 2, 1}, + {"test_sce_mean_3d_expanded", 2, 1}, + {"test_sce_mean_3d_log_prob", 2, 2}, + {"test_sce_mean_3d_log_prob_expanded", 2, 2}, + {"test_sce_mean_expanded", 2, 1}, + {"test_sce_mean_log_prob", 2, 2}, + {"test_sce_mean_log_prob_expanded", 2, 2}, + {"test_sce_mean_no_weight_ii", 2, 1}, + {"test_sce_mean_no_weight_ii_3d", 2, 1}, + {"test_sce_mean_no_weight_ii_3d_expanded", 2, 1}, + {"test_sce_mean_no_weight_ii_3d_log_prob", 2, 2}, + {"test_sce_mean_no_weight_ii_3d_log_prob_expanded", 2, 2}, + {"test_sce_mean_no_weight_ii_4d", 2, 1}, + {"test_sce_mean_no_weight_ii_4d_expanded", 2, 1}, + {"test_sce_mean_no_weight_ii_4d_log_prob", 2, 2}, + {"test_sce_mean_no_weight_ii_4d_log_prob_expanded", 2, 2}, + {"test_sce_mean_no_weight_ii_expanded", 2, 1}, + {"test_sce_mean_no_weight_ii_log_prob", 2, 2}, + {"test_sce_mean_no_weight_ii_log_prob_expanded", 2, 2}, + {"test_sce_mean_weight", 3, 1}, + {"test_sce_mean_weight_expanded", 3, 1}, + {"test_sce_mean_weight_ii", 3, 1}, + {"test_sce_mean_weight_ii_3d", 3, 1}, + {"test_sce_mean_weight_ii_3d_expanded", 3, 1}, + {"test_sce_mean_weight_ii_3d_log_prob", 3, 2}, + {"test_sce_mean_weight_ii_3d_log_prob_expanded", 3, 2}, + {"test_sce_mean_weight_ii_4d", 3, 1}, + {"test_sce_mean_weight_ii_4d_expanded", 3, 1}, + {"test_sce_mean_weight_ii_4d_log_prob", 3, 2}, + {"test_sce_mean_weight_ii_4d_log_prob_expanded", 3, 2}, + {"test_sce_mean_weight_ii_expanded", 3, 1}, + {"test_sce_mean_weight_ii_log_prob", 3, 2}, + {"test_sce_mean_weight_ii_log_prob_expanded", 3, 2}, + {"test_sce_mean_weight_log_prob", 3, 2}, + {"test_sce_mean_weight_log_prob_expanded", 3, 2}, + {"test_sce_none", 2, 1}, + {"test_sce_none_expanded", 2, 1}, + {"test_sce_none_log_prob", 2, 2}, + {"test_sce_none_log_prob_expanded", 2, 2}, + {"test_sce_none_weights", 3, 1}, + {"test_sce_none_weights_expanded", 3, 1}, + {"test_sce_none_weights_log_prob", 3, 2}, + {"test_sce_none_weights_log_prob_expanded", 3, 2}, + {"test_sce_sum", 2, 1}, + {"test_sce_sum_expanded", 2, 1}, + {"test_sce_sum_log_prob", 2, 2}, + {"test_sce_sum_log_prob_expanded", 2, 2}, + {"test_selu", 1, 1}, + {"test_selu_default", 1, 1}, + {"test_selu_example", 1, 1}, + {"test_sequence_insert_at_back", 2, 1}, + {"test_sequence_insert_at_front", 3, 1}, + {"test_shape", 1, 1}, + {"test_shape_clip_end", 1, 1}, + {"test_shape_clip_start", 1, 1}, + {"test_shape_end_1", 1, 1}, + {"test_shape_end_negative_1", 1, 1}, + {"test_shape_example", 1, 1}, + {"test_shape_start_1", 1, 1}, + {"test_shape_start_1_end_2", 1, 1}, + {"test_shape_start_1_end_negative_1", 1, 1}, + {"test_shape_start_negative_1", 1, 1}, + {"test_shrink_hard", 1, 1}, + {"test_shrink_soft", 1, 1}, + {"test_sigmoid", 1, 1}, + {"test_sigmoid_example", 1, 1}, + {"test_sign", 1, 1}, + {"test_simple_rnn_batchwise", 3, 2}, + {"test_simple_rnn_defaults", 3, 1}, + {"test_simple_rnn_with_initial_bias", 4, 1}, + {"test_sin", 1, 1}, + {"test_sin_example", 1, 1}, + {"test_sinh", 1, 1}, + {"test_sinh_example", 1, 1}, + {"test_size", 1, 1}, + {"test_size_example", 1, 1}, + {"test_slice", 5, 1}, + {"test_slice_default_axes", 3, 1}, + {"test_slice_default_steps", 4, 1}, + {"test_slice_end_out_of_bounds", 5, 1}, + {"test_slice_neg", 5, 1}, + {"test_slice_neg_steps", 5, 1}, + {"test_slice_negative_axes", 4, 1}, + {"test_slice_start_out_of_bounds", 5, 1}, + {"test_softmax_axis_0", 1, 1}, + {"test_softmax_axis_0_expanded", 1, 1}, + {"test_softmax_axis_1", 1, 1}, + {"test_softmax_axis_1_expanded", 1, 1}, + {"test_softmax_axis_2", 1, 1}, + {"test_softmax_axis_2_expanded", 1, 1}, + {"test_softmax_default_axis", 1, 1}, + {"test_softmax_default_axis_expanded", 1, 1}, + {"test_softmax_example", 1, 1}, + {"test_softmax_example_expanded", 1, 1}, + {"test_softmax_large_number", 1, 1}, + {"test_softmax_large_number_expanded", 1, 1}, + {"test_softmax_negative_axis", 1, 1}, + {"test_softmax_negative_axis_expanded", 1, 1}, + {"test_softplus", 1, 1}, + {"test_softplus_example", 1, 1}, + {"test_softsign", 1, 1}, + {"test_softsign_example", 1, 1}, + {"test_spacetodepth", 1, 1}, + {"test_spacetodepth_example", 1, 1}, + {"test_split_equal_parts_1d", 1, 3}, + {"test_split_equal_parts_2d", 1, 2}, + {"test_split_equal_parts_default_axis", 1, 3}, + {"test_split_variable_parts_1d", 2, 2}, + {"test_split_variable_parts_2d", 2, 2}, + {"test_split_variable_parts_default_axis", 2, 2}, + {"test_split_zero_size_splits", 2, 3}, + {"test_sqrt", 1, 1}, + {"test_sqrt_example", 1, 1}, + {"test_squeeze", 2, 1}, + {"test_squeeze_negative_axes", 2, 1}, + {"test_strnormalizer_export_monday_casesensintive_lower", 1, 1}, + {"test_strnormalizer_export_monday_casesensintive_nochangecase", 1, 1}, + {"test_strnormalizer_export_monday_casesensintive_upper", 1, 1}, + {"test_strnormalizer_export_monday_empty_output", 1, 1}, + {"test_strnormalizer_export_monday_insensintive_upper_twodim", 1, 1}, + {"test_strnormalizer_nostopwords_nochangecase", 1, 1}, + {"test_sub", 2, 1}, + {"test_sub_bcast", 2, 1}, + {"test_sub_example", 2, 1}, + {"test_sub_uint8", 2, 1}, + {"test_sum_example", 3, 1}, + {"test_sum_one_input", 1, 1}, + {"test_sum_two_inputs", 2, 1}, + {"test_tan", 1, 1}, + {"test_tan_example", 1, 1}, + {"test_tanh", 1, 1}, + {"test_tanh_example", 1, 1}, + {"test_tfidfvectorizer_tf_batch_onlybigrams_skip0", 1, 1}, + {"test_tfidfvectorizer_tf_batch_onlybigrams_skip5", 1, 1}, + {"test_tfidfvectorizer_tf_batch_uniandbigrams_skip5", 1, 1}, + {"test_tfidfvectorizer_tf_only_bigrams_skip0", 1, 1}, + {"test_tfidfvectorizer_tf_onlybigrams_levelempty", 1, 1}, + {"test_tfidfvectorizer_tf_onlybigrams_skip5", 1, 1}, + {"test_tfidfvectorizer_tf_uniandbigrams_skip5", 1, 1}, + {"test_thresholdedrelu", 1, 1}, + {"test_thresholdedrelu_default", 1, 1}, + {"test_thresholdedrelu_example", 1, 1}, + {"test_tile", 2, 1}, + {"test_tile_precomputed", 2, 1}, + {"test_top_k", 2, 2}, + {"test_top_k_negative_axis", 2, 2}, + {"test_top_k_smallest", 2, 2}, + {"test_training_dropout", 3, 1}, + {"test_training_dropout_default", 3, 1}, + {"test_training_dropout_default_mask", 3, 2}, + {"test_training_dropout_mask", 3, 2}, + {"test_training_dropout_zero_ratio", 3, 1}, + {"test_training_dropout_zero_ratio_mask", 3, 2}, + {"test_transpose_all_permutations_0", 1, 1}, + {"test_transpose_all_permutations_1", 1, 1}, + {"test_transpose_all_permutations_2", 1, 1}, + {"test_transpose_all_permutations_3", 1, 1}, + {"test_transpose_all_permutations_4", 1, 1}, + {"test_transpose_all_permutations_5", 1, 1}, + {"test_transpose_default", 1, 1}, + {"test_tril", 1, 1}, + {"test_tril_neg", 2, 1}, + {"test_tril_one_row_neg", 1, 1}, + {"test_tril_out_neg", 2, 1}, + {"test_tril_out_pos", 2, 1}, + {"test_tril_pos", 2, 1}, + {"test_tril_square", 1, 1}, + {"test_tril_square_neg", 2, 1}, + {"test_tril_zero", 2, 1}, + {"test_triu", 1, 1}, + {"test_triu_neg", 2, 1}, + {"test_triu_one_row", 2, 1}, + {"test_triu_out_neg_out", 2, 1}, + {"test_triu_out_pos", 2, 1}, + {"test_triu_pos", 2, 1}, + {"test_triu_square", 1, 1}, + {"test_triu_square_neg", 2, 1}, + {"test_triu_zero", 2, 1}, + {"test_unique_not_sorted_without_axis", 1, 4}, + {"test_unique_sorted_with_axis", 1, 4}, + {"test_unique_sorted_with_axis_3d", 1, 4}, + {"test_unique_sorted_with_negative_axis", 1, 4}, + {"test_unique_sorted_without_axis", 1, 4}, + {"test_unsqueeze_axis_0", 2, 1}, + {"test_unsqueeze_axis_1", 2, 1}, + {"test_unsqueeze_axis_2", 2, 1}, + {"test_unsqueeze_axis_3", 1, 1}, + {"test_unsqueeze_negative_axes", 2, 1}, + {"test_unsqueeze_three_axes", 2, 1}, + {"test_unsqueeze_two_axes", 2, 1}, + {"test_unsqueeze_unsorted_axes", 2, 1}, + {"test_upsample_nearest", 2, 1}, + {"test_where_example", 3, 1}, + {"test_where_long_example", 3, 1}, + {"test_xor2d", 2, 1}, + {"test_xor3d", 2, 1}, + {"test_xor4d", 2, 1}, + {"test_xor_bcast3v1d", 2, 1}, + {"test_xor_bcast3v2d", 2, 1}, + {"test_xor_bcast4v2d", 2, 1}, + {"test_xor_bcast4v3d", 2, 1}, + {"test_xor_bcast4v4d", 2, 1}, +}; + + +struct TestCaseInput +{ + std::vector input_paths; + std::vector output_paths; + std::string model_path; + std::string name; +}; + +std::ostream& operator<<(std::ostream& os, const TestCaseInput& test_case) +{ + return os << test_case.name; +} + +typedef tuple > ONNXConfParams; + +std::string printOnnxConfParams(const testing::TestParamInfo& params) +{ + TestCaseInput test_case = get<0>(params.param); + Backend backend = get<0>(get<1>(params.param)); + Target target = get<1>(get<1>(params.param)); + + std::stringstream ss; + ss << test_case.name << "_"; + PrintTo(backend, &ss); + ss << "_"; + PrintTo(target, &ss); + + return ss.str(); +} + +template +static std::string _tf(TString filename, bool required = true) +{ + return findDataFile(std::string("dnn/onnx/") + filename, required); +} + +std::vector readTestCases() +{ + std::vector ret; + for (size_t i = 0; i < sizeof(testConformanceConfig) / sizeof(testConformanceConfig[0]); ++i) + { + const TestCase& test_case = testConformanceConfig[i]; + + TestCaseInput input; + + std::string prefix = cv::format("conformance/node/%s", test_case.name); + input.name = test_case.name; + input.model_path = _tf(cv::format("%s/model.onnx", prefix.c_str())); + + for (int i = 0; i < test_case.inputs; ++i) + { + input.input_paths.push_back(_tf(cv::format("%s/test_data_set_0/input_%d.pb", prefix.c_str(), i))); + } + + for (int i = 0; i < test_case.outputs; ++i) + { + input.output_paths.push_back(_tf(cv::format("%s/test_data_set_0/output_%d.pb", prefix.c_str(), i))); + } + + ret.push_back(input); + } + + return ret; +} + +class Test_ONNX_conformance : public TestWithParam +{ +public: + TestCaseInput test_case; + Backend backend; + Target target; + + double default_l1; + double default_lInf; + + static std::set parser_deny_list; + static std::set global_deny_list; + static std::set opencl_fp16_deny_list; + static std::set opencl_deny_list; + static std::set cpu_deny_list; + + Test_ONNX_conformance() + { + test_case = get<0>(GetParam()); + backend = get<0>(get<1>(GetParam())); + target = get<1>(get<1>(GetParam())); + + if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) + { + default_l1 = 4e-3; + default_lInf = 2e-2; + } + else + { + default_l1 = 1e-5; + default_lInf = 1e-4; + } + } + + bool checkFallbacks(Net& net) const + { + // Check if all the layers are supported with current backend and target. + // Some layers might be fused so their timings equal to zero. + std::vector timings; + net.getPerfProfile(timings); + std::vector names = net.getLayerNames(); + CV_CheckEQ(names.size(), timings.size(), "DNN critical error"); + + bool hasFallbacks = false; + for (int i = 0; i < names.size(); ++i) + { + Ptr l = net.getLayer(net.getLayerId(names[i])); + bool fused = timings[i] == 0.; + if ((!l->supportBackend(backend) || l->preferableTarget != target) && !fused) + { + hasFallbacks = true; + std::cout << "FALLBACK: Layer [" << l->type << "]:[" << l->name << "] is expected to has backend implementation" << endl; + } + } + return hasFallbacks; + } + + static void initDenyList(std::set& deny_set, const char* const deny_list[], const size_t n) + { + for (size_t i = 0; i < n; ++i) + { + deny_set.insert(deny_list[i]); + } + } + + static void SetUpTestCase() + { + const char* const parser[] = { + #include "test_onnx_conformance_layer_parser_denylist.inl.hpp" + "" // dummy element of non empty list + }; + initDenyList(parser_deny_list, parser, sizeof(parser)/sizeof(parser[0])); + + const char* const global[] = { + #include "test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp" + "" // dummy element of non empty list + }; + initDenyList(global_deny_list, global, sizeof(global)/sizeof(global[0])); + + const char* const opencl_fp16[] = { + #include "test_onnx_conformance_layer_filter_opencv_ocl_fp16_denylist.inl.hpp" + "" // dummy element of non empty list + }; + initDenyList(opencl_fp16_deny_list, opencl_fp16, sizeof(opencl_fp16)/sizeof(opencl_fp16[0])); + + const char* const opencl[] = { + #include "test_onnx_conformance_layer_filter_opencv_ocl_fp32_denylist.inl.hpp" + "" // dummy element of non empty list + }; + initDenyList(opencl_deny_list, opencl, sizeof(opencl)/sizeof(opencl[0])); + + const char* const cpu[] = { + #include "test_onnx_conformance_layer_filter_opencv_cpu_denylist.inl.hpp" + "" // dummy element of non empty list + }; + initDenyList(cpu_deny_list, cpu, sizeof(cpu)/sizeof(cpu[0])); + } + + void checkFilterLists() const + { + const std::string& name = test_case.name; + if(parser_deny_list.find(name) != parser_deny_list.end()) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_PARSER, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + + if (backend == DNN_BACKEND_OPENCV) + { + if(global_deny_list.find(name) != global_deny_list.end()) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + if((target == DNN_TARGET_OPENCL_FP16) && (opencl_fp16_deny_list.find(name) != opencl_fp16_deny_list.end())) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + if((target == DNN_TARGET_OPENCL) && (opencl_deny_list.find(name) != opencl_deny_list.end())) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_OPENCL, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + if((target == DNN_TARGET_CPU) && (cpu_deny_list.find(name) != cpu_deny_list.end())) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_CPU, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + } +#if 0 //def HAVE_HALIDE + else if (backend == DNN_BACKEND_HALIDE) + { + #include "test_onnx_conformance_layer_filter__halide.inl.hpp" + } +#endif +#if 0 //def HAVE_INF_ENGINE + else if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + #include "test_onnx_conformance_layer_filter__ngraph.inl.hpp" + } +#endif +#if 0 //def HAVE_VULKAN + else if (backend == DNN_BACKEND_VKCOM) + { + #include "test_onnx_conformance_layer_filter__vulkan.inl.hpp" + } +#endif +#if 0 //def HAVE_CUDA + else if (backend == DNN_BACKEND_CUDA) + { + #include "test_onnx_conformance_layer_filter__cuda.inl.hpp" + } +#endif + else + { + std::ostringstream ss; + ss << "No test filter available for backend "; + PrintTo(backend, &ss); + ss << ". Run test by default"; + std::cout << ss.str() << std::endl; + } + } +}; + +std::set Test_ONNX_conformance::parser_deny_list; +std::set Test_ONNX_conformance::global_deny_list; +std::set Test_ONNX_conformance::opencl_fp16_deny_list; +std::set Test_ONNX_conformance::opencl_deny_list; +std::set Test_ONNX_conformance::cpu_deny_list; + +TEST_P(Test_ONNX_conformance, Layer_Test) +{ + std::string name = test_case.name; + ASSERT_FALSE(name.empty()); + + bool checkLayersFallbacks = true; + bool checkAccuracy = true; + + checkFilterLists(); + + std::vector inputs; + std::vector ref_outputs; + + Net net; + try + { + //cout << "Read ONNX inputs..." << endl; + std::transform(test_case.input_paths.begin(), test_case.input_paths.end(), + std::back_inserter(inputs), readTensorFromONNX); + + //cout << "Read ONNX reference outputs..." << endl; + std::transform(test_case.output_paths.begin(), test_case.output_paths.end(), + std::back_inserter(ref_outputs), readTensorFromONNX); + + //cout << "Parse model..." << endl; + net = readNetFromONNX(test_case.model_path); + if (net.empty()) + { + applyTestTag(CV_TEST_TAG_DNN_ERROR_PARSER); + } + } + catch (...) + { + cout << "Exception during ONNX model parse / loading input / loading reference data!" << endl; + applyTestTag(CV_TEST_TAG_DNN_ERROR_PARSER); + throw; + } + ASSERT_FALSE(net.empty()); + + std::vector inputNames; + for (int i = 0; i < inputs.size(); ++i) + inputNames.push_back(cv::format("%d", i)); + net.setInputsNames(inputNames); + + try + { + net.setPreferableBackend(backend); + net.setPreferableTarget(target); + + for (int i = 0; i < inputs.size(); ++i) + { + net.setInput(inputs[i], inputNames[i]); + } + } + catch (...) + { + cout << "Exception during network configuration!" << endl; + applyTestTag(CV_TEST_TAG_DNN_ERROR_NET_SETUP); + throw; + } + + std::vector layerNames = net.getUnconnectedOutLayersNames(); + std::vector< std::vector > outputs_; + try + { + net.forward(outputs_, layerNames); + } + catch (...) + { + cout << "Exception during net.forward() call!" << endl; + applyTestTag(CV_TEST_TAG_DNN_ERROR_FORWARD); + throw; + } + ASSERT_GE(outputs_.size(), 1); + const std::vector& outputs = outputs_[0]; + + if (checkLayersFallbacks && checkFallbacks(net)) + { + applyTestTag(CV_TEST_TAG_DNN_LAYER_FALLBACK); + } + + if (checkAccuracy) + { + try + { + if (ref_outputs.size() == 1) + { + // probably we found random unconnected layers. + normAssert(ref_outputs[0], outputs[0], "", default_l1, default_lInf); + } + else + { + ASSERT_EQ(outputs.size(), ref_outputs.size()); + for (size_t i = 0; i < ref_outputs.size(); ++i) + { + normAssert(ref_outputs[i], outputs[i], "", default_l1, default_lInf); + } + } + } + catch (...) + { + cout << "Exception during accuracy check!" << endl; + throw; + } + } + else + { + applyTestTag(CV_TEST_TAG_DNN_NO_ACCURACY_CHECK); + } + + if (!HasFailure()) + cout << "Test passed!" << endl; +} + +INSTANTIATE_TEST_CASE_P(/**/, Test_ONNX_conformance, + testing::Combine(testing::ValuesIn(readTestCases()), dnnBackendsAndTargets()), + printOnnxConfParams); + +}; diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp new file mode 100644 index 0000000000..cff1e93aa0 --- /dev/null +++ b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp @@ -0,0 +1,23 @@ +"test_add_bcast", +"test_averagepool_2d_pads_count_include_pad", +"test_averagepool_2d_precomputed_pads_count_include_pad", +"test_averagepool_2d_same_lower", +"test_cast_FLOAT_to_STRING", +"test_cast_STRING_to_FLOAT", +"test_castlike_FLOAT_to_STRING_expanded", +"test_castlike_STRING_to_FLOAT_expanded", +"test_concat_1d_axis_negative_1", +"test_flatten_axis0", +"test_flatten_axis2", +"test_flatten_axis3", +"test_flatten_negative_axis1", +"test_flatten_negative_axis2", +"test_flatten_negative_axis4", +"test_logsoftmax_default_axis", +"test_maxpool_2d_dilations", +"test_maxpool_2d_same_lower", +"test_maxpool_with_argmax_2d_precomputed_pads", +"test_maxpool_with_argmax_2d_precomputed_strides", +"test_softmax_default_axis", +"test_sub_bcast", +"test_upsample_nearest", diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_cpu_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_cpu_denylist.inl.hpp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp16_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp16_denylist.inl.hpp new file mode 100644 index 0000000000..573d847985 --- /dev/null +++ b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp16_denylist.inl.hpp @@ -0,0 +1,21 @@ +"test_averagepool_3d_default", +"test_dropout_default_ratio", +"test_globalmaxpool", +"test_globalmaxpool_precomputed", +"test_logsoftmax_large_number", +"test_logsoftmax_large_number_expanded", +"test_maxpool_1d_default", +"test_maxpool_2d_ceil", +"test_maxpool_2d_default", +"test_maxpool_2d_pads", +"test_maxpool_2d_precomputed_pads", +"test_maxpool_2d_precomputed_same_upper", +"test_maxpool_2d_precomputed_strides", +"test_maxpool_2d_same_upper", +"test_maxpool_2d_strides", +"test_maxpool_3d_default", +"test_softmax_large_number", +"test_softmax_large_number_expanded", +"test_split_equal_parts_1d", +"test_split_equal_parts_2d", +"test_split_equal_parts_default_axis", diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp32_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp32_denylist.inl.hpp new file mode 100644 index 0000000000..9a7a21f393 --- /dev/null +++ b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp32_denylist.inl.hpp @@ -0,0 +1,2 @@ +"test_averagepool_3d_default", +"test_maxpool_3d_default", diff --git a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp new file mode 100644 index 0000000000..5f1d99d4bb --- /dev/null +++ b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp @@ -0,0 +1,717 @@ +// The file is autogenerated +// Update note: execute /testdata/dnn/onnx/generate_conformance_list.py +"test_abs", +"test_acos", +"test_acos_example", +"test_acosh", +"test_acosh_example", +"test_adagrad", +"test_adagrad_multiple", +"test_adam", +"test_adam_multiple", +"test_add_uint8", +"test_and2d", +"test_and3d", +"test_and4d", +"test_and_bcast3v1d", +"test_and_bcast3v2d", +"test_and_bcast4v2d", +"test_and_bcast4v3d", +"test_and_bcast4v4d", +"test_argmax_default_axis_example", +"test_argmax_default_axis_example_select_last_index", +"test_argmax_default_axis_random", +"test_argmax_default_axis_random_select_last_index", +"test_argmax_keepdims_example", +"test_argmax_keepdims_example_select_last_index", +"test_argmax_keepdims_random", +"test_argmax_keepdims_random_select_last_index", +"test_argmax_negative_axis_keepdims_example", +"test_argmax_negative_axis_keepdims_example_select_last_index", +"test_argmax_negative_axis_keepdims_random", +"test_argmax_negative_axis_keepdims_random_select_last_index", +"test_argmax_no_keepdims_example", +"test_argmax_no_keepdims_example_select_last_index", +"test_argmax_no_keepdims_random", +"test_argmax_no_keepdims_random_select_last_index", +"test_argmin_default_axis_example", +"test_argmin_default_axis_example_select_last_index", +"test_argmin_default_axis_random", +"test_argmin_default_axis_random_select_last_index", +"test_argmin_keepdims_example", +"test_argmin_keepdims_example_select_last_index", +"test_argmin_keepdims_random", +"test_argmin_keepdims_random_select_last_index", +"test_argmin_negative_axis_keepdims_example", +"test_argmin_negative_axis_keepdims_example_select_last_index", +"test_argmin_negative_axis_keepdims_random", +"test_argmin_negative_axis_keepdims_random_select_last_index", +"test_argmin_no_keepdims_example", +"test_argmin_no_keepdims_example_select_last_index", +"test_argmin_no_keepdims_random", +"test_argmin_no_keepdims_random_select_last_index", +"test_asin", +"test_asin_example", +"test_asinh", +"test_asinh_example", +"test_atan", +"test_atan_example", +"test_atanh", +"test_atanh_example", +"test_basic_convinteger", +"test_batchnorm_epsilon", +"test_batchnorm_epsilon_training_mode", +"test_batchnorm_example", +"test_batchnorm_example_training_mode", +"test_bernoulli", +"test_bernoulli_double", +"test_bernoulli_double_expanded", +"test_bernoulli_expanded", +"test_bernoulli_seed", +"test_bernoulli_seed_expanded", +"test_bitshift_left_uint16", +"test_bitshift_left_uint32", +"test_bitshift_left_uint64", +"test_bitshift_left_uint8", +"test_bitshift_right_uint16", +"test_bitshift_right_uint32", +"test_bitshift_right_uint64", +"test_bitshift_right_uint8", +"test_cast_BFLOAT16_to_FLOAT", +"test_cast_DOUBLE_to_FLOAT", +"test_cast_DOUBLE_to_FLOAT16", +"test_cast_FLOAT16_to_DOUBLE", +"test_cast_FLOAT16_to_FLOAT", +"test_cast_FLOAT_to_BFLOAT16", +"test_cast_FLOAT_to_DOUBLE", +"test_cast_FLOAT_to_FLOAT16", +"test_castlike_BFLOAT16_to_FLOAT", +"test_castlike_BFLOAT16_to_FLOAT_expanded", +"test_castlike_DOUBLE_to_FLOAT", +"test_castlike_DOUBLE_to_FLOAT16", +"test_castlike_DOUBLE_to_FLOAT16_expanded", +"test_castlike_DOUBLE_to_FLOAT_expanded", +"test_castlike_FLOAT16_to_DOUBLE", +"test_castlike_FLOAT16_to_DOUBLE_expanded", +"test_castlike_FLOAT16_to_FLOAT", +"test_castlike_FLOAT16_to_FLOAT_expanded", +"test_castlike_FLOAT_to_BFLOAT16", +"test_castlike_FLOAT_to_BFLOAT16_expanded", +"test_castlike_FLOAT_to_DOUBLE", +"test_castlike_FLOAT_to_DOUBLE_expanded", +"test_castlike_FLOAT_to_FLOAT16", +"test_castlike_FLOAT_to_FLOAT16_expanded", +"test_castlike_FLOAT_to_STRING", +"test_castlike_STRING_to_FLOAT", +"test_ceil", +"test_ceil_example", +"test_celu", +"test_clip", +"test_clip_default_inbounds", +"test_clip_default_int8_inbounds", +"test_clip_default_int8_max", +"test_clip_default_int8_min", +"test_clip_default_max", +"test_clip_default_min", +"test_clip_example", +"test_clip_inbounds", +"test_clip_outbounds", +"test_clip_splitbounds", +"test_compress_0", +"test_compress_1", +"test_compress_default_axis", +"test_compress_negative_axis", +"test_constant", +"test_constant_pad", +"test_constantofshape_float_ones", +"test_constantofshape_int_shape_zero", +"test_constantofshape_int_zeros", +"test_convinteger_with_padding", +"test_convinteger_without_padding", +"test_convtranspose", +"test_convtranspose_1d", +"test_convtranspose_3d", +"test_convtranspose_autopad_same", +"test_convtranspose_dilations", +"test_convtranspose_kernel_shape", +"test_convtranspose_output_shape", +"test_convtranspose_pad", +"test_convtranspose_pads", +"test_convtranspose_with_kernel", +"test_cos", +"test_cos_example", +"test_cosh", +"test_cosh_example", +"test_cumsum_1d", +"test_cumsum_1d_exclusive", +"test_cumsum_1d_reverse", +"test_cumsum_1d_reverse_exclusive", +"test_cumsum_2d_axis_0", +"test_cumsum_2d_axis_1", +"test_cumsum_2d_negative_axis", +"test_depthtospace_crd_mode", +"test_depthtospace_crd_mode_example", +"test_depthtospace_dcr_mode", +"test_depthtospace_example", +"test_dequantizelinear", +"test_dequantizelinear_axis", +"test_det_2d", +"test_det_nd", +"test_div_example", +"test_div_uint8", +"test_dropout_default_mask", +"test_dropout_default_mask_ratio", +"test_dynamicquantizelinear", +"test_dynamicquantizelinear_expanded", +"test_dynamicquantizelinear_max_adjusted", +"test_dynamicquantizelinear_max_adjusted_expanded", +"test_dynamicquantizelinear_min_adjusted", +"test_dynamicquantizelinear_min_adjusted_expanded", +"test_edge_pad", +"test_einsum_batch_diagonal", +"test_einsum_batch_matmul", +"test_einsum_inner_prod", +"test_einsum_sum", +"test_einsum_transpose", +"test_equal", +"test_equal_bcast", +"test_erf", +"test_expand_dim_changed", +"test_expand_dim_unchanged", +"test_eyelike_populate_off_main_diagonal", +"test_eyelike_with_dtype", +"test_eyelike_without_dtype", +"test_floor", +"test_floor_example", +"test_gather_0", +"test_gather_1", +"test_gather_2d_indices", +"test_gather_elements_0", +"test_gather_elements_1", +"test_gather_elements_negative_indices", +"test_gather_negative_indices", +"test_gathernd_example_float32", +"test_gathernd_example_int32", +"test_gathernd_example_int32_batch_dim1", +"test_gemm_all_attributes", +"test_gemm_alpha", +"test_gemm_beta", +"test_gemm_default_matrix_bias", +"test_gemm_default_no_bias", +"test_gemm_default_scalar_bias", +"test_gemm_default_single_elem_vector_bias", +"test_gemm_default_vector_bias", +"test_gemm_default_zero_bias", +"test_gemm_transposeA", +"test_gemm_transposeB", +"test_greater", +"test_greater_bcast", +"test_greater_equal", +"test_greater_equal_bcast", +"test_greater_equal_bcast_expanded", +"test_greater_equal_expanded", +"test_gridsample", +"test_gridsample_aligncorners_true", +"test_gridsample_bicubic", +"test_gridsample_bilinear", +"test_gridsample_border_padding", +"test_gridsample_nearest", +"test_gridsample_reflection_padding", +"test_gridsample_zeros_padding", +"test_gru_batchwise", +"test_gru_defaults", +"test_gru_seq_length", +"test_gru_with_initial_bias", +"test_hardmax_axis_0", +"test_hardmax_axis_1", +"test_hardmax_axis_2", +"test_hardmax_default_axis", +"test_hardmax_example", +"test_hardmax_negative_axis", +"test_hardmax_one_hot", +"test_hardsigmoid", +"test_hardsigmoid_default", +"test_hardsigmoid_example", +"test_hardswish", +"test_hardswish_expanded", +"test_identity_opt", +"test_identity_sequence", +"test_if", +"test_if_opt", +"test_if_seq", +"test_instancenorm_epsilon", +"test_instancenorm_example", +"test_isinf", +"test_isinf_negative", +"test_isinf_positive", +"test_isnan", +"test_less", +"test_less_bcast", +"test_less_equal", +"test_less_equal_bcast", +"test_less_equal_bcast_expanded", +"test_less_equal_expanded", +"test_log", +"test_log_example", +"test_loop11", +"test_loop13_seq", +"test_loop16_seq_none", +"test_lstm_batchwise", +"test_lstm_defaults", +"test_lstm_with_initial_bias", +"test_lstm_with_peepholes", +"test_matmulinteger", +"test_max_example", +"test_max_float16", +"test_max_float32", +"test_max_float64", +"test_max_int16", +"test_max_int32", +"test_max_int64", +"test_max_int8", +"test_max_one_input", +"test_max_two_inputs", +"test_max_uint16", +"test_max_uint32", +"test_max_uint64", +"test_max_uint8", +"test_maxpool_2d_uint8", +"test_maxunpool_export_with_output_shape", +"test_maxunpool_export_without_output_shape", +"test_mean_example", +"test_mean_one_input", +"test_mean_two_inputs", +"test_min_example", +"test_min_float16", +"test_min_float32", +"test_min_float64", +"test_min_int16", +"test_min_int32", +"test_min_int64", +"test_min_int8", +"test_min_one_input", +"test_min_two_inputs", +"test_min_uint16", +"test_min_uint32", +"test_min_uint64", +"test_min_uint8", +"test_mod_broadcast", +"test_mod_int64_fmod", +"test_mod_mixed_sign_float16", +"test_mod_mixed_sign_float32", +"test_mod_mixed_sign_float64", +"test_mod_mixed_sign_int16", +"test_mod_mixed_sign_int32", +"test_mod_mixed_sign_int64", +"test_mod_mixed_sign_int8", +"test_mod_uint16", +"test_mod_uint32", +"test_mod_uint64", +"test_mod_uint8", +"test_momentum", +"test_momentum_multiple", +"test_mul_example", +"test_mul_uint8", +"test_mvn", +"test_mvn_expanded", +"test_nesterov_momentum", +"test_nllloss_NC", +"test_nllloss_NC_expanded", +"test_nllloss_NCd1", +"test_nllloss_NCd1_expanded", +"test_nllloss_NCd1_ii", +"test_nllloss_NCd1_ii_expanded", +"test_nllloss_NCd1_mean_weight_negative_ii", +"test_nllloss_NCd1_mean_weight_negative_ii_expanded", +"test_nllloss_NCd1_weight", +"test_nllloss_NCd1_weight_expanded", +"test_nllloss_NCd1_weight_ii", +"test_nllloss_NCd1_weight_ii_expanded", +"test_nllloss_NCd1d2", +"test_nllloss_NCd1d2_expanded", +"test_nllloss_NCd1d2_no_weight_reduction_mean_ii", +"test_nllloss_NCd1d2_no_weight_reduction_mean_ii_expanded", +"test_nllloss_NCd1d2_reduction_mean", +"test_nllloss_NCd1d2_reduction_mean_expanded", +"test_nllloss_NCd1d2_reduction_sum", +"test_nllloss_NCd1d2_reduction_sum_expanded", +"test_nllloss_NCd1d2_with_weight", +"test_nllloss_NCd1d2_with_weight_expanded", +"test_nllloss_NCd1d2_with_weight_reduction_mean", +"test_nllloss_NCd1d2_with_weight_reduction_mean_expanded", +"test_nllloss_NCd1d2_with_weight_reduction_sum", +"test_nllloss_NCd1d2_with_weight_reduction_sum_expanded", +"test_nllloss_NCd1d2_with_weight_reduction_sum_ii", +"test_nllloss_NCd1d2_with_weight_reduction_sum_ii_expanded", +"test_nllloss_NCd1d2d3_none_no_weight_negative_ii", +"test_nllloss_NCd1d2d3_none_no_weight_negative_ii_expanded", +"test_nllloss_NCd1d2d3_sum_weight_high_ii", +"test_nllloss_NCd1d2d3_sum_weight_high_ii_expanded", +"test_nllloss_NCd1d2d3d4d5_mean_weight", +"test_nllloss_NCd1d2d3d4d5_mean_weight_expanded", +"test_nllloss_NCd1d2d3d4d5_none_no_weight", +"test_nllloss_NCd1d2d3d4d5_none_no_weight_expanded", +"test_nonmaxsuppression_center_point_box_format", +"test_nonmaxsuppression_flipped_coordinates", +"test_nonmaxsuppression_identical_boxes", +"test_nonmaxsuppression_limit_output_size", +"test_nonmaxsuppression_single_box", +"test_nonmaxsuppression_suppress_by_IOU", +"test_nonmaxsuppression_suppress_by_IOU_and_scores", +"test_nonmaxsuppression_two_batches", +"test_nonmaxsuppression_two_classes", +"test_nonzero_example", +"test_not_2d", +"test_not_3d", +"test_not_4d", +"test_onehot_negative_indices", +"test_onehot_with_axis", +"test_onehot_with_negative_axis", +"test_onehot_without_axis", +"test_optional_get_element", +"test_optional_get_element_sequence", +"test_optional_has_element", +"test_optional_has_element_empty", +"test_or2d", +"test_or3d", +"test_or4d", +"test_or_bcast3v1d", +"test_or_bcast3v2d", +"test_or_bcast4v2d", +"test_or_bcast4v3d", +"test_or_bcast4v4d", +"test_pow", +"test_pow_bcast_array", +"test_pow_bcast_scalar", +"test_pow_example", +"test_pow_types_float", +"test_pow_types_float32_int32", +"test_pow_types_float32_int64", +"test_pow_types_float32_uint32", +"test_pow_types_float32_uint64", +"test_pow_types_int", +"test_pow_types_int32_float32", +"test_pow_types_int32_int32", +"test_pow_types_int64_float32", +"test_pow_types_int64_int64", +"test_prelu_broadcast", +"test_prelu_example", +"test_qlinearconv", +"test_qlinearmatmul_2D", +"test_qlinearmatmul_3D", +"test_quantizelinear", +"test_quantizelinear_axis", +"test_range_float_type_positive_delta", +"test_range_float_type_positive_delta_expanded", +"test_range_int32_type_negative_delta", +"test_range_int32_type_negative_delta_expanded", +"test_reciprocal", +"test_reciprocal_example", +"test_reduce_l1_default_axes_keepdims_example", +"test_reduce_l1_default_axes_keepdims_random", +"test_reduce_l1_do_not_keepdims_example", +"test_reduce_l1_do_not_keepdims_random", +"test_reduce_l1_keep_dims_example", +"test_reduce_l1_keep_dims_random", +"test_reduce_l1_negative_axes_keep_dims_example", +"test_reduce_l1_negative_axes_keep_dims_random", +"test_reduce_l2_default_axes_keepdims_example", +"test_reduce_l2_default_axes_keepdims_random", +"test_reduce_l2_do_not_keepdims_example", +"test_reduce_l2_do_not_keepdims_random", +"test_reduce_l2_keep_dims_example", +"test_reduce_l2_keep_dims_random", +"test_reduce_l2_negative_axes_keep_dims_example", +"test_reduce_l2_negative_axes_keep_dims_random", +"test_reduce_log_sum", +"test_reduce_log_sum_asc_axes", +"test_reduce_log_sum_default", +"test_reduce_log_sum_desc_axes", +"test_reduce_log_sum_exp_default_axes_keepdims_example", +"test_reduce_log_sum_exp_default_axes_keepdims_random", +"test_reduce_log_sum_exp_do_not_keepdims_example", +"test_reduce_log_sum_exp_do_not_keepdims_random", +"test_reduce_log_sum_exp_keepdims_example", +"test_reduce_log_sum_exp_keepdims_random", +"test_reduce_log_sum_exp_negative_axes_keepdims_example", +"test_reduce_log_sum_exp_negative_axes_keepdims_random", +"test_reduce_log_sum_negative_axes", +"test_reduce_max_default_axes_keepdim_example", +"test_reduce_max_default_axes_keepdims_random", +"test_reduce_mean_default_axes_keepdims_example", +"test_reduce_mean_default_axes_keepdims_random", +"test_reduce_min_default_axes_keepdims_example", +"test_reduce_min_default_axes_keepdims_random", +"test_reduce_min_do_not_keepdims_example", +"test_reduce_min_do_not_keepdims_random", +"test_reduce_min_keepdims_example", +"test_reduce_min_keepdims_random", +"test_reduce_min_negative_axes_keepdims_example", +"test_reduce_min_negative_axes_keepdims_random", +"test_reduce_prod_default_axes_keepdims_example", +"test_reduce_prod_default_axes_keepdims_random", +"test_reduce_prod_do_not_keepdims_example", +"test_reduce_prod_do_not_keepdims_random", +"test_reduce_prod_keepdims_example", +"test_reduce_prod_keepdims_random", +"test_reduce_prod_negative_axes_keepdims_example", +"test_reduce_prod_negative_axes_keepdims_random", +"test_reduce_sum_default_axes_keepdims_example", +"test_reduce_sum_default_axes_keepdims_random", +"test_reduce_sum_do_not_keepdims_example", +"test_reduce_sum_do_not_keepdims_random", +"test_reduce_sum_empty_axes_input_noop_example", +"test_reduce_sum_empty_axes_input_noop_random", +"test_reduce_sum_keepdims_example", +"test_reduce_sum_keepdims_random", +"test_reduce_sum_negative_axes_keepdims_example", +"test_reduce_sum_negative_axes_keepdims_random", +"test_reduce_sum_square_default_axes_keepdims_example", +"test_reduce_sum_square_default_axes_keepdims_random", +"test_reduce_sum_square_do_not_keepdims_example", +"test_reduce_sum_square_do_not_keepdims_random", +"test_reduce_sum_square_keepdims_example", +"test_reduce_sum_square_keepdims_random", +"test_reduce_sum_square_negative_axes_keepdims_example", +"test_reduce_sum_square_negative_axes_keepdims_random", +"test_reflect_pad", +"test_reshape_allowzero_reordered", +"test_reshape_extended_dims", +"test_reshape_negative_dim", +"test_reshape_negative_extended_dims", +"test_reshape_one_dim", +"test_reshape_reduced_dims", +"test_reshape_reordered_all_dims", +"test_reshape_reordered_last_dims", +"test_reshape_zero_and_negative_dim", +"test_reshape_zero_dim", +"test_resize_downsample_scales_cubic", +"test_resize_downsample_scales_cubic_A_n0p5_exclude_outside", +"test_resize_downsample_scales_cubic_align_corners", +"test_resize_downsample_scales_linear", +"test_resize_downsample_scales_linear_align_corners", +"test_resize_downsample_scales_nearest", +"test_resize_downsample_sizes_cubic", +"test_resize_downsample_sizes_linear_pytorch_half_pixel", +"test_resize_downsample_sizes_nearest", +"test_resize_downsample_sizes_nearest_tf_half_pixel_for_nn", +"test_resize_tf_crop_and_resize", +"test_resize_upsample_scales_cubic", +"test_resize_upsample_scales_cubic_A_n0p5_exclude_outside", +"test_resize_upsample_scales_cubic_align_corners", +"test_resize_upsample_scales_cubic_asymmetric", +"test_resize_upsample_scales_linear", +"test_resize_upsample_scales_linear_align_corners", +"test_resize_upsample_scales_nearest", +"test_resize_upsample_sizes_cubic", +"test_resize_upsample_sizes_nearest", +"test_resize_upsample_sizes_nearest_ceil_half_pixel", +"test_resize_upsample_sizes_nearest_floor_align_corners", +"test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric", +"test_reversesequence_batch", +"test_reversesequence_time", +"test_rnn_seq_length", +"test_roialign_aligned_false", +"test_roialign_aligned_true", +"test_round", +"test_scan9_sum", +"test_scan_sum", +"test_scatter_elements_with_axis", +"test_scatter_elements_with_duplicate_indices", +"test_scatter_elements_with_negative_indices", +"test_scatter_elements_without_axis", +"test_scatter_with_axis", +"test_scatter_without_axis", +"test_scatternd", +"test_scatternd_add", +"test_scatternd_multiply", +"test_sce_NCd1_mean_weight_negative_ii", +"test_sce_NCd1_mean_weight_negative_ii_expanded", +"test_sce_NCd1_mean_weight_negative_ii_log_prob", +"test_sce_NCd1_mean_weight_negative_ii_log_prob_expanded", +"test_sce_NCd1d2d3_none_no_weight_negative_ii", +"test_sce_NCd1d2d3_none_no_weight_negative_ii_expanded", +"test_sce_NCd1d2d3_none_no_weight_negative_ii_log_prob", +"test_sce_NCd1d2d3_none_no_weight_negative_ii_log_prob_expanded", +"test_sce_NCd1d2d3_sum_weight_high_ii", +"test_sce_NCd1d2d3_sum_weight_high_ii_expanded", +"test_sce_NCd1d2d3_sum_weight_high_ii_log_prob", +"test_sce_NCd1d2d3_sum_weight_high_ii_log_prob_expanded", +"test_sce_NCd1d2d3d4d5_mean_weight", +"test_sce_NCd1d2d3d4d5_mean_weight_expanded", +"test_sce_NCd1d2d3d4d5_mean_weight_log_prob", +"test_sce_NCd1d2d3d4d5_mean_weight_log_prob_expanded", +"test_sce_NCd1d2d3d4d5_none_no_weight", +"test_sce_NCd1d2d3d4d5_none_no_weight_expanded", +"test_sce_NCd1d2d3d4d5_none_no_weight_log_prob", +"test_sce_NCd1d2d3d4d5_none_no_weight_log_prob_expanded", +"test_sce_mean", +"test_sce_mean_3d", +"test_sce_mean_3d_expanded", +"test_sce_mean_3d_log_prob", +"test_sce_mean_3d_log_prob_expanded", +"test_sce_mean_expanded", +"test_sce_mean_log_prob", +"test_sce_mean_log_prob_expanded", +"test_sce_mean_no_weight_ii", +"test_sce_mean_no_weight_ii_3d", +"test_sce_mean_no_weight_ii_3d_expanded", +"test_sce_mean_no_weight_ii_3d_log_prob", +"test_sce_mean_no_weight_ii_3d_log_prob_expanded", +"test_sce_mean_no_weight_ii_4d", +"test_sce_mean_no_weight_ii_4d_expanded", +"test_sce_mean_no_weight_ii_4d_log_prob", +"test_sce_mean_no_weight_ii_4d_log_prob_expanded", +"test_sce_mean_no_weight_ii_expanded", +"test_sce_mean_no_weight_ii_log_prob", +"test_sce_mean_no_weight_ii_log_prob_expanded", +"test_sce_mean_weight", +"test_sce_mean_weight_expanded", +"test_sce_mean_weight_ii", +"test_sce_mean_weight_ii_3d", +"test_sce_mean_weight_ii_3d_expanded", +"test_sce_mean_weight_ii_3d_log_prob", +"test_sce_mean_weight_ii_3d_log_prob_expanded", +"test_sce_mean_weight_ii_4d", +"test_sce_mean_weight_ii_4d_expanded", +"test_sce_mean_weight_ii_4d_log_prob", +"test_sce_mean_weight_ii_4d_log_prob_expanded", +"test_sce_mean_weight_ii_expanded", +"test_sce_mean_weight_ii_log_prob", +"test_sce_mean_weight_ii_log_prob_expanded", +"test_sce_mean_weight_log_prob", +"test_sce_mean_weight_log_prob_expanded", +"test_sce_none", +"test_sce_none_expanded", +"test_sce_none_log_prob", +"test_sce_none_log_prob_expanded", +"test_sce_none_weights", +"test_sce_none_weights_expanded", +"test_sce_none_weights_log_prob", +"test_sce_none_weights_log_prob_expanded", +"test_sce_sum", +"test_sce_sum_expanded", +"test_sce_sum_log_prob", +"test_sce_sum_log_prob_expanded", +"test_selu", +"test_selu_default", +"test_selu_example", +"test_sequence_insert_at_back", +"test_sequence_insert_at_front", +"test_shape", +"test_shape_clip_end", +"test_shape_clip_start", +"test_shape_end_1", +"test_shape_end_negative_1", +"test_shape_example", +"test_shape_start_1", +"test_shape_start_1_end_2", +"test_shape_start_1_end_negative_1", +"test_shape_start_negative_1", +"test_shrink_hard", +"test_shrink_soft", +"test_sign", +"test_simple_rnn_batchwise", +"test_simple_rnn_defaults", +"test_simple_rnn_with_initial_bias", +"test_sin", +"test_sin_example", +"test_sinh", +"test_sinh_example", +"test_size", +"test_size_example", +"test_slice", +"test_slice_default_axes", +"test_slice_default_steps", +"test_slice_end_out_of_bounds", +"test_slice_neg", +"test_slice_neg_steps", +"test_slice_negative_axes", +"test_slice_start_out_of_bounds", +"test_softplus", +"test_softplus_example", +"test_softsign", +"test_softsign_example", +"test_spacetodepth", +"test_spacetodepth_example", +"test_split_variable_parts_1d", +"test_split_variable_parts_2d", +"test_split_variable_parts_default_axis", +"test_split_zero_size_splits", +"test_sqrt", +"test_sqrt_example", +"test_squeeze", +"test_squeeze_negative_axes", +"test_strnormalizer_export_monday_casesensintive_lower", +"test_strnormalizer_export_monday_casesensintive_nochangecase", +"test_strnormalizer_export_monday_casesensintive_upper", +"test_strnormalizer_export_monday_empty_output", +"test_strnormalizer_export_monday_insensintive_upper_twodim", +"test_strnormalizer_nostopwords_nochangecase", +"test_sub_example", +"test_sub_uint8", +"test_sum_example", +"test_sum_two_inputs", +"test_tan", +"test_tan_example", +"test_tfidfvectorizer_tf_batch_onlybigrams_skip0", +"test_tfidfvectorizer_tf_batch_onlybigrams_skip5", +"test_tfidfvectorizer_tf_batch_uniandbigrams_skip5", +"test_tfidfvectorizer_tf_only_bigrams_skip0", +"test_tfidfvectorizer_tf_onlybigrams_levelempty", +"test_tfidfvectorizer_tf_onlybigrams_skip5", +"test_tfidfvectorizer_tf_uniandbigrams_skip5", +"test_thresholdedrelu", +"test_thresholdedrelu_default", +"test_thresholdedrelu_example", +"test_tile", +"test_tile_precomputed", +"test_top_k", +"test_top_k_negative_axis", +"test_top_k_smallest", +"test_training_dropout", +"test_training_dropout_default", +"test_training_dropout_default_mask", +"test_training_dropout_mask", +"test_training_dropout_zero_ratio", +"test_training_dropout_zero_ratio_mask", +"test_tril", +"test_tril_neg", +"test_tril_one_row_neg", +"test_tril_out_neg", +"test_tril_out_pos", +"test_tril_pos", +"test_tril_square", +"test_tril_square_neg", +"test_tril_zero", +"test_triu", +"test_triu_neg", +"test_triu_one_row", +"test_triu_out_neg_out", +"test_triu_out_pos", +"test_triu_pos", +"test_triu_square", +"test_triu_square_neg", +"test_triu_zero", +"test_unique_not_sorted_without_axis", +"test_unique_sorted_with_axis", +"test_unique_sorted_with_axis_3d", +"test_unique_sorted_with_negative_axis", +"test_unique_sorted_without_axis", +"test_unsqueeze_axis_0", +"test_unsqueeze_axis_1", +"test_unsqueeze_axis_2", +"test_unsqueeze_negative_axes", +"test_unsqueeze_three_axes", +"test_unsqueeze_two_axes", +"test_unsqueeze_unsorted_axes", +"test_where_example", +"test_where_long_example", +"test_xor2d", +"test_xor3d", +"test_xor4d", +"test_xor_bcast3v1d", +"test_xor_bcast3v2d", +"test_xor_bcast4v2d", +"test_xor_bcast4v3d", +"test_xor_bcast4v4d", diff --git a/modules/ts/misc/testlog_parser.py b/modules/ts/misc/testlog_parser.py index 2e9718be3e..f52a051c8f 100755 --- a/modules/ts/misc/testlog_parser.py +++ b/modules/ts/misc/testlog_parser.py @@ -182,9 +182,12 @@ class TestInfo(object): return 1 return 0 + def __lt__(self, other): + return self.__cmp__(other) == -1 + # This is a Sequence for compatibility with old scripts, # which treat parseLogFile's return value as a list. -class TestRunInfo(collections.Sequence): +class TestRunInfo(object): def __init__(self, properties, tests): self.properties = properties self.tests = tests From d8b1fc45aae30480ea033e5f9393ebe40ca65f99 Mon Sep 17 00:00:00 2001 From: Sinitsina Maria <49319156+SinM9@users.noreply.github.com> Date: Tue, 14 Dec 2021 20:33:26 +0300 Subject: [PATCH 169/226] Merge pull request #20934 from SinM9:spectrogram_samples AudioIO: add spectrogram samples for C++/python --- samples/cpp/audio_spectrogram.cpp | 1071 +++++++++++++++++++++++++++ samples/python/audio_spectrogram.py | 804 ++++++++++++++++++++ 2 files changed, 1875 insertions(+) create mode 100644 samples/cpp/audio_spectrogram.cpp create mode 100644 samples/python/audio_spectrogram.py diff --git a/samples/cpp/audio_spectrogram.cpp b/samples/cpp/audio_spectrogram.cpp new file mode 100644 index 0000000000..80bfc1fbde --- /dev/null +++ b/samples/cpp/audio_spectrogram.cpp @@ -0,0 +1,1071 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +using namespace cv; +using namespace std; + + +class AudioDrawing +{ + +public: + + AudioDrawing(const CommandLineParser& parser) { + if (!initAndCheckArgs(parser)) + { + cerr << "Error: Wrong input arguments" << endl; + exit(0); + } + Draw(); + } + + void Draw() { + if (draw == "static") + { + vectorinputAudio = {}; + int samplingRate = 0; + if (inputType == "file") + { + samplingRate = readAudioFile(audio, inputAudio); + } + else if (inputType == "microphone") + { + samplingRate = readAudioMicrophone(inputAudio); + } + if ((inputAudio.size() == 0) || samplingRate <= 0) + { + cerr << "Error: problems with audio reading, check input arguments" << endl; + return; + } + + int duration = static_cast(inputAudio.size()) / samplingRate; + + // since the dimensional grid is counted in integer seconds, + // if the input audio has an incomplete last second, + // then it is filled with zeros to complete + int remainder = static_cast(inputAudio.size()) % samplingRate; + if (remainder) + { + int sizeToFullSec = samplingRate - remainder; + for (int j = 0; j < sizeToFullSec; ++j) + { + inputAudio.push_back(0); + } + duration += 1; + cout << "Update duration of audio to full last second with " << + sizeToFullSec << " zero samples" << endl; + cout << "New number of samples " << inputAudio.size() << endl; + } + cout << "Duration of audio = " << duration << " seconds" << endl; + + // since the dimensional grid is counted in integer seconds, + // if duration of file is less than xmarkup, to avoid an incorrect display, + // xmarkup will be taken equal to duration + if (duration <= xmarkup) + { + xmarkup = duration + 1; + } + + if (graph == "ampl") + { + Mat imgAmplitude = drawAmplitude(inputAudio); + imgAmplitude = drawAmplitudeScale(imgAmplitude, inputAudio, samplingRate); + imshow("Display amplitude graph", imgAmplitude); + waitKey(0); + } + else if (graph == "spec") + { + vector>stft = STFT(inputAudio); + Mat imgSpec = drawSpectrogram(stft); + imgSpec = drawSpectrogramColorbar(imgSpec, inputAudio, samplingRate, stft); + imshow("Display spectrogram", imgSpec); + waitKey(0); + } + else if (graph == "ampl_and_spec") + { + Mat imgAmplitude = drawAmplitude(inputAudio); + imgAmplitude = drawAmplitudeScale(imgAmplitude, inputAudio, samplingRate); + vector>stft = STFT(inputAudio); + Mat imgSpec = drawSpectrogram(stft); + imgSpec = drawSpectrogramColorbar(imgSpec, inputAudio, samplingRate, stft); + Mat imgTotal = concatenateImages(imgAmplitude, imgSpec); + imshow("Display amplitude graph and spectrogram", imgTotal); + waitKey(0); + } + } + else if (draw == "dynamic") + { + if (inputType == "file") + { + dynamicFile(audio); + } + else if (inputType == "microphone") + { + dynamicMicrophone(); + } + } + } + + ~AudioDrawing() { + } + + int readAudioFile(string file, vector& inputAudio) + { + VideoCapture cap; + vector params { CAP_PROP_AUDIO_STREAM, audioStream, + CAP_PROP_VIDEO_STREAM, -1, + CAP_PROP_AUDIO_DATA_DEPTH, CV_16S }; + + cap.open(file, CAP_ANY, params); + if (!cap.isOpened()) + { + cerr << "Error : Can't read audio file: '" << audio << "' with audioStream = " << audioStream << endl; + return -1; + } + const int audioBaseIndex = (int)cap.get(CAP_PROP_AUDIO_BASE_INDEX); + const int numberOfChannels = (int)cap.get(CAP_PROP_AUDIO_TOTAL_CHANNELS); + cout << "CAP_PROP_AUDIO_DATA_DEPTH: " << depthToString((int)cap.get(CAP_PROP_AUDIO_DATA_DEPTH)) << endl; + int samplingRate = static_cast(cap.get(CAP_PROP_AUDIO_SAMPLES_PER_SECOND)); + cout << "CAP_PROP_AUDIO_SAMPLES_PER_SECOND: " << cap.get(CAP_PROP_AUDIO_SAMPLES_PER_SECOND) << endl; + cout << "CAP_PROP_AUDIO_TOTAL_CHANNELS: " << numberOfChannels << endl; + cout << "CAP_PROP_AUDIO_TOTAL_STREAMS: " << cap.get(CAP_PROP_AUDIO_TOTAL_STREAMS) << endl; + + vector frameVec; + Mat frame; + for (;;) + { + if (cap.grab()) + { + cap.retrieve(frame, audioBaseIndex); + frameVec = frame; + inputAudio.insert(inputAudio.end(), frameVec.begin(), frameVec.end()); + } + else + { + cout << "Number of samples: " << inputAudio.size() << endl; + break; + } + } + return samplingRate; + } + + int readAudioMicrophone(vector& inputAudio) + { + VideoCapture cap; + vector params { CAP_PROP_AUDIO_STREAM, 0, + CAP_PROP_VIDEO_STREAM, -1 }; + + cap.open(0, CAP_ANY, params); + if (!cap.isOpened()) + { + cerr << "Error: Can't open microphone" << endl; + return -1; + } + + const int audioBaseIndex = static_cast(cap.get(CAP_PROP_AUDIO_BASE_INDEX)); + const int numberOfChannels = static_cast(cap.get(CAP_PROP_AUDIO_TOTAL_CHANNELS)); + cout << "CAP_PROP_AUDIO_DATA_DEPTH: " << depthToString( static_cast(cap.get(CAP_PROP_AUDIO_DATA_DEPTH))) << endl; + int samplingRate = static_cast(cap.get(CAP_PROP_AUDIO_SAMPLES_PER_SECOND)); + cout << "CAP_PROP_AUDIO_SAMPLES_PER_SECOND: " << samplingRate << endl; + cout << "CAP_PROP_AUDIO_TOTAL_CHANNELS: " << numberOfChannels << endl; + cout << "CAP_PROP_AUDIO_TOTAL_STREAMS: " << cap.get(CAP_PROP_AUDIO_TOTAL_STREAMS) << endl; + + const double cvTickFreq = getTickFrequency(); + int64 sysTimeCurr = getTickCount(); + int64 sysTimePrev = sysTimeCurr; + + vector frameVec; + Mat frame; + while ((sysTimeCurr - sysTimePrev) / cvTickFreq < microTime) + { + if (cap.grab()) + { + cap.retrieve(frame, audioBaseIndex); + frameVec = frame; + inputAudio.insert(inputAudio.end(), frameVec.begin(), frameVec.end()); + sysTimeCurr = getTickCount(); + } + else + { + cerr << "Error: Grab error" << endl; + break; + } + } + cout << "Number of samples: " << inputAudio.size() << endl; + return samplingRate; + } + + + Mat drawAmplitude(vector& inputAudio) + { + Scalar color = Scalar(247,111,87); + int thickness = 5; + int frameVectorRows = 500; + int middle = frameVectorRows / 2; + // usually the input data is too big, so it is necessary + // to reduce size using interpolation of data + int frameVectorCols = 40000; + if (static_cast(inputAudio.size()) < frameVectorCols) + { + frameVectorCols = static_cast(inputAudio.size()); + } + + Mat img(frameVectorRows, frameVectorCols, CV_8UC3 , Scalar(255,255,255)); // white background + + vectorreshapeAudio(inputAudio.size()); + for (size_t i = 0; i < inputAudio.size(); ++i) + { + reshapeAudio[i]=static_cast(inputAudio[i]); + } + + Mat img_frameVector( 1, static_cast(reshapeAudio.size()), CV_64F , reshapeAudio.data()); + Mat img_frameVector_resize; + resize(img_frameVector, img_frameVector_resize, Size(frameVectorCols, 1), INTER_LINEAR); + reshapeAudio = img_frameVector_resize; + + // normalization data by maximum element + normalize(reshapeAudio, reshapeAudio, 1.0, 0.0, NORM_INF); + + for (size_t i = 0; i < reshapeAudio.size(); ++i) + { + reshapeAudio[i] = middle - reshapeAudio[i] * middle; + } + + for (int i = 1; i < static_cast(reshapeAudio.size()); ++i) + { + line(img, Point(i-1, static_cast(reshapeAudio[i-1])), Point(i, static_cast(reshapeAudio[i])), color, thickness); + } + Mat resImage; + resize(img, resImage, Size(900, 400), INTER_AREA ); + return resImage; + } + + Mat drawAmplitudeScale(Mat& inputImg, const vector& inputAudio, int samplingRate, + int xmin = 0, int xmax = 0) + { + // function of layout drawing for graph of volume amplitudes + // x axis for time + // y axis for amplitudes + + // parameters for the new image size + int preCol = 100; + int aftCol = 100; + int preLine = 40; + int aftLine = 50; + + int frameVectorRows = inputImg.rows; + int frameVectorCols = inputImg.cols; + + int totalRows = preLine + frameVectorRows + aftLine; + int totalCols = preCol + frameVectorCols + aftCol; + + Mat imgTotal = Mat(totalRows, totalCols, CV_8UC3, Scalar(255, 255, 255)); + inputImg.copyTo(imgTotal(Rect(preCol, preLine, inputImg.cols, inputImg.rows))); + + + // calculating values on x axis + if (xmax == 0) + { + xmax = static_cast(inputAudio.size()) / samplingRate; + } + std::vector xList(xmarkup); + if (xmax >= xmarkup) + { + double deltax = (xmax - xmin) / (xmarkup - 1); + for (int i = 0; i < xmarkup; ++i) + { + xList[i] = (xmin + deltax * i); + } + } + else + { + // this case is used to display a dynamic update + vector tmpXList; + for (int i = xmin; i < xmax; ++i) + { + tmpXList.push_back(i + 1); + } + int k = 0; + for (int i = xmarkup - static_cast(tmpXList.size()); i < xmarkup; ++i) + { + xList[i] = tmpXList[k]; + k += 1; + } + } + + // calculating values on y axis + double minCv; double maxCv; Point minLoc; Point maxLoc; + minMaxLoc(inputAudio, &minCv, &maxCv, &minLoc, &maxLoc); + int ymin = static_cast(minCv); + int ymax = static_cast(maxCv); + + std::vector yList(ymarkup); + double deltay = (ymax - ymin) / (ymarkup - 1); + for (int i = 0; i < ymarkup; ++i) + { + yList[i] = ymin + deltay * i; + } + + // parameters for layout drawing + int textThickness = 1; + int gridThickness = 1; + Scalar gridColor(0, 0, 0); + Scalar textColor(0, 0, 0); + float fontScale = 0.5; + + // horizontal axis + line(imgTotal, Point(preCol, totalRows - aftLine), Point(preCol + frameVectorCols, totalRows - aftLine), + gridColor, gridThickness); + // vertical axis + line(imgTotal, Point(preCol, preLine), Point(preCol, preLine + frameVectorRows), + gridColor, gridThickness); + + // parameters for layout calculation + int serifSize = 10; + int indentDownX = serifSize * 2; + int indentDownY = serifSize / 2; + int indentLeftX = serifSize; + int indentLeftY = 2 * preCol / 3; + + + // drawing layout for x axis + int numX = frameVectorCols / (xmarkup - 1); + for (size_t i = 0; i < xList.size(); ++i) + { + int a1 = static_cast(preCol + i * numX); + int a2 = frameVectorRows + preLine; + + int b1 = a1; + int b2 = a2 + serifSize; + + if (enableGrid) + { + int d1 = a1; + int d2 = preLine; + line(imgTotal, Point(a1, a2), Point(d1, d2), gridColor, gridThickness); + } + line(imgTotal, Point(a1, a2), Point(b1, b2), gridColor, gridThickness); + putText(imgTotal, to_string(int(xList[i])), Point(b1 - indentLeftX, b2 + indentDownX), + FONT_HERSHEY_SIMPLEX, fontScale, textColor, textThickness); + } + + // drawing layout for y axis + int numY = frameVectorRows / (ymarkup - 1); + for (size_t i = 0; i < yList.size(); ++i) { + int a1 = preCol; + int a2 = static_cast(totalRows - aftLine - i * numY); + int b1 = preCol - serifSize; + int b2 = a2; + if (enableGrid) + { + int d1 = preCol + frameVectorCols; + int d2 = a2; + line(imgTotal, Point(a1, a2), Point(d1, d2), gridColor, gridThickness); + } + line(imgTotal, Point(a1, a2), Point(b1, b2), gridColor, gridThickness); + putText(imgTotal, to_string(int(yList[i])), Point(b1 - indentLeftY, b2 + indentDownY), + FONT_HERSHEY_SIMPLEX, fontScale, textColor, textThickness); + } + Mat resImage; + resize(imgTotal, resImage, Size(cols, rows), INTER_AREA ); + return resImage; + } + + vector> STFT(const vector& inputAudio) + { + // The Short-time Fourier transform (STFT), is a Fourier-related transform used to + // determine the sinusoidal frequency and phase content of local sections of a signal + // as it changes over time. + // In practice, the procedure for computing STFTs is to divide a longer time signal + // into shorter segments of equal length and then compute the Fourier transform separately + // on each shorter segment. This reveals the Fourier spectrum on each shorter segment. + // One then usually plots the changing spectra as a function of time, known as a spectrogram + // or waterfall plot. + // https://en.wikipedia.org/wiki/Short-time_Fourier_transform + + int timeStep = windLen - overlap; + Mat dstMat; + vector stftRow; + vector WindType; + if (windowType == "Hann") + { + // https://en.wikipedia.org/wiki/Window_function#Hann_and_Hamming_windows + for (int j = 1 - windLen; j < windLen; j+=2) + { + WindType.push_back(j * (0.5 * (1 - cos(CV_PI * j / (windLen - 1))))); + } + } + else if (windowType == "Hamming") + { + // https://en.wikipedia.org/wiki/Window_function#Hann_and_Hamming_windows + for (int j = 1 - windLen; j < windLen; j+=2) + { + WindType.push_back(j * (0.53836 - 0.46164 * (cos(CV_PI * j / (windLen - 1))))); + } + } + for (size_t i = 0; i < inputAudio.size(); i += timeStep) + { + vectorsection(windLen, 0); + for (int j = 0; j < windLen; ++j) + { + section[j] = inputAudio[j + i]; + } + if (windowType == "Hann" || windowType == "Hamming") + { + for (size_t j = 0; j < section.size(); ++j) + { + section[j] *= WindType[j]; + } + } + + dft(section, dstMat, DFT_COMPLEX_OUTPUT); + + for (int j = 0; j < dstMat.cols / 4; ++j) + { + double complModule = sqrt(dstMat.at(2*j) * dstMat.at(2*j) + + dstMat.at(2*j+1) * dstMat.at(2*j+1)); + stftRow.push_back(complModule); + } + } + + size_t xSize = inputAudio.size() / timeStep + 1; + // we need only the first part of the spectrum, the second part is symmetrical + size_t ySize = dstMat.cols / 4; + + vector> stft(ySize, vector(xSize, 0.)); + for (size_t i = 0; i < xSize; ++i) + { + for (size_t j = 0; j < ySize; ++j) + { + // write elements with transposition and convert it to the decibel scale + double stftElem = stftRow[ i * ySize + j]; + if (stftElem != 0.) + { + stft[j][i] = 10 * log10(stftElem); + } + } + } + return stft; + } + + Mat drawSpectrogram(const vector>& stft) + { + int frameVectorRows = static_cast(stft.size()); + int frameVectorCols = static_cast(stft[0].size()); + + // Normalization of image values from 0 to 255 to get more contrast image + // and this normalization will be taken into account in the scale drawing + int colormapImageRows = 255; + + double minCv; double maxCv; Point minLoc; Point maxLoc; + minMaxLoc(stft[0], &minCv, &maxCv, &minLoc, &maxLoc); + double maxStft = max(abs(maxCv), abs(minCv)); + + for (int i = 1; i < frameVectorRows; ++i) + { + minMaxLoc( stft[i], &minCv, &maxCv, &minLoc, &maxLoc); + maxStft = max(maxStft, max(abs(maxCv), abs(minCv))); + } + // if maxStft is zero (silence) + if (maxStft == 0.) + { + maxStft = 1; + } + Mat imgSpec(frameVectorRows, frameVectorCols, CV_8UC1, Scalar(255, 255, 255)); + + for (int i = 0; i < frameVectorRows; ++i) + { + for (int j = 0; j < frameVectorCols; ++j) + { + imgSpec.at(frameVectorRows - i - 1, j) = static_cast(stft[i][j] * colormapImageRows / maxStft); + } + } + applyColorMap(imgSpec, imgSpec, COLORMAP_INFERNO); + Mat resImage; + resize(imgSpec, resImage, Size(900, 400), INTER_AREA); + return resImage; + } + + Mat drawSpectrogramColorbar(Mat& inputImg, const vector& inputAudio, + int samplingRate, const vector>& stft, + int xmin = 0, int xmax = 0) + { + // function of layout drawing for the three-dimensional graph of the spectrogram + // x axis for time + // y axis for frequencies + // z axis for magnitudes of frequencies shown by color scale + + // parameters for the new image size + int preCol = 100; + int aftCol = 100; + int preLine = 40; + int aftLine = 50; + int colColor = 20; + int indCol = 20; + + int frameVectorRows = inputImg.rows; + int frameVectorCols = inputImg.cols; + + int totalRows = preLine + frameVectorRows + aftLine; + int totalCols = preCol + frameVectorCols + aftCol; + + Mat imgTotal = Mat(totalRows, totalCols, CV_8UC3 , Scalar(255, 255, 255)); + inputImg.copyTo(imgTotal(Rect(preCol, preLine, frameVectorCols, frameVectorRows))); + + // colorbar image due to drawSpectrogram(..) picture has been normalised from 255 to 0, + // so here colorbar has values from 255 to 0 + int colorArrSize = 256; + Mat imgColorBar = Mat (colorArrSize, colColor, CV_8UC1 , Scalar(255,255,255)); + for (int i = 0; i < colorArrSize; ++i) + { + for( int j = 0; j < colColor; ++j) + { + imgColorBar.at(i, j) = static_cast(colorArrSize - 1 - i); // from 255 to 0 + } + } + + applyColorMap(imgColorBar, imgColorBar, COLORMAP_INFERNO); + resize(imgColorBar, imgColorBar, Size(colColor, frameVectorRows), INTER_AREA); + imgColorBar.copyTo(imgTotal(Rect(preCol + frameVectorCols + indCol, preLine, colColor, frameVectorRows))); + + + // calculating values on x axis + if (xmax == 0) + { + xmax = static_cast(inputAudio.size()) / samplingRate + 1; + } + vector xList(xmarkup, 0); + if (xmax >= xmarkup) + { + double deltax = (xmax - xmin) / (xmarkup - 1); + for(int i = 0; i < xmarkup; ++i) + { + xList[i] = xmin + deltax * i; + } + } + else + { + // this case is used to display a dynamic update + vector tmpXList; + for(int i = xmin; i < xmax; ++i) + { + tmpXList.push_back(i + 1); + } + int k = 0; + for (int i = xmarkup - static_cast(tmpXList.size()); i < xmarkup; ++i) + { + xList[i] = tmpXList[k]; + k += 1; + } + } + + // calculating values on y axis + // according to the Nyquist sampling theorem, + // signal should posses frequencies equal to half of sampling rate + int ymin = 0; + int ymax = static_cast(samplingRate / 2); + + vector yList; + double deltay = (ymax - ymin) / (ymarkup - 1); + for(int i = 0; i < ymarkup; ++i) + { + yList.push_back(ymin + deltay * i); + } + + // calculating values on z axis + double minCv; double maxCv; Point minLoc; Point maxLoc; + minMaxLoc( stft[0], &minCv, &maxCv, &minLoc, &maxLoc); + double zmin = minCv, zmax = maxCv; + + std::vector zList; + for (size_t i = 1; i < stft.size(); ++i) + { + minMaxLoc( stft[i], &minCv, &maxCv, &minLoc, &maxLoc); + zmax = max(zmax, maxCv); + zmin = min(zmin, minCv); + } + double deltaz = (zmax - zmin) / (zmarkup - 1); + for(int i = 0; i < zmarkup; ++i) + { + zList.push_back(zmin + deltaz * i); + } + + // parameters for layout drawing + int textThickness = 1; + int gridThickness = 1; + Scalar gridColor(0,0,0); + Scalar textColor(0,0,0); + float fontScale = 0.5; + + int serifSize = 10; + int indentDownX = serifSize * 2; + int indentDownY = serifSize / 2; + int indentLeftX = serifSize; + int indentLeftY = 2 * preCol / 3; + + // horizontal axis + line(imgTotal, Point(preCol, totalRows - aftLine), Point(preCol + frameVectorCols, totalRows - aftLine), + gridColor, gridThickness); + // vertical axis + line(imgTotal, Point(preCol, preLine), Point(preCol, preLine + frameVectorRows), + gridColor, gridThickness); + + // drawing layout for x axis + int numX = frameVectorCols / (xmarkup - 1); + for (size_t i = 0; i < xList.size(); ++i) + { + int a1 = static_cast(preCol + i * numX); + int a2 = frameVectorRows + preLine; + + int b1 = a1; + int b2 = a2 + serifSize; + + line(imgTotal, Point(a1, a2), Point(b1, b2), gridColor, gridThickness); + putText(imgTotal, to_string(static_cast(xList[i])), Point(b1 - indentLeftX, b2 + indentDownX), + FONT_HERSHEY_SIMPLEX, fontScale, textColor, textThickness); + } + + // drawing layout for y axis + int numY = frameVectorRows / (ymarkup - 1); + for (size_t i = 0; i < yList.size(); ++i) + { + int a1 = preCol; + int a2 = static_cast(totalRows - aftLine - i * numY); + + int b1 = preCol - serifSize; + int b2 = a2; + + line(imgTotal, Point(a1, a2), Point(b1, b2), gridColor, gridThickness); + putText(imgTotal, to_string(static_cast(yList[i])), Point(b1 - indentLeftY, b2 + indentDownY), + FONT_HERSHEY_SIMPLEX, fontScale, textColor, textThickness); + } + + // drawing layout for z axis + int numZ = frameVectorRows / (zmarkup - 1); + for (size_t i = 0; i < zList.size(); ++i) + { + int a1 = preCol + frameVectorCols + indCol + colColor; + int a2 = static_cast(totalRows - aftLine - i * numZ); + + int b1 = a1 + serifSize; + int b2 = a2; + + line(imgTotal, Point(a1, a2), Point(b1, b2), gridColor, gridThickness); + putText(imgTotal, to_string(static_cast(zList[i])), Point(b1 + 10, b2 + indentDownY), + FONT_HERSHEY_SIMPLEX, fontScale, textColor, textThickness); + } + Mat resImage; + resize(imgTotal, resImage, Size(cols, rows), INTER_AREA ); + return resImage; + } + + Mat concatenateImages(Mat& img1, Mat& img2) + { + // first image will be under the second image + int totalRows = img1.rows + img2.rows; + int totalCols = max(img1.cols , img2.cols); + // if images columns do not match, the difference is filled in white + Mat imgTotal = Mat (totalRows, totalCols, CV_8UC3 , Scalar(255, 255, 255)); + + img1.copyTo(imgTotal(Rect(0, 0, img1.cols, img1.rows))); + img2.copyTo(imgTotal(Rect(0, img1.rows, img2.cols, img2.rows))); + return imgTotal; + } + + void dynamicFile(const string file) + { + VideoCapture cap; + vector params { CAP_PROP_AUDIO_STREAM, audioStream, + CAP_PROP_VIDEO_STREAM, -1, + CAP_PROP_AUDIO_DATA_DEPTH, CV_16S }; + + cap.open(file, CAP_ANY, params); + if (!cap.isOpened()) + { + cerr << "Error : Can't read audio file: '" << audio << "' with audioStream = " << audioStream << endl; + return; + } + + const int audioBaseIndex = static_cast(cap.get(CAP_PROP_AUDIO_BASE_INDEX)); + const int numberOfChannels = static_cast(cap.get(CAP_PROP_AUDIO_TOTAL_CHANNELS)); + int samplingRate = static_cast(cap.get(CAP_PROP_AUDIO_SAMPLES_PER_SECOND)); + + cout << "CAP_PROP_AUDIO_DATA_DEPTH: " << depthToString(static_cast(cap.get(CAP_PROP_AUDIO_DATA_DEPTH))) << endl; + cout << "CAP_PROP_AUDIO_SAMPLES_PER_SECOND: " << cap.get(CAP_PROP_AUDIO_SAMPLES_PER_SECOND) << endl; + cout << "CAP_PROP_AUDIO_TOTAL_CHANNELS: " << numberOfChannels << endl; + cout << "CAP_PROP_AUDIO_TOTAL_STREAMS: " << cap.get(CAP_PROP_AUDIO_TOTAL_STREAMS) << endl; + + int step = static_cast(updateTime * samplingRate); + int frameSize = static_cast(frameSizeTime * samplingRate); + + // since the dimensional grid is counted in integer seconds, + // if duration of audio frame is less than xmarkup, to avoid an incorrect display, + // xmarkup will be taken equal to duration + if (frameSizeTime <= xmarkup) + { + xmarkup = frameSizeTime; + } + + vector buffer; + vector frameVector; + vector section(frameSize, 0); + vector>stft; + Mat frame, imgAmplitude, imgSpec, imgTotal; + int currentSamples = 0; + int xmin = 0; + int xmax = 0; + + for (;;) + { + if (cap.grab()) + { + cap.retrieve(frame, audioBaseIndex); + frameVector = frame; + buffer.insert(buffer.end(), frameVector.begin(), frameVector.end()); + int bufferSize = static_cast(buffer.size()); + if (bufferSize >= step) + { + currentSamples += bufferSize; + section.erase(section.begin(), section.begin() + step); + section.insert(section.end(), buffer.begin(), buffer.end()); + buffer.erase(buffer.begin(), buffer.begin() + step); + if (currentSamples < frameSize) + { + xmin = 0; + xmax = (currentSamples) / samplingRate; + } + else + { + xmin = (currentSamples - frameSize) / samplingRate + 1; + xmax = (currentSamples) / samplingRate; + } + + if (graph == "ampl") + { + imgAmplitude = drawAmplitude(section); + imgAmplitude = drawAmplitudeScale(imgAmplitude, section, samplingRate, xmin, xmax); + imshow("Display amplitude graph", imgAmplitude); + waitKey(waitTime); + } + else if (graph == "spec") + { + stft = STFT(section); + imgSpec = drawSpectrogram(stft); + imgSpec = drawSpectrogramColorbar(imgSpec, section, samplingRate, stft, xmin, xmax); + imshow("Display spectrogram", imgSpec); + waitKey(waitTime); + } + else if (graph == "ampl_and_spec") + { + imgAmplitude = drawAmplitude(section); + imgAmplitude = drawAmplitudeScale(imgAmplitude, section, samplingRate, xmin, xmax); + stft = STFT(section); + imgSpec = drawSpectrogram(stft); + imgSpec = drawSpectrogramColorbar(imgSpec, section, samplingRate, stft, xmin, xmax); + imgTotal = concatenateImages(imgAmplitude, imgSpec); + imshow("Display amplitude graph and spectrogram", imgTotal); + waitKey(waitTime); + } + } + } + else + { + break; + } + } + + } + + void dynamicMicrophone() + { + VideoCapture cap; + vector params { CAP_PROP_AUDIO_STREAM, 0, + CAP_PROP_VIDEO_STREAM, -1 }; + + cap.open(0, CAP_MSMF, params); + if (!cap.isOpened()) + { + cerr << "Error: Can't open microphone" << endl; + return; + } + + const int audioBaseIndex = static_cast(cap.get(CAP_PROP_AUDIO_BASE_INDEX)); + const int numberOfChannels = static_cast(cap.get(CAP_PROP_AUDIO_TOTAL_CHANNELS)); + int samplingRate = static_cast(cap.get(CAP_PROP_AUDIO_SAMPLES_PER_SECOND)); + cout << "CAP_PROP_AUDIO_DATA_DEPTH: " << depthToString(static_cast(cap.get(CAP_PROP_AUDIO_DATA_DEPTH))) << endl; + cout << "CAP_PROP_AUDIO_SAMPLES_PER_SECOND: " << cap.get(CAP_PROP_AUDIO_SAMPLES_PER_SECOND) << endl; + cout << "CAP_PROP_AUDIO_TOTAL_CHANNELS: " << numberOfChannels << endl; + cout << "CAP_PROP_AUDIO_TOTAL_STREAMS: " << cap.get(CAP_PROP_AUDIO_TOTAL_STREAMS) << endl; + + const double cvTickFreq = getTickFrequency(); + int64 sysTimeCurr = getTickCount(); + int64 sysTimePrev = sysTimeCurr; + + int step = (updateTime * samplingRate); + int frameSize = (frameSizeTime * samplingRate); + // since the dimensional grid is counted in integer seconds, + // if duration of audio frame is less than xmarkup, to avoid an incorrect display, + // xmarkup will be taken equal to duration + if (frameSizeTime <= xmarkup) + { + xmarkup = frameSizeTime; + } + + vector frameVector; + vector buffer; + vector section(frameSize, 0); + Mat frame, imgAmplitude, imgSpec, imgTotal; + + int currentSamples = 0; + vector> stft; + int xmin = 0; + int xmax = 0; + waitTime = updateTime * 1000; + while ((sysTimeCurr - sysTimePrev) / cvTickFreq < microTime) + { + if (cap.grab()) + { + cap.retrieve(frame, audioBaseIndex); + frameVector = frame; + buffer.insert(buffer.end(), frameVector.begin(), frameVector.end()); + sysTimeCurr = getTickCount(); + + int bufferSize = static_cast(buffer.size()); + if (bufferSize >= step) + { + currentSamples += step; + section.erase(section.begin(), section.begin() + step); + section.insert(section.end(), buffer.begin(), buffer.end()); + buffer.erase(buffer.begin(), buffer.begin() + step); + + if (currentSamples < frameSize) + { + xmin = 0; + xmax = (currentSamples) / samplingRate; + } + else + { + xmin = (currentSamples - frameSize) / samplingRate + 1; + xmax = (currentSamples) / samplingRate; + } + + if (graph == "ampl") + { + imgAmplitude = drawAmplitude(section); + imgAmplitude = drawAmplitudeScale(imgAmplitude, section, samplingRate, xmin, xmax); + imshow("Display amplitude graph", imgAmplitude); + waitKey(waitTime); + } + else if (graph == "spec") + { + stft = STFT(section); + imgSpec = drawSpectrogram(stft); + imgSpec = drawSpectrogramColorbar(imgSpec, section, samplingRate, stft, xmin, xmax); + imshow("Display spectrogram", imgSpec); + waitKey(waitTime); + } + else if (graph == "ampl_and_spec") + { + imgAmplitude = drawAmplitude(section); + imgAmplitude = drawAmplitudeScale(imgAmplitude, section, samplingRate, xmin, xmax); + stft = STFT(section); + imgSpec = drawSpectrogram(stft); + imgSpec = drawSpectrogramColorbar(imgSpec, section, samplingRate, stft, xmin, xmax); + imgTotal = concatenateImages(imgAmplitude, imgSpec); + imshow("Display amplitude graph and spectrogram", imgTotal); + waitKey(waitTime); + } + } + } + else + { + cerr << "Error: Grab error" << endl; + break; + } + } + + } + + bool initAndCheckArgs(const CommandLineParser& parser) + { + inputType = parser.get("inputType"); + if ((inputType != "file") && (inputType != "microphone")) + { + cout << "Error: " << inputType << " input method doesnt exist" << endl; + return false; + } + + draw = parser.get("draw"); + if ((draw != "static") && (draw != "dynamic")) + { + cout << "Error: " << draw << " draw type doesnt exist" << endl; + return false; + } + + graph = parser.get("graph"); + if ((graph != "ampl") && (graph != "spec") && (graph != "ampl_and_spec")) + { + cout << "Error: " << graph << " type of graph doesnt exist" << endl; + return false; + } + + audio = samples::findFile(parser.get("audio")); + + audioStream = parser.get("audioStream"); + if (audioStream < 0) + { + cout << "Error: audioStream = " << audioStream << " - incorrect value. Must be >= 0" << endl; + return false; + } + windowType = parser.get("windowType"); + if ((windowType != "Rect") && (windowType != "Hann") && (windowType != "Hamming")) + { + cout << "Error: " << windowType << " type of window doesnt exist" << endl; + return false; + } + + windLen = parser.get("windLen"); + if (windLen <= 0) + { + cout << "Error: windLen = " << windLen << " - incorrect value. Must be > 0" << endl; + return false; + } + + overlap = parser.get("overlap"); + if (overlap <= 0) + { + cout << "Error: overlap = " << overlap << " - incorrect value. Must be > 0" << endl; + return false; + } + + enableGrid = parser.get("enableGrid"); + + rows = parser.get("rows"); + if (rows <= 0) + { + cout << "Error: rows = " << rows << " - incorrect value. Must be > 0" << endl; + return false; + } + cols = parser.get("cols"); + + if (cols <= 0) + { + cout << "Error: cols = " << cols << " - incorrect value. Must be > 0" << endl; + return false; + } + xmarkup = parser.get("xmarkup"); + if (xmarkup < 2) + { + cout << "Error: xmarkup = " << xmarkup << " - incorrect value. Must be >= 2" << endl; + return false; + } + ymarkup = parser.get("ymarkup"); + if (ymarkup < 2) + { + cout << "Error: ymarkup = " << ymarkup << " - incorrect value. Must be >= 2" << endl; + return false; + } + zmarkup = parser.get("zmarkup"); + if (zmarkup < 2) + { + cout << "Error: zmarkup = " << zmarkup << " - incorrect value. Must be >= 2" << endl; + return false; + } + microTime = parser.get("microTime"); + if (microTime <= 0) + { + cout << "Error: microTime = " << microTime << " - incorrect value. Must be > 0" << endl; + return false; + } + frameSizeTime = parser.get("frameSizeTime"); + if (frameSizeTime <= 0) + { + cout << "Error: frameSizeTime = " << frameSizeTime << " - incorrect value. Must be > 0" << endl; + return false; + } + updateTime = parser.get("updateTime"); + if (updateTime <= 0) + { + cout << "Error: updateTime = " << updateTime << " - incorrect value. Must be > 0" << endl; + return false; + } + waitTime = parser.get("waitTime"); + if (waitTime < 0) + { + cout << "Error: waitTime = " << waitTime << " - incorrect value. Must be >= 0" << endl; + return false; + } + return true; + } + +private : + string inputType; + string draw; + string graph; + string audio; + int audioStream; + + string windowType; + int windLen; + int overlap; + + bool enableGrid; + + int rows; + int cols; + + int xmarkup; + int ymarkup; + int zmarkup; + + int microTime; + int frameSizeTime; + int updateTime; + int waitTime; + +}; + +int main(int argc, char** argv) +{ + const String keys = + "{help h usage ? | | this sample draws a volume graph and/or spectrogram of audio/video files and microphone \n\t\tDefault usage: ./Spectrogram.exe}" + "{inputType i | file | file or microphone }" + "{draw d | static | type of drawing: \n\t\t\tstatic - for plotting graph(s) across the entire input audio \n\t\t\tdynamic - for plotting graph(s) in a time-updating window}" + "{graph g | ampl_and_spec | type of graph: amplitude graph or/and spectrogram. Please use tags below : \n\t\t\tampl - draw the amplitude graph \n\t\t\tspec - draw the spectrogram\n\t\t\tampl_and_spec - draw the amplitude graph and spectrogram on one image under each other}" + "{audio a | Megamind.avi | name and path to file }" + "{audioStream s | 1 | CAP_PROP_AUDIO_STREAM value. Select audio stream number }" + "{windowType t | Rect | type of window for STFT. Please use tags below : \n\t\t\tRect/Hann/Hamming }" + "{windLen l | 256 | size of window for STFT }" + "{overlap o | 128 | overlap of windows for STFT }" + + "{enableGrid | false | grid on the amplitude graph }" + + "{rows r | 400 | rows of output image }" + "{cols c | 900 | cols of output image }" + + "{xmarkup x | 5 | number of x axis divisions (time asix) }" + "{ymarkup y | 5 | number of y axis divisions (frequency or/and amplitude axis) }" + "{zmarkup z | 5 | number of z axis divisions (colorbar) }" + + "{microTime m | 20 | time of recording audio with microphone in seconds }" + "{frameSizeTime f| 5 | size of sliding window in seconds }" + "{updateTime u | 1 | update time of sliding window in seconds }" + "{waitTime w | 10 | parameter to cv.waitKey() for dynamic update of file input, takes values in milliseconds }" + ; + + CommandLineParser parser(argc, argv, keys); + if (parser.has("help")) + { + parser.printMessage(); + return 0; + } + + AudioDrawing draw(parser); + return 0; +} \ No newline at end of file diff --git a/samples/python/audio_spectrogram.py b/samples/python/audio_spectrogram.py new file mode 100644 index 0000000000..f211bbb584 --- /dev/null +++ b/samples/python/audio_spectrogram.py @@ -0,0 +1,804 @@ +import numpy as np +import cv2 as cv +import math +import argparse + +class AudioDrawing: + ''' + Used for drawing audio graphics + ''' + def __init__(self, args): + + self.inputType = args.inputType + self.draw = args.draw + self.graph = args.graph + self.audio = cv.samples.findFile(args.audio) + self.audioStream = args.audioStream + + self.windowType = args.windowType + self.windLen = args.windLen + self.overlap = args.overlap + + self.enableGrid = args.enableGrid + + self.rows = args.rows + self.cols = args.cols + + self.xmarkup = args.xmarkup + self.ymarkup = args.ymarkup + self.zmarkup = args.zmarkup + + self.microTime = args.microTime + self.frameSizeTime = args.frameSizeTime + self.updateTime = args.updateTime + self.waitTime = args.waitTime + + if self.initAndCheckArgs(args) is False: + exit() + + + def Draw(self): + if self.draw == "static": + + if self.inputType == "file": + samplingRate, inputAudio = self.readAudioFile(self.audio) + + elif self.inputType == "microphone": + samplingRate, inputAudio = self.readAudioMicrophone() + + duration = len(inputAudio) // samplingRate + + # since the dimensional grid is counted in integer seconds, + # if the input audio has an incomplete last second, + # then it is filled with zeros to complete + remainder = len(inputAudio) % samplingRate + if remainder != 0: + sizeToFullSec = samplingRate - remainder + zeroArr = np.zeros(sizeToFullSec) + inputAudio = np.concatenate((inputAudio, zeroArr), axis=0) + duration += 1 + print("Update duration of audio to full second with ", + sizeToFullSec, " zero samples") + print("New number of samples ", len(inputAudio)) + + if duration <= self.xmarkup: + self.xmarkup = duration + 1 + + if self.graph == "ampl": + imgAmplitude = self.drawAmplitude(inputAudio) + imgAmplitude = self.drawAmplitudeScale(imgAmplitude, inputAudio, samplingRate) + cv.imshow("Display window", imgAmplitude) + cv.waitKey(0) + + elif self.graph == "spec": + stft = self.STFT(inputAudio) + imgSpec = self.drawSpectrogram(stft) + imgSpec = self.drawSpectrogramColorbar(imgSpec, inputAudio, samplingRate, stft) + cv.imshow("Display window", imgSpec) + cv.waitKey(0) + + elif self.graph == "ampl_and_spec": + imgAmplitude = self.drawAmplitude(inputAudio) + imgAmplitude = self.drawAmplitudeScale(imgAmplitude, inputAudio, samplingRate) + + stft = self.STFT(inputAudio) + imgSpec = self.drawSpectrogram(stft) + imgSpec = self.drawSpectrogramColorbar(imgSpec, inputAudio, samplingRate, stft) + + imgTotal = self.concatenateImages(imgAmplitude, imgSpec) + cv.imshow("Display window", imgTotal) + cv.waitKey(0) + + elif self.draw == "dynamic": + + if self.inputType == "file": + self.dynamicFile(self.audio) + + elif self.inputType == "microphone": + self.dynamicMicrophone() + + + def readAudioFile(self, file): + cap = cv.VideoCapture(file) + + params = [cv.CAP_PROP_AUDIO_STREAM, self.audioStream, + cv.CAP_PROP_VIDEO_STREAM, -1, + cv.CAP_PROP_AUDIO_DATA_DEPTH, cv.CV_16S] + params = np.asarray(params) + + cap.open(file, cv.CAP_ANY, params) + if cap.isOpened() == False: + print("Error : Can't read audio file: '", self.audio, "' with audioStream = ", self.audioStream) + print("Error: problems with audio reading, check input arguments") + exit() + audioBaseIndex = int(cap.get(cv.CAP_PROP_AUDIO_BASE_INDEX)) + numberOfChannels = int(cap.get(cv.CAP_PROP_AUDIO_TOTAL_CHANNELS)) + + print("CAP_PROP_AUDIO_DATA_DEPTH: ", str((int(cap.get(cv.CAP_PROP_AUDIO_DATA_DEPTH))))) + print("CAP_PROP_AUDIO_SAMPLES_PER_SECOND: ", cap.get(cv.CAP_PROP_AUDIO_SAMPLES_PER_SECOND)) + print("CAP_PROP_AUDIO_TOTAL_CHANNELS: ", numberOfChannels) + print("CAP_PROP_AUDIO_TOTAL_STREAMS: ", cap.get(cv.CAP_PROP_AUDIO_TOTAL_STREAMS)) + + frame = [] + frame = np.asarray(frame) + inputAudio = [] + + while (1): + if (cap.grab()): + frame = [] + frame = np.asarray(frame) + frame = cap.retrieve(frame, audioBaseIndex) + for i in range(len(frame[1][0])): + inputAudio.append(frame[1][0][i]) + else: + break + + inputAudio = np.asarray(inputAudio) + print("Number of samples: ", len(inputAudio)) + samplingRate = int(cap.get(cv.CAP_PROP_AUDIO_SAMPLES_PER_SECOND)) + return samplingRate, inputAudio + + + def readAudioMicrophone(self): + cap = cv.VideoCapture() + + params = [cv.CAP_PROP_AUDIO_STREAM, 0, cv.CAP_PROP_VIDEO_STREAM, -1] + params = np.asarray(params) + + cap.open(0, cv.CAP_ANY, params) + if cap.isOpened() == False: + print("Error: Can't open microphone") + print("Error: problems with audio reading, check input arguments") + exit() + audioBaseIndex = int(cap.get(cv.CAP_PROP_AUDIO_BASE_INDEX)) + numberOfChannels = int(cap.get(cv.CAP_PROP_AUDIO_TOTAL_CHANNELS)) + + print("CAP_PROP_AUDIO_DATA_DEPTH: ", str((int(cap.get(cv.CAP_PROP_AUDIO_DATA_DEPTH))))) + print("CAP_PROP_AUDIO_SAMPLES_PER_SECOND: ", cap.get(cv.CAP_PROP_AUDIO_SAMPLES_PER_SECOND)) + print("CAP_PROP_AUDIO_TOTAL_CHANNELS: ", numberOfChannels) + print("CAP_PROP_AUDIO_TOTAL_STREAMS: ", cap.get(cv.CAP_PROP_AUDIO_TOTAL_STREAMS)) + + cvTickFreq = cv.getTickFrequency() + sysTimeCurr = cv.getTickCount() + sysTimePrev = sysTimeCurr + + frame = [] + frame = np.asarray(frame) + inputAudio = [] + + while ((sysTimeCurr - sysTimePrev) / cvTickFreq < self.microTime): + if (cap.grab()): + frame = [] + frame = np.asarray(frame) + frame = cap.retrieve(frame, audioBaseIndex) + for i in range(len(frame[1][0])): + inputAudio.append(frame[1][0][i]) + sysTimeCurr = cv.getTickCount() + else: + print("Error: Grab error") + break + + inputAudio = np.asarray(inputAudio) + print("Number of samples: ", len(inputAudio)) + samplingRate = int(cap.get(cv.CAP_PROP_AUDIO_SAMPLES_PER_SECOND)) + + return samplingRate, inputAudio + + + def drawAmplitude(self, inputAudio): + color = (247, 111, 87) + thickness = 5 + frameVectorRows = 500 + middle = frameVectorRows // 2 + + # usually the input data is too big, so it is necessary + # to reduce size using interpolation of data + frameVectorCols = 40000 + if len(inputAudio) < frameVectorCols: + frameVectorCols = len(inputAudio) + + img = np.zeros((frameVectorRows, frameVectorCols, 3), np.uint8) + img += 255 # white background + + audio = np.array(0) + audio = cv.resize(inputAudio, (1, frameVectorCols), interpolation=cv.INTER_LINEAR) + reshapeAudio = np.reshape(audio, (-1)) + + # normalization data by maximum element + minCv, maxCv, _, _ = cv.minMaxLoc(reshapeAudio) + maxElem = int(max(abs(minCv), abs(maxCv))) + + # if all data values are zero (silence) + if maxElem == 0: + maxElem = 1 + for i in range(len(reshapeAudio)): + reshapeAudio[i] = middle - reshapeAudio[i] * middle // maxElem + + for i in range(1, frameVectorCols, 1): + cv.line(img, (i - 1, int(reshapeAudio[i - 1])), (i, int(reshapeAudio[i])), color, thickness) + + img = cv.resize(img, (900, 400), interpolation=cv.INTER_AREA) + return img + + + def drawAmplitudeScale(self, inputImg, inputAudio, samplingRate, xmin=None, xmax=None): + # function of layout drawing for graph of volume amplitudes + # x axis for time + # y axis for amplitudes + + # parameters for the new image size + preCol = 100 + aftCol = 100 + preLine = 40 + aftLine = 50 + + frameVectorRows = inputImg.shape[0] + frameVectorCols = inputImg.shape[1] + + totalRows = preLine + frameVectorRows + aftLine + totalCols = preCol + frameVectorCols + aftCol + + imgTotal = np.zeros((totalRows, totalCols, 3), np.uint8) + imgTotal += 255 # white background + imgTotal[preLine: preLine + frameVectorRows, preCol: preCol + frameVectorCols] = inputImg + + # calculating values on x axis + if xmin is None: + xmin = 0 + if xmax is None: + xmax = len(inputAudio) / samplingRate + + if xmax > self.xmarkup: + xList = np.linspace(xmin, xmax, self.xmarkup).astype(int) + else: + # this case is used to display a dynamic update + tmp = np.arange(xmin, xmax, 1).astype(int) + 1 + xList = np.concatenate((np.zeros(self.xmarkup - len(tmp)), tmp[:]), axis=None) + + # calculating values on y axis + ymin = np.min(inputAudio) + ymax = np.max(inputAudio) + yList = np.linspace(ymin, ymax, self.ymarkup) + + # parameters for layout drawing + textThickness = 1 + gridThickness = 1 + gridColor = (0, 0, 0) + textColor = (0, 0, 0) + font = cv.FONT_HERSHEY_SIMPLEX + fontScale = 0.5 + + # horizontal axis under the graph + cv.line(imgTotal, (preCol, totalRows - aftLine), + (preCol + frameVectorCols, totalRows - aftLine), + gridColor, gridThickness) + # vertical axis for amplitude + cv.line(imgTotal, (preCol, preLine), (preCol, preLine + frameVectorRows), + gridColor, gridThickness) + + # parameters for layout calculation + serifSize = 10 + indentDownX = serifSize * 2 + indentDownY = serifSize // 2 + indentLeftX = serifSize + indentLeftY = 2 * preCol // 3 + + # drawing layout for x axis + numX = frameVectorCols // (self.xmarkup - 1) + for i in range(len(xList)): + a1 = preCol + i * numX + a2 = frameVectorRows + preLine + b1 = a1 + b2 = a2 + serifSize + if self.enableGrid is True: + d1 = a1 + d2 = preLine + cv.line(imgTotal, (a1, a2), (d1, d2), gridColor, gridThickness) + cv.line(imgTotal, (a1, a2), (b1, b2), gridColor, gridThickness) + cv.putText(imgTotal, str(int(xList[i])), (b1 - indentLeftX, b2 + indentDownX), + font, fontScale, textColor, textThickness) + + # drawing layout for y axis + numY = frameVectorRows // (self.ymarkup - 1) + for i in range(len(yList)): + a1 = preCol + a2 = totalRows - aftLine - i * numY + b1 = preCol - serifSize + b2 = a2 + if self.enableGrid is True: + d1 = preCol + frameVectorCols + d2 = a2 + cv.line(imgTotal, (a1, a2), (d1, d2), gridColor, gridThickness) + cv.line(imgTotal, (a1, a2), (b1, b2), gridColor, gridThickness) + cv.putText(imgTotal, str(int(yList[i])), (b1 - indentLeftY, b2 + indentDownY), + font, fontScale, textColor, textThickness) + imgTotal = cv.resize(imgTotal, (self.cols, self.rows), interpolation=cv.INTER_AREA) + return imgTotal + + + def STFT(self, inputAudio): + """ + The Short-time Fourier transform (STFT), is a Fourier-related transform used to determine + the sinusoidal frequency and phase content of local sections of a signal as it changes over + time. + In practice, the procedure for computing STFTs is to divide a longer time signal into + shorter segments of equal length and then compute the Fourier transform separately on each + shorter segment. This reveals the Fourier spectrum on each shorter segment. One then usually + plots the changing spectra as a function of time, known as a spectrogram or waterfall plot. + + https://en.wikipedia.org/wiki/Short-time_Fourier_transform + """ + + time_step = self.windLen - self.overlap + stft = [] + + if self.windowType == "Hann": + # https://en.wikipedia.org/wiki/Window_function#Hann_and_Hamming_windows + Hann_wind = [] + for i in range (1 - self.windLen, self.windLen, 2): + Hann_wind.append(i * (0.5 + 0.5 * math.cos(math.pi * i / (self.windLen - 1)))) + Hann_wind = np.asarray(Hann_wind) + + elif self.windowType == "Hamming": + # https://en.wikipedia.org/wiki/Window_function#Hann_and_Hamming_windows + Hamming_wind = [] + for i in range (1 - self.windLen, self.windLen, 2): + Hamming_wind.append(i * (0.53836 - 0.46164 * (math.cos(2 * math.pi * i / (self.windLen - 1))))) + Hamming_wind = np.asarray(Hamming_wind) + + for index in np.arange(0, len(inputAudio), time_step).astype(int): + + section = inputAudio[index:index + self.windLen] + zeroArray = np.zeros(self.windLen - len(section)) + section = np.concatenate((section, zeroArray), axis=None) + + if self.windowType == "Hann": + section *= Hann_wind + elif self.windowType == "Hamming": + section *= Hamming_wind + + dst = np.empty(0) + dst = cv.dft(section, dst, flags=cv.DFT_COMPLEX_OUTPUT) + reshape_dst = np.reshape(dst, (-1)) + # we need only the first part of the spectrum, the second part is symmetrical + complexArr = np.zeros(len(dst) // 4, dtype=complex) + for i in range(len(dst) // 4): + complexArr[i] = complex(reshape_dst[2 * i], reshape_dst[2 * i + 1]) + stft.append(np.abs(complexArr)) + + stft = np.array(stft).transpose() + # convert elements to the decibel scale + np.log10(stft, out=stft, where=(stft != 0.)) + return 10 * stft + + + def drawSpectrogram(self, stft): + + frameVectorRows = stft.shape[0] + frameVectorCols = stft.shape[1] + + # Normalization of image values from 0 to 255 to get more contrast image + # and this normalization will be taken into account in the scale drawing + colormapImageRows = 255 + + imgSpec = np.zeros((frameVectorRows, frameVectorCols, 3), np.uint8) + stftMat = np.zeros((frameVectorRows, frameVectorCols), np.float64) + cv.normalize(stft, stftMat, 1.0, 0.0, cv.NORM_INF) + + for i in range(frameVectorRows): + for j in range(frameVectorCols): + imgSpec[frameVectorRows - i - 1, j] = int(stftMat[i][j] * colormapImageRows) + + imgSpec = cv.applyColorMap(imgSpec, cv.COLORMAP_INFERNO) + imgSpec = cv.resize(imgSpec, (900, 400), interpolation=cv.INTER_LINEAR) + return imgSpec + + + def drawSpectrogramColorbar(self, inputImg, inputAudio, samplingRate, stft, xmin=None, xmax=None): + # function of layout drawing for the three-dimensional graph of the spectrogram + # x axis for time + # y axis for frequencies + # z axis for magnitudes of frequencies shown by color scale + + # parameters for the new image size + preCol = 100 + aftCol = 100 + preLine = 40 + aftLine = 50 + colColor = 20 + ind_col = 20 + + frameVectorRows = inputImg.shape[0] + frameVectorCols = inputImg.shape[1] + + totalRows = preLine + frameVectorRows + aftLine + totalCols = preCol + frameVectorCols + aftCol + colColor + + imgTotal = np.zeros((totalRows, totalCols, 3), np.uint8) + imgTotal += 255 # white background + imgTotal[preLine: preLine + frameVectorRows, preCol: preCol + frameVectorCols] = inputImg + + # colorbar image due to drawSpectrogram(..) picture has been normalised from 255 to 0, + # so here colorbar has values from 255 to 0 + colorArrSize = 256 + imgColorBar = np.zeros((colorArrSize, colColor, 1), np.uint8) + + for i in range(colorArrSize): + imgColorBar[i] += colorArrSize - 1 - i + + imgColorBar = cv.applyColorMap(imgColorBar, cv.COLORMAP_INFERNO) + imgColorBar = cv.resize(imgColorBar, (colColor, frameVectorRows), interpolation=cv.INTER_AREA) # + + imgTotal[preLine: preLine + frameVectorRows, + preCol + frameVectorCols + ind_col: + preCol + frameVectorCols + ind_col + colColor] = imgColorBar + + # calculating values on x axis + if xmin is None: + xmin = 0 + if xmax is None: + xmax = len(inputAudio) / samplingRate + if xmax > self.xmarkup: + xList = np.linspace(xmin, xmax, self.xmarkup).astype(int) + else: + # this case is used to display a dynamic update + tmpXList = np.arange(xmin, xmax, 1).astype(int) + 1 + xList = np.concatenate((np.zeros(self.xmarkup - len(tmpXList)), tmpXList[:]), axis=None) + + # calculating values on y axis + # according to the Nyquist sampling theorem, + # signal should posses frequencies equal to half of sampling rate + ymin = 0 + ymax = int(samplingRate / 2.) + yList = np.linspace(ymin, ymax, self.ymarkup).astype(int) + + # calculating values on z axis + zList = np.linspace(np.min(stft), np.max(stft), self.zmarkup) + + # parameters for layout drawing + textThickness = 1 + textColor = (0, 0, 0) + gridThickness = 1 + gridColor = (0, 0, 0) + font = cv.FONT_HERSHEY_SIMPLEX + fontScale = 0.5 + + serifSize = 10 + indentDownX = serifSize * 2 + indentDownY = serifSize // 2 + indentLeftX = serifSize + indentLeftY = 2 * preCol // 3 + + # horizontal axis + cv.line(imgTotal, (preCol, totalRows - aftLine), (preCol + frameVectorCols, totalRows - aftLine), + gridColor, gridThickness) + # vertical axis + cv.line(imgTotal, (preCol, preLine), (preCol, preLine + frameVectorRows), + gridColor, gridThickness) + + # drawing layout for x axis + numX = frameVectorCols // (self.xmarkup - 1) + for i in range(len(xList)): + a1 = preCol + i * numX + a2 = frameVectorRows + preLine + b1 = a1 + b2 = a2 + serifSize + cv.line(imgTotal, (a1, a2), (b1, b2), gridColor, gridThickness) + cv.putText(imgTotal, str(int(xList[i])), (b1 - indentLeftX, b2 + indentDownX), + font, fontScale, textColor, textThickness) + + # drawing layout for y axis + numY = frameVectorRows // (self.ymarkup - 1) + for i in range(len(yList)): + a1 = preCol + a2 = totalRows - aftLine - i * numY + b1 = preCol - serifSize + b2 = a2 + cv.line(imgTotal, (a1, a2), (b1, b2), gridColor, gridThickness) + cv.putText(imgTotal, str(int(yList[i])), (b1 - indentLeftY, b2 + indentDownY), + font, fontScale, textColor, textThickness) + + # drawing layout for z axis + numZ = frameVectorRows // (self.zmarkup - 1) + for i in range(len(zList)): + a1 = preCol + frameVectorCols + ind_col + colColor + a2 = totalRows - aftLine - i * numZ + b1 = a1 + serifSize + b2 = a2 + cv.line(imgTotal, (a1, a2), (b1, b2), gridColor, gridThickness) + cv.putText(imgTotal, str(int(zList[i])), (b1 + 10, b2 + indentDownY), + font, fontScale, textColor, textThickness) + imgTotal = cv.resize(imgTotal, (self.cols, self.rows), interpolation=cv.INTER_AREA) + return imgTotal + + + def concatenateImages(self, img1, img2): + # first image will be under the second image + totalRows = img1.shape[0] + img2.shape[0] + totalCols = max(img1.shape[1], img2.shape[1]) + + # if images columns do not match, the difference is filled in white + imgTotal = np.zeros((totalRows, totalCols, 3), np.uint8) + imgTotal += 255 + + imgTotal[:img1.shape[0], :img1.shape[1]] = img1 + imgTotal[img2.shape[0]:, :img2.shape[1]] = img2 + + return imgTotal + + + def dynamicFile(self, file): + cap = cv.VideoCapture(file) + params = [cv.CAP_PROP_AUDIO_STREAM, self.audioStream, + cv.CAP_PROP_VIDEO_STREAM, -1, + cv.CAP_PROP_AUDIO_DATA_DEPTH, cv.CV_16S] + params = np.asarray(params) + + cap.open(file, cv.CAP_ANY, params) + if cap.isOpened() == False: + print("ERROR! Can't to open file") + return + + audioBaseIndex = int(cap.get(cv.CAP_PROP_AUDIO_BASE_INDEX)) + numberOfChannels = int(cap.get(cv.CAP_PROP_AUDIO_TOTAL_CHANNELS)) + samplingRate = int(cap.get(cv.CAP_PROP_AUDIO_SAMPLES_PER_SECOND)) + + print("CAP_PROP_AUDIO_DATA_DEPTH: ", str((int(cap.get(cv.CAP_PROP_AUDIO_DATA_DEPTH))))) + print("CAP_PROP_AUDIO_SAMPLES_PER_SECOND: ", cap.get(cv.CAP_PROP_AUDIO_SAMPLES_PER_SECOND)) + print("CAP_PROP_AUDIO_TOTAL_CHANNELS: ", numberOfChannels) + print("CAP_PROP_AUDIO_TOTAL_STREAMS: ", cap.get(cv.CAP_PROP_AUDIO_TOTAL_STREAMS)) + + step = int(self.updateTime * samplingRate) + frameSize = int(self.frameSizeTime * samplingRate) + # since the dimensional grid is counted in integer seconds, + # if duration of audio frame is less than xmarkup, to avoid an incorrect display, + # xmarkup will be taken equal to duration + if self.frameSizeTime <= self.xmarkup: + self.xmarkup = self.frameSizeTime + + buffer = [] + section = np.zeros(frameSize, dtype=np.int16) + currentSamples = 0 + + while (1): + if (cap.grab()): + frame = [] + frame = np.asarray(frame) + frame = cap.retrieve(frame, audioBaseIndex) + + for i in range(len(frame[1][0])): + buffer.append(frame[1][0][i]) + + buffer_size = len(buffer) + if (buffer_size >= step): + + section = list(section) + currentSamples += step + + del section[0:step] + section.extend(buffer[0:step]) + del buffer[0:step] + + section = np.asarray(section) + + if currentSamples < frameSize: + xmin = 0 + xmax = (currentSamples) / samplingRate + else: + xmin = (currentSamples - frameSize) / samplingRate + 1 + xmax = (currentSamples) / samplingRate + + if self.graph == "ampl": + imgAmplitude = self.drawAmplitude(section) + imgAmplitude = self.drawAmplitudeScale(imgAmplitude, section, samplingRate, xmin, xmax) + cv.imshow("Display amplitude graph", imgAmplitude) + cv.waitKey(self.waitTime) + + elif self.graph == "spec": + stft = self.STFT(section) + imgSpec = self.drawSpectrogram(stft) + imgSpec = self.drawSpectrogramColorbar(imgSpec, section, samplingRate, stft, xmin, xmax) + cv.imshow("Display spectrogram", imgSpec) + cv.waitKey(self.waitTime) + + elif self.graph == "ampl_and_spec": + + imgAmplitude = self.drawAmplitude(section) + stft = self.STFT(section) + imgSpec = self.drawSpectrogram(stft) + + imgAmplitude = self.drawAmplitudeScale(imgAmplitude, section, samplingRate, xmin, xmax) + imgSpec = self.drawSpectrogramColorbar(imgSpec, section, samplingRate, stft, xmin, xmax) + + imgTotal = self.concatenateImages(imgAmplitude, imgSpec) + cv.imshow("Display amplitude graph and spectrogram", imgTotal) + cv.waitKey(self.waitTime) + else: + break + + + def dynamicMicrophone(self): + cap = cv.VideoCapture() + params = [cv.CAP_PROP_AUDIO_STREAM, 0, cv.CAP_PROP_VIDEO_STREAM, -1] + params = np.asarray(params) + + cap.open(0, cv.CAP_ANY, params) + if cap.isOpened() == False: + print("ERROR! Can't to open file") + return + audioBaseIndex = int(cap.get(cv.CAP_PROP_AUDIO_BASE_INDEX)) + numberOfChannels = int(cap.get(cv.CAP_PROP_AUDIO_TOTAL_CHANNELS)) + + print("CAP_PROP_AUDIO_DATA_DEPTH: ", str((int(cap.get(cv.CAP_PROP_AUDIO_DATA_DEPTH))))) + print("CAP_PROP_AUDIO_SAMPLES_PER_SECOND: ", cap.get(cv.CAP_PROP_AUDIO_SAMPLES_PER_SECOND)) + print("CAP_PROP_AUDIO_TOTAL_CHANNELS: ", numberOfChannels) + print("CAP_PROP_AUDIO_TOTAL_STREAMS: ", cap.get(cv.CAP_PROP_AUDIO_TOTAL_STREAMS)) + + frame = [] + frame = np.asarray(frame) + samplingRate = int(cap.get(cv.CAP_PROP_AUDIO_SAMPLES_PER_SECOND)) + + step = int(self.updateTime * samplingRate) + frameSize = int(self.frameSizeTime * samplingRate) + self.xmarkup = self.frameSizeTime + + currentSamples = 0 + + buffer = [] + section = np.zeros(frameSize, dtype=np.int16) + + cvTickFreq = cv.getTickFrequency() + sysTimeCurr = cv.getTickCount() + sysTimePrev = sysTimeCurr + self.waitTime = self.updateTime * 1000 + while ((sysTimeCurr - sysTimePrev) / cvTickFreq < self.microTime): + if (cap.grab()): + frame = [] + frame = np.asarray(frame) + frame = cap.retrieve(frame, audioBaseIndex) + + for i in range(len(frame[1][0])): + buffer.append(frame[1][0][i]) + + sysTimeCurr = cv.getTickCount() + buffer_size = len(buffer) + if (buffer_size >= step): + + section = list(section) + currentSamples += step + + del section[0:step] + section.extend(buffer[0:step]) + del buffer[0:step] + + section = np.asarray(section) + + if currentSamples < frameSize: + xmin = 0 + xmax = (currentSamples) / samplingRate + else: + xmin = (currentSamples - frameSize) / samplingRate + 1 + xmax = (currentSamples) / samplingRate + + if self.graph == "ampl": + imgAmplitude = self.drawAmplitude(section) + imgAmplitude = self.drawAmplitudeScale(imgAmplitude, section, samplingRate, xmin, xmax) + cv.imshow("Display amplitude graph", imgAmplitude) + cv.waitKey(self.waitTime) + + elif self.graph == "spec": + stft = self.STFT(section) + imgSpec = self.drawSpectrogram(stft) + imgSpec = self.drawSpectrogramColorbar(imgSpec, section, samplingRate, stft, xmin, xmax) + cv.imshow("Display spectrogram", imgSpec) + cv.waitKey(self.waitTime) + + elif self.graph == "ampl_and_spec": + imgAmplitude = self.drawAmplitude(section) + stft = self.STFT(section) + imgSpec = self.drawSpectrogram(stft) + + imgAmplitude = self.drawAmplitudeScale(imgAmplitude, section, samplingRate, xmin, xmax) + imgSpec = self.drawSpectrogramColorbar(imgSpec, section, samplingRate, stft, xmin, xmax) + + imgTotal = self.concatenateImages(imgAmplitude, imgSpec) + cv.imshow("Display amplitude graph and spectrogram", imgTotal) + cv.waitKey(self.waitTime) + else: + break + + + def initAndCheckArgs(self, args): + if args.inputType != "file" and args.inputType != "microphone": + print("Error: ", args.inputType, " input method doesnt exist") + return False + if args.draw != "static" and args.draw != "dynamic": + print("Error: ", args.draw, " draw type doesnt exist") + return False + if args.graph != "ampl" and args.graph != "spec" and args.graph != "ampl_and_spec": + print("Error: ", args.graph, " type of graph doesnt exist") + return False + if args.windowType != "Rect" and args.windowType != "Hann" and args.windowType != "Hamming": + print("Error: ", args.windowType, " type of window doesnt exist") + return False + if args.windLen <= 0: + print("Error: windLen = ", args.windLen, " - incorrect value. Must be > 0") + return False + if args.overlap <= 0: + print("Error: overlap = ", args.overlap, " - incorrect value. Must be > 0") + return False + if args.rows <= 0: + print("Error: rows = ", args.rows, " - incorrect value. Must be > 0") + return False + if args.cols <= 0: + print("Error: cols = ", args.cols, " - incorrect value. Must be > 0") + return False + if args.xmarkup < 2: + print("Error: xmarkup = ", args.xmarkup, " - incorrect value. Must be >= 2") + return False + if args.ymarkup < 2: + print("Error: ymarkup = ", args.ymarkup, " - incorrect value. Must be >= 2") + return False + if args.zmarkup < 2: + print("Error: zmarkup = ", args.zmarkup, " - incorrect value. Must be >= 2") + return False + if args.microTime <= 0: + print("Error: microTime = ", args.microTime, " - incorrect value. Must be > 0") + return False + if args.frameSizeTime <= 0: + print("Error: frameSizeTime = ", args.frameSizeTime, " - incorrect value. Must be > 0") + return False + if args.updateTime <= 0: + print("Error: updateTime = ", args.updateTime, " - incorrect value. Must be > 0") + return False + if args.waitTime < 0: + print("Error: waitTime = ", args.waitTime, " - incorrect value. Must be >= 0") + return False + return True + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, + description='''this sample draws a volume graph and/or spectrogram of audio/video files and microphone\nDefault usage: ./Spectrogram.exe''') + + parser.add_argument("-i", "--inputType", dest="inputType", type=str, default="file", help="file or microphone") + parser.add_argument("-d", "--draw", dest="draw", type=str, default="static", + help="type of drawing: static - for plotting graph(s) across the entire input audio; dynamic - for plotting graph(s) in a time-updating window") + parser.add_argument("-g", "--graph", dest="graph", type=str, default="ampl_and_spec", + help="type of graph: amplitude graph or/and spectrogram. Please use tags below : ampl - draw the amplitude graph; spec - draw the spectrogram; ampl_and_spec - draw the amplitude graph and spectrogram on one image under each other") + + parser.add_argument("-a", "--audio", dest="audio", type=str, default='Megamind.avi', + help="name and path to file") + parser.add_argument("-s", "--audioStream", dest="audioStream", type=int, default=1, + help=" CAP_PROP_AUDIO_STREAM value") + + parser.add_argument("-t", '--windowType', dest="windowType", type=str, default="Rect", + help="type of window for STFT. Please use tags below : Rect/Hann/Hamming") + parser.add_argument("-l", '--windLen', dest="windLen", type=int, default=256, help="size of window for STFT") + parser.add_argument("-o", '--overlap', dest="overlap", type=int, default=128, help="overlap of windows for STFT") + + parser.add_argument("-gd", '--grid', dest="enableGrid", type=bool, default=False, help="grid on amplitude graph(on/off)") + + parser.add_argument("-r", '--rows', dest="rows", type=int, default=400, help="rows of output image") + parser.add_argument("-c", '--cols', dest="cols", type=int, default=900, help="cols of output image") + + parser.add_argument("-x", '--xmarkup', dest="xmarkup", type=int, default=5, + help="number of x axis divisions (time asix)") + parser.add_argument("-y", '--ymarkup', dest="ymarkup", type=int, default=5, + help="number of y axis divisions (frequency or/and amplitude axis)") # ? + parser.add_argument("-z", '--zmarkup', dest="zmarkup", type=int, default=5, + help="number of z axis divisions (colorbar)") # ? + + parser.add_argument("-m", '--microTime', dest="microTime", type=int, default=20, + help="time of recording audio with microphone in seconds") + parser.add_argument("-f", '--frameSizeTime', dest="frameSizeTime", type=int, default=5, + help="size of sliding window in seconds") + parser.add_argument("-u", '--updateTime', dest="updateTime", type=int, default=1, + help="update time of sliding window in seconds") + parser.add_argument("-w", '--waitTime', dest="waitTime", type=int, default=10, + help="parameter to cv.waitKey() for dynamic update, takes values in milliseconds") + + args = parser.parse_args() + + AudioDrawing(args).Draw() From e97c7e042b2e4a15b1b5a284873efbff8e0dce02 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Tue, 14 Dec 2021 20:54:43 +0300 Subject: [PATCH 170/226] fix max_unpool missing attributes, add default value of keepdims in reducemean/max/sum, add support for keepdims=true in full reduction branch, add new padding type to Pad --- modules/dnn/src/layers/padding_layer.cpp | 9 ++-- modules/dnn/src/onnx/onnx_importer.cpp | 48 +++++++++++++++++-- ..._conformance_layer_parser_denylist.inl.hpp | 5 -- 3 files changed, 49 insertions(+), 13 deletions(-) diff --git a/modules/dnn/src/layers/padding_layer.cpp b/modules/dnn/src/layers/padding_layer.cpp index af3dacdd7a..9d7ca23714 100644 --- a/modules/dnn/src/layers/padding_layer.cpp +++ b/modules/dnn/src/layers/padding_layer.cpp @@ -130,12 +130,13 @@ public: outputs[0].setTo(paddingValue); inputs[0].copyTo(outputs[0](dstRanges)); } - else if (paddingType == "reflect") + else if (paddingType == "reflect" || paddingType == "edge") { CV_Assert(inputs.size() == 1); CV_Assert(outputs.size() == 1); CV_Assert(inputs[0].dims == 4); CV_Assert(outputs[0].dims == 4); + int borderType = paddingType == "reflect" ? BORDER_REFLECT_101 : BORDER_REPLICATE; if (inputs[0].size[0] != outputs[0].size[0] || inputs[0].size[1] != outputs[0].size[1]) CV_Error(Error::StsNotImplemented, "Only spatial reflection padding is supported."); @@ -148,8 +149,8 @@ public: const int padBottom = outHeight - dstRanges[2].end; const int padLeft = dstRanges[3].start; const int padRight = outWidth - dstRanges[3].end; - CV_CheckLT(padTop, inpHeight, ""); CV_CheckLT(padBottom, inpHeight, ""); - CV_CheckLT(padLeft, inpWidth, ""); CV_CheckLT(padRight, inpWidth, ""); + CV_CheckLE(padTop, inpHeight, ""); CV_CheckLE(padBottom, inpHeight, ""); + CV_CheckLE(padLeft, inpWidth, ""); CV_CheckLE(padRight, inpWidth, ""); for (size_t n = 0; n < inputs[0].size[0]; ++n) { @@ -158,7 +159,7 @@ public: copyMakeBorder(getPlane(inputs[0], n, ch), getPlane(outputs[0], n, ch), padTop, padBottom, padLeft, padRight, - BORDER_REFLECT_101); + borderType); } } } diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 8ff91f5e44..052b1b8529 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -131,6 +131,7 @@ private: typedef void (ONNXImporter::*ONNXImporterNodeParser)(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); typedef std::map DispatchMap; + void parseMaxUnpool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseMaxPool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseAveragePool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseReduce (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); @@ -625,6 +626,41 @@ void setCeilMode(LayerParams& layerParams) } } +void ONNXImporter::parseMaxUnpool(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "MaxUnpool"; + + DictValue kernel_shape = layerParams.get("kernel_size"); + CV_Assert(kernel_shape.size() == 2); + layerParams.set("pool_k_w", kernel_shape.get(0)); + layerParams.set("pool_k_h", kernel_shape.get(1)); + + int pool_pad_w = 0, pool_pad_h = 0; + if (layerParams.has("pad")) + { + DictValue pads = layerParams.get("pad"); + CV_CheckEQ(pads.size(), 2, ""); + pool_pad_w = pads.get(0); + pool_pad_h = pads.get(1); + } + layerParams.set("pool_pad_w", pool_pad_w); + layerParams.set("pool_pad_h", pool_pad_h); + + + int pool_stride_w = 1, pool_stride_h = 1; + if (layerParams.has("stride")) + { + DictValue strides = layerParams.get("stride"); + CV_CheckEQ(strides.size(), 2, ""); + pool_stride_w = strides.get(0); + pool_stride_h = strides.get(1); + } + layerParams.set("pool_stride_w", pool_stride_w); + layerParams.set("pool_stride_h", pool_stride_h); + + addLayer(layerParams, node_proto); +} + void ONNXImporter::parseMaxPool(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { layerParams.type = "Pooling"; @@ -659,11 +695,11 @@ void ONNXImporter::parseReduce(LayerParams& layerParams, const opencv_onnx::Node pool = "AVE"; layerParams.set("pool", pool); layerParams.set("global_pooling", !layerParams.has("axes")); + bool keepdims = layerParams.get("keepdims", 1) == 1; if (layerParams.has("axes") && (layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax")) { MatShape inpShape = outShapes[node_proto.input(0)]; DictValue axes = layerParams.get("axes"); - bool keepdims = layerParams.get("keepdims"); MatShape targetShape; std::vector shouldDelete(inpShape.size(), false); for (int i = 0; i < axes.size(); i++) { @@ -771,7 +807,10 @@ void ONNXImporter::parseReduce(LayerParams& layerParams, const opencv_onnx::Node } else if (!layerParams.has("axes") && (layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax")) { - CV_CheckEQ(layerParams.get("keepdims"), 0, "layer only supports keepdims = false"); + IterShape_t shapeIt = outShapes.find(node_proto.input(0)); + CV_Assert(shapeIt != outShapes.end()); + const size_t dims = keepdims ? shapeIt->second.size() : 1; + LayerParams reshapeLp; reshapeLp.name = layerParams.name + "/reshape"; reshapeLp.type = "Reshape"; @@ -793,8 +832,8 @@ void ONNXImporter::parseReduce(LayerParams& layerParams, const opencv_onnx::Node addLayer(poolLp, node_proto); layerParams.type = "Reshape"; - int targetShape[] = {1}; - layerParams.set("dim", DictValue::arrayInt(&targetShape[0], 1)); + std::vector targetShape(dims, 1); + layerParams.set("dim", DictValue::arrayInt(targetShape.data(), targetShape.size())); node_proto.set_input(0, node_proto.output(0)); node_proto.set_output(0, layerParams.name); @@ -2341,6 +2380,7 @@ const ONNXImporter::DispatchMap ONNXImporter::buildDispatchMap() { DispatchMap dispatch; + dispatch["MaxUnpool"] = &ONNXImporter::parseMaxUnpool; dispatch["MaxPool"] = &ONNXImporter::parseMaxPool; dispatch["AveragePool"] = &ONNXImporter::parseAveragePool; dispatch["GlobalAveragePool"] = dispatch["GlobalMaxPool"] = dispatch["ReduceMean"] = dispatch["ReduceSum"] = diff --git a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp index 5f1d99d4bb..2489105e91 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp @@ -277,7 +277,6 @@ "test_max_uint8", "test_maxpool_2d_uint8", "test_maxunpool_export_with_output_shape", -"test_maxunpool_export_without_output_shape", "test_mean_example", "test_mean_one_input", "test_mean_two_inputs", @@ -436,10 +435,6 @@ "test_reduce_log_sum_exp_negative_axes_keepdims_example", "test_reduce_log_sum_exp_negative_axes_keepdims_random", "test_reduce_log_sum_negative_axes", -"test_reduce_max_default_axes_keepdim_example", -"test_reduce_max_default_axes_keepdims_random", -"test_reduce_mean_default_axes_keepdims_example", -"test_reduce_mean_default_axes_keepdims_random", "test_reduce_min_default_axes_keepdims_example", "test_reduce_min_default_axes_keepdims_random", "test_reduce_min_do_not_keepdims_example", From f3ba88c87c79115531b9d74dd78682b1d0151713 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 15 Dec 2021 08:57:50 +0000 Subject: [PATCH 171/226] dnn(test): update ONNX conformance filters --- modules/dnn/test/test_common.impl.hpp | 2 +- modules/dnn/test/test_onnx_conformance.cpp | 216 +- ...ance_layer_filter__halide_denylist.inl.hpp | 60 + ...conformance_layer_filter__openvino.inl.hpp | 1945 +++++++++++++++++ 4 files changed, 2106 insertions(+), 117 deletions(-) create mode 100644 modules/dnn/test/test_onnx_conformance_layer_filter__halide_denylist.inl.hpp create mode 100644 modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp diff --git a/modules/dnn/test/test_common.impl.hpp b/modules/dnn/test/test_common.impl.hpp index a55a8f788c..5546f68916 100644 --- a/modules/dnn/test/test_common.impl.hpp +++ b/modules/dnn/test/test_common.impl.hpp @@ -368,7 +368,7 @@ void initDNNTests() #if defined(HAVE_HALIDE) registerGlobalSkipTag( CV_TEST_TAG_DNN_SKIP_HALIDE - ) + ); #endif #if defined(INF_ENGINE_RELEASE) registerGlobalSkipTag( diff --git a/modules/dnn/test/test_onnx_conformance.cpp b/modules/dnn/test/test_onnx_conformance.cpp index a1d83bd614..71ac4bef19 100644 --- a/modules/dnn/test/test_onnx_conformance.cpp +++ b/modules/dnn/test/test_onnx_conformance.cpp @@ -898,24 +898,16 @@ static const TestCase testConformanceConfig[] = { }; -struct TestCaseInput -{ - std::vector input_paths; - std::vector output_paths; - std::string model_path; - std::string name; -}; - -std::ostream& operator<<(std::ostream& os, const TestCaseInput& test_case) +std::ostream& operator<<(std::ostream& os, const TestCase& test_case) { return os << test_case.name; } -typedef tuple > ONNXConfParams; +typedef tuple > ONNXConfParams; std::string printOnnxConfParams(const testing::TestParamInfo& params) { - TestCaseInput test_case = get<0>(params.param); + TestCase test_case = get<0>(params.param); Backend backend = get<0>(get<1>(params.param)); Target target = get<1>(get<1>(params.param)); @@ -928,45 +920,11 @@ std::string printOnnxConfParams(const testing::TestParamInfo& pa return ss.str(); } -template -static std::string _tf(TString filename, bool required = true) -{ - return findDataFile(std::string("dnn/onnx/") + filename, required); -} - -std::vector readTestCases() -{ - std::vector ret; - for (size_t i = 0; i < sizeof(testConformanceConfig) / sizeof(testConformanceConfig[0]); ++i) - { - const TestCase& test_case = testConformanceConfig[i]; - - TestCaseInput input; - - std::string prefix = cv::format("conformance/node/%s", test_case.name); - input.name = test_case.name; - input.model_path = _tf(cv::format("%s/model.onnx", prefix.c_str())); - - for (int i = 0; i < test_case.inputs; ++i) - { - input.input_paths.push_back(_tf(cv::format("%s/test_data_set_0/input_%d.pb", prefix.c_str(), i))); - } - - for (int i = 0; i < test_case.outputs; ++i) - { - input.output_paths.push_back(_tf(cv::format("%s/test_data_set_0/output_%d.pb", prefix.c_str(), i))); - } - - ret.push_back(input); - } - - return ret; -} - class Test_ONNX_conformance : public TestWithParam { public: - TestCaseInput test_case; + + TestCase test_case; Backend backend; Target target; @@ -978,6 +936,9 @@ public: static std::set opencl_fp16_deny_list; static std::set opencl_deny_list; static std::set cpu_deny_list; +#ifdef HAVE_HALIDE + static std::set halide_deny_list; +#endif Test_ONNX_conformance() { @@ -1059,68 +1020,16 @@ public: "" // dummy element of non empty list }; initDenyList(cpu_deny_list, cpu, sizeof(cpu)/sizeof(cpu[0])); + +#ifdef HAVE_HALIDE + const char* const halide_deny_list_[] = { + #include "test_onnx_conformance_layer_filter__halide_denylist.inl.hpp" + "" // dummy element of non empty list + }; + initDenyList(halide_deny_list, halide_deny_list_, sizeof(halide_deny_list_)/sizeof(halide_deny_list_[0])); +#endif } - void checkFilterLists() const - { - const std::string& name = test_case.name; - if(parser_deny_list.find(name) != parser_deny_list.end()) - { - applyTestTag(CV_TEST_TAG_DNN_SKIP_PARSER, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); - } - - if (backend == DNN_BACKEND_OPENCV) - { - if(global_deny_list.find(name) != global_deny_list.end()) - { - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); - } - if((target == DNN_TARGET_OPENCL_FP16) && (opencl_fp16_deny_list.find(name) != opencl_fp16_deny_list.end())) - { - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); - } - if((target == DNN_TARGET_OPENCL) && (opencl_deny_list.find(name) != opencl_deny_list.end())) - { - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_OPENCL, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); - } - if((target == DNN_TARGET_CPU) && (cpu_deny_list.find(name) != cpu_deny_list.end())) - { - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_CPU, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); - } - } -#if 0 //def HAVE_HALIDE - else if (backend == DNN_BACKEND_HALIDE) - { - #include "test_onnx_conformance_layer_filter__halide.inl.hpp" - } -#endif -#if 0 //def HAVE_INF_ENGINE - else if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - { - #include "test_onnx_conformance_layer_filter__ngraph.inl.hpp" - } -#endif -#if 0 //def HAVE_VULKAN - else if (backend == DNN_BACKEND_VKCOM) - { - #include "test_onnx_conformance_layer_filter__vulkan.inl.hpp" - } -#endif -#if 0 //def HAVE_CUDA - else if (backend == DNN_BACKEND_CUDA) - { - #include "test_onnx_conformance_layer_filter__cuda.inl.hpp" - } -#endif - else - { - std::ostringstream ss; - ss << "No test filter available for backend "; - PrintTo(backend, &ss); - ss << ". Run test by default"; - std::cout << ss.str() << std::endl; - } - } }; std::set Test_ONNX_conformance::parser_deny_list; @@ -1128,33 +1037,104 @@ std::set Test_ONNX_conformance::global_deny_list; std::set Test_ONNX_conformance::opencl_fp16_deny_list; std::set Test_ONNX_conformance::opencl_deny_list; std::set Test_ONNX_conformance::cpu_deny_list; +#ifdef HAVE_HALIDE +std::set Test_ONNX_conformance::halide_deny_list; +#endif TEST_P(Test_ONNX_conformance, Layer_Test) { - std::string name = test_case.name; + const std::string& name = test_case.name; ASSERT_FALSE(name.empty()); bool checkLayersFallbacks = true; bool checkAccuracy = true; - checkFilterLists(); + if (parser_deny_list.find(name) != parser_deny_list.end()) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_PARSER, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + + if (backend == DNN_BACKEND_OPENCV) + { + if (global_deny_list.find(name) != global_deny_list.end()) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + if ((target == DNN_TARGET_OPENCL_FP16) && (opencl_fp16_deny_list.find(name) != opencl_fp16_deny_list.end())) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + if ((target == DNN_TARGET_OPENCL) && (opencl_deny_list.find(name) != opencl_deny_list.end())) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL, CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + if ((target == DNN_TARGET_CPU) && (cpu_deny_list.find(name) != cpu_deny_list.end())) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_CPU, CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + } +#ifdef HAVE_HALIDE + else if (backend == DNN_BACKEND_HALIDE) + { + if (halide_deny_list.find(name) != halide_deny_list.end()) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_HALIDE, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + } +#endif +#ifdef HAVE_INF_ENGINE + else if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + #include "test_onnx_conformance_layer_filter__openvino.inl.hpp" + } +#endif +#if 0 //def HAVE_VULKAN + else if (backend == DNN_BACKEND_VKCOM) + { + #include "test_onnx_conformance_layer_filter__vulkan.inl.hpp" + } +#endif +#if 0 //def HAVE_CUDA + else if (backend == DNN_BACKEND_CUDA) + { + #include "test_onnx_conformance_layer_filter__cuda.inl.hpp" + } +#endif + else + { + std::ostringstream ss; + ss << "No test filter available for backend "; + PrintTo(backend, &ss); + ss << ". Run test by default"; + std::cout << ss.str() << std::endl; + } std::vector inputs; std::vector ref_outputs; + std::string prefix = cv::format("dnn/onnx/conformance/node/%s", test_case.name); + Net net; try { + std::string model_path = findDataFile(prefix + "/model.onnx"); + //cout << "Read ONNX inputs..." << endl; - std::transform(test_case.input_paths.begin(), test_case.input_paths.end(), - std::back_inserter(inputs), readTensorFromONNX); + for (int i = 0; i < test_case.inputs; ++i) + { + Mat input = readTensorFromONNX(findDataFile(prefix + cv::format("/test_data_set_0/input_%d.pb", i))); + inputs.push_back(input); + } //cout << "Read ONNX reference outputs..." << endl; - std::transform(test_case.output_paths.begin(), test_case.output_paths.end(), - std::back_inserter(ref_outputs), readTensorFromONNX); + for (int i = 0; i < test_case.outputs; ++i) + { + Mat output = readTensorFromONNX(findDataFile(prefix + cv::format("/test_data_set_0/output_%d.pb", i))); + ref_outputs.push_back(output); + } //cout << "Parse model..." << endl; - net = readNetFromONNX(test_case.model_path); + net = readNetFromONNX(model_path); if (net.empty()) { applyTestTag(CV_TEST_TAG_DNN_ERROR_PARSER); @@ -1244,7 +1224,11 @@ TEST_P(Test_ONNX_conformance, Layer_Test) } INSTANTIATE_TEST_CASE_P(/**/, Test_ONNX_conformance, - testing::Combine(testing::ValuesIn(readTestCases()), dnnBackendsAndTargets()), - printOnnxConfParams); + testing::Combine( + testing::ValuesIn(testConformanceConfig), + dnnBackendsAndTargets(/*withInferenceEngine=*/true, /*withHalide=*/true) + ), + printOnnxConfParams +); }; diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__halide_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__halide_denylist.inl.hpp new file mode 100644 index 0000000000..dd0a249081 --- /dev/null +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__halide_denylist.inl.hpp @@ -0,0 +1,60 @@ +"test_add", +"test_add_bcast", +"test_averagepool_2d_ceil", +"test_averagepool_2d_pads_count_include_pad", +"test_averagepool_2d_precomputed_pads_count_include_pad", +"test_averagepool_2d_precomputed_strides", +"test_averagepool_2d_same_lower", +"test_averagepool_2d_same_upper", +"test_cast_FLOAT_to_STRING", +"test_cast_STRING_to_FLOAT", +"test_castlike_FLOAT_to_STRING_expanded", +"test_castlike_STRING_to_FLOAT_expanded", +"test_concat_1d_axis_negative_1", +"test_concat_3d_axis_1", +"test_div", +"test_div_bcast", +"test_elu", +"test_elu_default", +"test_exp", +"test_flatten_axis0", +"test_flatten_axis2", +"test_flatten_axis3", +"test_flatten_negative_axis1", +"test_flatten_negative_axis2", +"test_flatten_negative_axis4", +"test_leakyrelu", +"test_leakyrelu_default", +"test_logsoftmax_axis_1", +"test_logsoftmax_axis_1_expanded", +"test_logsoftmax_default_axis", +"test_logsoftmax_example_1", +"test_logsoftmax_large_number", +"test_matmul_2d", +"test_matmul_3d", +"test_matmul_4d", +"test_maxpool_2d_dilations", +"test_maxpool_2d_same_lower", +"test_maxpool_with_argmax_2d_precomputed_pads", +"test_maxpool_with_argmax_2d_precomputed_strides", +"test_mul", +"test_mul_bcast", +"test_neg", +"test_reduce_max_default_axes_keepdim_example", +"test_reduce_max_default_axes_keepdims_random", +"test_reduce_max_do_not_keepdims_example", +"test_reduce_max_do_not_keepdims_random", +"test_reduce_max_keepdims_example", +"test_reduce_max_keepdims_random", +"test_reduce_max_negative_axes_keepdims_example", +"test_reduce_max_negative_axes_keepdims_random", +"test_relu", +"test_sigmoid", +"test_softmax_axis_1", +"test_softmax_axis_1_expanded", +"test_softmax_default_axis", +"test_softmax_large_number", +"test_sub", +"test_sub_bcast", +"test_tanh", +"test_upsample_nearest", diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp new file mode 100644 index 0000000000..25bb8dff9a --- /dev/null +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp @@ -0,0 +1,1945 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +// not a standalone file, see test_onnx_conformance.cpp +#if 0 +cout << "Filtering is disabled: OpenVINO" << endl; +#else + + +#if 0 +// Stats for --gtest_filter=*ONNX_conformance*NGRAPH* +[ SKIPSTAT ] TAG='dnn_skip_ie_myriadx' skip 48 tests +[ SKIPSTAT ] TAG='dnn_skip_ie' skip 0 tests (149 times in extra skip list) +[ SKIPSTAT ] TAG='dnn_skip_ie_ngraph' skip 0 tests (149 times in extra skip list) +[ SKIPSTAT ] TAG='dnn_skip_ie_cpu' skip 29 tests +[ SKIPSTAT ] TAG='dnn_skip_ie_ocl' skip 34 tests +[ SKIPSTAT ] TAG='dnn_skip_ie_ocl_fp16' skip 38 tests +#endif + + +#define SKIP_TAGS \ + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, \ + CV_TEST_TAG_DNN_SKIP_IE, \ + CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE +#define SKIP_(...) applyTestTag(__VA_ARGS__, SKIP_TAGS) +#define SKIP applyTestTag(tag_target_skip, SKIP_TAGS) +#define SKIP_CPU if (target == DNN_TARGET_CPU) applyTestTag(tag_target_skip, SKIP_TAGS) +#define SKIP_NON_CPU if (target != DNN_TARGET_CPU) applyTestTag(tag_target_skip, SKIP_TAGS) +#define SKIP_OPENCL if (target == DNN_TARGET_OPENCL) applyTestTag(tag_target_skip, SKIP_TAGS) +#define SKIP_OPENCL_FP16 if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(tag_target_skip, SKIP_TAGS) +#define SKIP_MYRIAD if (target == DNN_TARGET_MYRIAD) applyTestTag(tag_target_skip, SKIP_TAGS) + +std::string tag_target_skip = + (target == DNN_TARGET_CPU) ? CV_TEST_TAG_DNN_SKIP_IE_CPU : + (target == DNN_TARGET_OPENCL) ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : + (target == DNN_TARGET_OPENCL_FP16) ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16 : + (target == DNN_TARGET_MYRIAD) ? CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X : + ""; + +ASSERT_FALSE(name.empty()); + +#define EOF_LABEL exit_filter_opencv +#define BEGIN_SWITCH() \ +if (name.empty() /*false*/) \ +{ + +#define CASE(t) \ + goto EOF_LABEL; \ +} \ +if (name == #t) \ +{ \ + filterApplied = true; + +#define END_SWITCH() \ + goto EOF_LABEL; \ +} \ +EOF_LABEL: + +bool filterApplied = false; + +// Update note: execute /testdata/dnn/onnx/generate_conformance_list.py +BEGIN_SWITCH() +CASE(test_abs) + // no filter +CASE(test_acos) + // no filter +CASE(test_acos_example) + // no filter +CASE(test_acosh) + // no filter +CASE(test_acosh_example) + // no filter +CASE(test_adagrad) + // no filter +CASE(test_adagrad_multiple) + // no filter +CASE(test_adam) + // no filter +CASE(test_adam_multiple) + // no filter +CASE(test_add) + // no filter +CASE(test_add_bcast) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_add_uint8) + // no filter +CASE(test_and2d) + // no filter +CASE(test_and3d) + // no filter +CASE(test_and4d) + // no filter +CASE(test_and_bcast3v1d) + // no filter +CASE(test_and_bcast3v2d) + // no filter +CASE(test_and_bcast4v2d) + // no filter +CASE(test_and_bcast4v3d) + // no filter +CASE(test_and_bcast4v4d) + // no filter +CASE(test_argmax_default_axis_example) + // no filter +CASE(test_argmax_default_axis_example_select_last_index) + // no filter +CASE(test_argmax_default_axis_random) + // no filter +CASE(test_argmax_default_axis_random_select_last_index) + // no filter +CASE(test_argmax_keepdims_example) + // no filter +CASE(test_argmax_keepdims_example_select_last_index) + // no filter +CASE(test_argmax_keepdims_random) + // no filter +CASE(test_argmax_keepdims_random_select_last_index) + // no filter +CASE(test_argmax_negative_axis_keepdims_example) + // no filter +CASE(test_argmax_negative_axis_keepdims_example_select_last_index) + // no filter +CASE(test_argmax_negative_axis_keepdims_random) + // no filter +CASE(test_argmax_negative_axis_keepdims_random_select_last_index) + // no filter +CASE(test_argmax_no_keepdims_example) + // no filter +CASE(test_argmax_no_keepdims_example_select_last_index) + // no filter +CASE(test_argmax_no_keepdims_random) + // no filter +CASE(test_argmax_no_keepdims_random_select_last_index) + // no filter +CASE(test_argmin_default_axis_example) + // no filter +CASE(test_argmin_default_axis_example_select_last_index) + // no filter +CASE(test_argmin_default_axis_random) + // no filter +CASE(test_argmin_default_axis_random_select_last_index) + // no filter +CASE(test_argmin_keepdims_example) + // no filter +CASE(test_argmin_keepdims_example_select_last_index) + // no filter +CASE(test_argmin_keepdims_random) + // no filter +CASE(test_argmin_keepdims_random_select_last_index) + // no filter +CASE(test_argmin_negative_axis_keepdims_example) + // no filter +CASE(test_argmin_negative_axis_keepdims_example_select_last_index) + // no filter +CASE(test_argmin_negative_axis_keepdims_random) + // no filter +CASE(test_argmin_negative_axis_keepdims_random_select_last_index) + // no filter +CASE(test_argmin_no_keepdims_example) + // no filter +CASE(test_argmin_no_keepdims_example_select_last_index) + // no filter +CASE(test_argmin_no_keepdims_random) + // no filter +CASE(test_argmin_no_keepdims_random_select_last_index) + // no filter +CASE(test_asin) + // no filter +CASE(test_asin_example) + // no filter +CASE(test_asinh) + // no filter +CASE(test_asinh_example) + // no filter +CASE(test_atan) + // no filter +CASE(test_atan_example) + // no filter +CASE(test_atanh) + // no filter +CASE(test_atanh_example) + // no filter +CASE(test_averagepool_1d_default) + // no filter +CASE(test_averagepool_2d_ceil) + // no filter +CASE(test_averagepool_2d_default) + // no filter +CASE(test_averagepool_2d_pads) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_averagepool_2d_pads_count_include_pad) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_CPU; + // MYRIAD is ok + SKIP_OPENCL; + SKIP_OPENCL_FP16; +#endif +CASE(test_averagepool_2d_precomputed_pads) + // no filter +CASE(test_averagepool_2d_precomputed_pads_count_include_pad) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_averagepool_2d_precomputed_same_upper) + // no filter +CASE(test_averagepool_2d_precomputed_strides) + // no filter +CASE(test_averagepool_2d_same_lower) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_averagepool_2d_same_upper) + // no filter +CASE(test_averagepool_2d_strides) + // no filter +CASE(test_averagepool_3d_default) + // no filter +CASE(test_basic_conv_with_padding) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_basic_conv_without_padding) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_basic_convinteger) + // no filter +CASE(test_batchnorm_epsilon) + // no filter +CASE(test_batchnorm_epsilon_training_mode) + // no filter +CASE(test_batchnorm_example) + // no filter +CASE(test_batchnorm_example_training_mode) + // no filter +CASE(test_bernoulli) + // no filter +CASE(test_bernoulli_double) + // no filter +CASE(test_bernoulli_double_expanded) + // no filter +CASE(test_bernoulli_expanded) + // no filter +CASE(test_bernoulli_seed) + // no filter +CASE(test_bernoulli_seed_expanded) + // no filter +CASE(test_bitshift_left_uint16) + // no filter +CASE(test_bitshift_left_uint32) + // no filter +CASE(test_bitshift_left_uint64) + // no filter +CASE(test_bitshift_left_uint8) + // no filter +CASE(test_bitshift_right_uint16) + // no filter +CASE(test_bitshift_right_uint32) + // no filter +CASE(test_bitshift_right_uint64) + // no filter +CASE(test_bitshift_right_uint8) + // no filter +CASE(test_cast_BFLOAT16_to_FLOAT) + // no filter +CASE(test_cast_DOUBLE_to_FLOAT) + // no filter +CASE(test_cast_DOUBLE_to_FLOAT16) + // no filter +CASE(test_cast_FLOAT16_to_DOUBLE) + // no filter +CASE(test_cast_FLOAT16_to_FLOAT) + // no filter +CASE(test_cast_FLOAT_to_BFLOAT16) + // no filter +CASE(test_cast_FLOAT_to_DOUBLE) + // no filter +CASE(test_cast_FLOAT_to_FLOAT16) + // no filter +CASE(test_cast_FLOAT_to_STRING) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_cast_STRING_to_FLOAT) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_castlike_BFLOAT16_to_FLOAT) + // no filter +CASE(test_castlike_BFLOAT16_to_FLOAT_expanded) + // no filter +CASE(test_castlike_DOUBLE_to_FLOAT) + // no filter +CASE(test_castlike_DOUBLE_to_FLOAT16) + // no filter +CASE(test_castlike_DOUBLE_to_FLOAT16_expanded) + // no filter +CASE(test_castlike_DOUBLE_to_FLOAT_expanded) + // no filter +CASE(test_castlike_FLOAT16_to_DOUBLE) + // no filter +CASE(test_castlike_FLOAT16_to_DOUBLE_expanded) + // no filter +CASE(test_castlike_FLOAT16_to_FLOAT) + // no filter +CASE(test_castlike_FLOAT16_to_FLOAT_expanded) + // no filter +CASE(test_castlike_FLOAT_to_BFLOAT16) + // no filter +CASE(test_castlike_FLOAT_to_BFLOAT16_expanded) + // no filter +CASE(test_castlike_FLOAT_to_DOUBLE) + // no filter +CASE(test_castlike_FLOAT_to_DOUBLE_expanded) + // no filter +CASE(test_castlike_FLOAT_to_FLOAT16) + // no filter +CASE(test_castlike_FLOAT_to_FLOAT16_expanded) + // no filter +CASE(test_castlike_FLOAT_to_STRING) + // no filter +CASE(test_castlike_FLOAT_to_STRING_expanded) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_castlike_STRING_to_FLOAT) + // no filter +CASE(test_castlike_STRING_to_FLOAT_expanded) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_ceil) + // no filter +CASE(test_ceil_example) + // no filter +CASE(test_celu) + // no filter +CASE(test_celu_expanded) + // no filter +CASE(test_clip) + // no filter +CASE(test_clip_default_inbounds) + // no filter +CASE(test_clip_default_int8_inbounds) + // no filter +CASE(test_clip_default_int8_max) + // no filter +CASE(test_clip_default_int8_min) + // no filter +CASE(test_clip_default_max) + // no filter +CASE(test_clip_default_min) + // no filter +CASE(test_clip_example) + // no filter +CASE(test_clip_inbounds) + // no filter +CASE(test_clip_outbounds) + // no filter +CASE(test_clip_splitbounds) + // no filter +CASE(test_compress_0) + // no filter +CASE(test_compress_1) + // no filter +CASE(test_compress_default_axis) + // no filter +CASE(test_compress_negative_axis) + // no filter +CASE(test_concat_1d_axis_0) + // no filter +CASE(test_concat_1d_axis_negative_1) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_concat_2d_axis_0) + // no filter +CASE(test_concat_2d_axis_1) + // no filter +CASE(test_concat_2d_axis_negative_1) + // no filter +CASE(test_concat_2d_axis_negative_2) + // no filter +CASE(test_concat_3d_axis_0) + // no filter +CASE(test_concat_3d_axis_1) + // no filter +CASE(test_concat_3d_axis_2) + // no filter +CASE(test_concat_3d_axis_negative_1) + // no filter +CASE(test_concat_3d_axis_negative_2) + // no filter +CASE(test_concat_3d_axis_negative_3) + // no filter +CASE(test_constant) + // no filter +CASE(test_constant_pad) + // no filter +CASE(test_constantofshape_float_ones) + // no filter +CASE(test_constantofshape_int_shape_zero) + // no filter +CASE(test_constantofshape_int_zeros) + // no filter +CASE(test_conv_with_autopad_same) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_conv_with_strides_and_asymmetric_padding) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_conv_with_strides_no_padding) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_conv_with_strides_padding) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_convinteger_with_padding) + // no filter +CASE(test_convinteger_without_padding) + // no filter +CASE(test_convtranspose) + // no filter +CASE(test_convtranspose_1d) + // no filter +CASE(test_convtranspose_3d) + // no filter +CASE(test_convtranspose_autopad_same) + // no filter +CASE(test_convtranspose_dilations) + // no filter +CASE(test_convtranspose_kernel_shape) + // no filter +CASE(test_convtranspose_output_shape) + // no filter +CASE(test_convtranspose_pad) + // no filter +CASE(test_convtranspose_pads) + // no filter +CASE(test_convtranspose_with_kernel) + // no filter +CASE(test_cos) + // no filter +CASE(test_cos_example) + // no filter +CASE(test_cosh) + // no filter +CASE(test_cosh_example) + // no filter +CASE(test_cumsum_1d) + // no filter +CASE(test_cumsum_1d_exclusive) + // no filter +CASE(test_cumsum_1d_reverse) + // no filter +CASE(test_cumsum_1d_reverse_exclusive) + // no filter +CASE(test_cumsum_2d_axis_0) + // no filter +CASE(test_cumsum_2d_axis_1) + // no filter +CASE(test_cumsum_2d_negative_axis) + // no filter +CASE(test_depthtospace_crd_mode) + // no filter +CASE(test_depthtospace_crd_mode_example) + // no filter +CASE(test_depthtospace_dcr_mode) + // no filter +CASE(test_depthtospace_example) + // no filter +CASE(test_dequantizelinear) + // no filter +CASE(test_dequantizelinear_axis) + // no filter +CASE(test_det_2d) + // no filter +CASE(test_det_nd) + // no filter +CASE(test_div) + // no filter +CASE(test_div_bcast) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_div_example) + // no filter +CASE(test_div_uint8) + // no filter +CASE(test_dropout_default) + // no filter +CASE(test_dropout_default_mask) + // no filter +CASE(test_dropout_default_mask_ratio) + // no filter +CASE(test_dropout_default_old) + // no filter +CASE(test_dropout_default_ratio) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_dropout_random_old) + // no filter +CASE(test_dynamicquantizelinear) + // no filter +CASE(test_dynamicquantizelinear_expanded) + // no filter +CASE(test_dynamicquantizelinear_max_adjusted) + // no filter +CASE(test_dynamicquantizelinear_max_adjusted_expanded) + // no filter +CASE(test_dynamicquantizelinear_min_adjusted) + // no filter +CASE(test_dynamicquantizelinear_min_adjusted_expanded) + // no filter +CASE(test_edge_pad) + // no filter +CASE(test_einsum_batch_diagonal) + // no filter +CASE(test_einsum_batch_matmul) + // no filter +CASE(test_einsum_inner_prod) + // no filter +CASE(test_einsum_sum) + // no filter +CASE(test_einsum_transpose) + // no filter +CASE(test_elu) + // no filter +CASE(test_elu_default) + // no filter +CASE(test_elu_example) + // no filter +CASE(test_equal) + // no filter +CASE(test_equal_bcast) + // no filter +CASE(test_erf) + // no filter +CASE(test_exp) + // no filter +CASE(test_exp_example) + // no filter +CASE(test_expand_dim_changed) + // no filter +CASE(test_expand_dim_unchanged) + // no filter +CASE(test_eyelike_populate_off_main_diagonal) + // no filter +CASE(test_eyelike_with_dtype) + // no filter +CASE(test_eyelike_without_dtype) + // no filter +CASE(test_flatten_axis0) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_flatten_axis1) + // no filter +CASE(test_flatten_axis2) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_flatten_axis3) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_flatten_default_axis) + // no filter +CASE(test_flatten_negative_axis1) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_flatten_negative_axis2) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_flatten_negative_axis3) + // no filter +CASE(test_flatten_negative_axis4) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_floor) + // no filter +CASE(test_floor_example) + // no filter +CASE(test_gather_0) + // no filter +CASE(test_gather_1) + // no filter +CASE(test_gather_2d_indices) + // no filter +CASE(test_gather_elements_0) + // no filter +CASE(test_gather_elements_1) + // no filter +CASE(test_gather_elements_negative_indices) + // no filter +CASE(test_gather_negative_indices) + // no filter +CASE(test_gathernd_example_float32) + // no filter +CASE(test_gathernd_example_int32) + // no filter +CASE(test_gathernd_example_int32_batch_dim1) + // no filter +CASE(test_gemm_all_attributes) + // no filter +CASE(test_gemm_alpha) + // no filter +CASE(test_gemm_beta) + // no filter +CASE(test_gemm_default_matrix_bias) + // no filter +CASE(test_gemm_default_no_bias) + // no filter +CASE(test_gemm_default_scalar_bias) + // no filter +CASE(test_gemm_default_single_elem_vector_bias) + // no filter +CASE(test_gemm_default_vector_bias) + // no filter +CASE(test_gemm_default_zero_bias) + // no filter +CASE(test_gemm_transposeA) + // no filter +CASE(test_gemm_transposeB) + // no filter +CASE(test_globalaveragepool) + // no filter +CASE(test_globalaveragepool_precomputed) + // no filter +CASE(test_globalmaxpool) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_globalmaxpool_precomputed) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_greater) + // no filter +CASE(test_greater_bcast) + // no filter +CASE(test_greater_equal) + // no filter +CASE(test_greater_equal_bcast) + // no filter +CASE(test_greater_equal_bcast_expanded) + // no filter +CASE(test_greater_equal_expanded) + // no filter +CASE(test_gridsample) + // no filter +CASE(test_gridsample_aligncorners_true) + // no filter +CASE(test_gridsample_bicubic) + // no filter +CASE(test_gridsample_bilinear) + // no filter +CASE(test_gridsample_border_padding) + // no filter +CASE(test_gridsample_nearest) + // no filter +CASE(test_gridsample_reflection_padding) + // no filter +CASE(test_gridsample_zeros_padding) + // no filter +CASE(test_gru_batchwise) + // no filter +CASE(test_gru_defaults) + // no filter +CASE(test_gru_seq_length) + // no filter +CASE(test_gru_with_initial_bias) + // no filter +CASE(test_hardmax_axis_0) + // no filter +CASE(test_hardmax_axis_1) + // no filter +CASE(test_hardmax_axis_2) + // no filter +CASE(test_hardmax_default_axis) + // no filter +CASE(test_hardmax_example) + // no filter +CASE(test_hardmax_negative_axis) + // no filter +CASE(test_hardmax_one_hot) + // no filter +CASE(test_hardsigmoid) + // no filter +CASE(test_hardsigmoid_default) + // no filter +CASE(test_hardsigmoid_example) + // no filter +CASE(test_hardswish) + // no filter +CASE(test_hardswish_expanded) + // no filter +CASE(test_identity) + // no filter +CASE(test_identity_opt) + // no filter +CASE(test_identity_sequence) + // no filter +CASE(test_if) + // no filter +CASE(test_if_opt) + // no filter +CASE(test_if_seq) + // no filter +CASE(test_instancenorm_epsilon) + // no filter +CASE(test_instancenorm_example) + // no filter +CASE(test_isinf) + // no filter +CASE(test_isinf_negative) + // no filter +CASE(test_isinf_positive) + // no filter +CASE(test_isnan) + // no filter +CASE(test_leakyrelu) + // no filter +CASE(test_leakyrelu_default) + // no filter +CASE(test_leakyrelu_example) + // no filter +CASE(test_less) + // no filter +CASE(test_less_bcast) + // no filter +CASE(test_less_equal) + // no filter +CASE(test_less_equal_bcast) + // no filter +CASE(test_less_equal_bcast_expanded) + // no filter +CASE(test_less_equal_expanded) + // no filter +CASE(test_log) + // no filter +CASE(test_log_example) + // no filter +CASE(test_logsoftmax_axis_0) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_OPENCL; + SKIP_OPENCL_FP16; +#endif +CASE(test_logsoftmax_axis_0_expanded) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_OPENCL; + SKIP_OPENCL_FP16; +#endif +CASE(test_logsoftmax_axis_1) + // no filter +CASE(test_logsoftmax_axis_1_expanded) + // no filter +CASE(test_logsoftmax_axis_2) + // no filter +CASE(test_logsoftmax_axis_2_expanded) + // no filter +CASE(test_logsoftmax_default_axis) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_logsoftmax_default_axis_expanded) + // no filter +CASE(test_logsoftmax_example_1) + // no filter +CASE(test_logsoftmax_example_1_expanded) + // no filter +CASE(test_logsoftmax_large_number) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_OPENCL_FP16; + SKIP_MYRIAD; +#endif +CASE(test_logsoftmax_large_number_expanded) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_OPENCL_FP16; + SKIP_MYRIAD; +#endif +CASE(test_logsoftmax_negative_axis) + // no filter +CASE(test_logsoftmax_negative_axis_expanded) + // no filter +CASE(test_loop11) + // no filter +CASE(test_loop13_seq) + // no filter +CASE(test_loop16_seq_none) + // no filter +CASE(test_lrn) + // no filter +CASE(test_lrn_default) + // no filter +CASE(test_lstm_batchwise) + // no filter +CASE(test_lstm_defaults) + // no filter +CASE(test_lstm_with_initial_bias) + // no filter +CASE(test_lstm_with_peepholes) + // no filter +CASE(test_matmul_2d) + // no filter +CASE(test_matmul_3d) + // no filter +CASE(test_matmul_4d) + // no filter +CASE(test_matmulinteger) + // no filter +CASE(test_max_example) + // no filter +CASE(test_max_float16) + // no filter +CASE(test_max_float32) + // no filter +CASE(test_max_float64) + // no filter +CASE(test_max_int16) + // no filter +CASE(test_max_int32) + // no filter +CASE(test_max_int64) + // no filter +CASE(test_max_int8) + // no filter +CASE(test_max_one_input) + // no filter +CASE(test_max_two_inputs) + // no filter +CASE(test_max_uint16) + // no filter +CASE(test_max_uint32) + // no filter +CASE(test_max_uint64) + // no filter +CASE(test_max_uint8) + // no filter +CASE(test_maxpool_1d_default) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_maxpool_2d_ceil) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_maxpool_2d_default) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_maxpool_2d_dilations) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_maxpool_2d_pads) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_maxpool_2d_precomputed_pads) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_maxpool_2d_precomputed_same_upper) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_maxpool_2d_precomputed_strides) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_maxpool_2d_same_lower) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_maxpool_2d_same_upper) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_maxpool_2d_strides) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_maxpool_2d_uint8) + // no filter +CASE(test_maxpool_3d_default) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_NON_CPU; +#endif +CASE(test_maxpool_with_argmax_2d_precomputed_pads) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_maxpool_with_argmax_2d_precomputed_strides) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_maxunpool_export_with_output_shape) + // no filter +CASE(test_maxunpool_export_without_output_shape) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_mean_example) + // no filter +CASE(test_mean_one_input) + // no filter +CASE(test_mean_two_inputs) + // no filter +CASE(test_min_example) + // no filter +CASE(test_min_float16) + // no filter +CASE(test_min_float32) + // no filter +CASE(test_min_float64) + // no filter +CASE(test_min_int16) + // no filter +CASE(test_min_int32) + // no filter +CASE(test_min_int64) + // no filter +CASE(test_min_int8) + // no filter +CASE(test_min_one_input) + // no filter +CASE(test_min_two_inputs) + // no filter +CASE(test_min_uint16) + // no filter +CASE(test_min_uint32) + // no filter +CASE(test_min_uint64) + // no filter +CASE(test_min_uint8) + // no filter +CASE(test_mod_broadcast) + // no filter +CASE(test_mod_int64_fmod) + // no filter +CASE(test_mod_mixed_sign_float16) + // no filter +CASE(test_mod_mixed_sign_float32) + // no filter +CASE(test_mod_mixed_sign_float64) + // no filter +CASE(test_mod_mixed_sign_int16) + // no filter +CASE(test_mod_mixed_sign_int32) + // no filter +CASE(test_mod_mixed_sign_int64) + // no filter +CASE(test_mod_mixed_sign_int8) + // no filter +CASE(test_mod_uint16) + // no filter +CASE(test_mod_uint32) + // no filter +CASE(test_mod_uint64) + // no filter +CASE(test_mod_uint8) + // no filter +CASE(test_momentum) + // no filter +CASE(test_momentum_multiple) + // no filter +CASE(test_mul) + // no filter +CASE(test_mul_bcast) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_mul_example) + // no filter +CASE(test_mul_uint8) + // no filter +CASE(test_mvn) + // no filter +CASE(test_mvn_expanded) + // no filter +CASE(test_neg) + // no filter +CASE(test_neg_example) + // no filter +CASE(test_nesterov_momentum) + // no filter +CASE(test_nllloss_NC) + // no filter +CASE(test_nllloss_NC_expanded) + // no filter +CASE(test_nllloss_NCd1) + // no filter +CASE(test_nllloss_NCd1_expanded) + // no filter +CASE(test_nllloss_NCd1_ii) + // no filter +CASE(test_nllloss_NCd1_ii_expanded) + // no filter +CASE(test_nllloss_NCd1_mean_weight_negative_ii) + // no filter +CASE(test_nllloss_NCd1_mean_weight_negative_ii_expanded) + // no filter +CASE(test_nllloss_NCd1_weight) + // no filter +CASE(test_nllloss_NCd1_weight_expanded) + // no filter +CASE(test_nllloss_NCd1_weight_ii) + // no filter +CASE(test_nllloss_NCd1_weight_ii_expanded) + // no filter +CASE(test_nllloss_NCd1d2) + // no filter +CASE(test_nllloss_NCd1d2_expanded) + // no filter +CASE(test_nllloss_NCd1d2_no_weight_reduction_mean_ii) + // no filter +CASE(test_nllloss_NCd1d2_no_weight_reduction_mean_ii_expanded) + // no filter +CASE(test_nllloss_NCd1d2_reduction_mean) + // no filter +CASE(test_nllloss_NCd1d2_reduction_mean_expanded) + // no filter +CASE(test_nllloss_NCd1d2_reduction_sum) + // no filter +CASE(test_nllloss_NCd1d2_reduction_sum_expanded) + // no filter +CASE(test_nllloss_NCd1d2_with_weight) + // no filter +CASE(test_nllloss_NCd1d2_with_weight_expanded) + // no filter +CASE(test_nllloss_NCd1d2_with_weight_reduction_mean) + // no filter +CASE(test_nllloss_NCd1d2_with_weight_reduction_mean_expanded) + // no filter +CASE(test_nllloss_NCd1d2_with_weight_reduction_sum) + // no filter +CASE(test_nllloss_NCd1d2_with_weight_reduction_sum_expanded) + // no filter +CASE(test_nllloss_NCd1d2_with_weight_reduction_sum_ii) + // no filter +CASE(test_nllloss_NCd1d2_with_weight_reduction_sum_ii_expanded) + // no filter +CASE(test_nllloss_NCd1d2d3_none_no_weight_negative_ii) + // no filter +CASE(test_nllloss_NCd1d2d3_none_no_weight_negative_ii_expanded) + // no filter +CASE(test_nllloss_NCd1d2d3_sum_weight_high_ii) + // no filter +CASE(test_nllloss_NCd1d2d3_sum_weight_high_ii_expanded) + // no filter +CASE(test_nllloss_NCd1d2d3d4d5_mean_weight) + // no filter +CASE(test_nllloss_NCd1d2d3d4d5_mean_weight_expanded) + // no filter +CASE(test_nllloss_NCd1d2d3d4d5_none_no_weight) + // no filter +CASE(test_nllloss_NCd1d2d3d4d5_none_no_weight_expanded) + // no filter +CASE(test_nonmaxsuppression_center_point_box_format) + // no filter +CASE(test_nonmaxsuppression_flipped_coordinates) + // no filter +CASE(test_nonmaxsuppression_identical_boxes) + // no filter +CASE(test_nonmaxsuppression_limit_output_size) + // no filter +CASE(test_nonmaxsuppression_single_box) + // no filter +CASE(test_nonmaxsuppression_suppress_by_IOU) + // no filter +CASE(test_nonmaxsuppression_suppress_by_IOU_and_scores) + // no filter +CASE(test_nonmaxsuppression_two_batches) + // no filter +CASE(test_nonmaxsuppression_two_classes) + // no filter +CASE(test_nonzero_example) + // no filter +CASE(test_not_2d) + // no filter +CASE(test_not_3d) + // no filter +CASE(test_not_4d) + // no filter +CASE(test_onehot_negative_indices) + // no filter +CASE(test_onehot_with_axis) + // no filter +CASE(test_onehot_with_negative_axis) + // no filter +CASE(test_onehot_without_axis) + // no filter +CASE(test_optional_get_element) + // no filter +CASE(test_optional_get_element_sequence) + // no filter +CASE(test_optional_has_element) + // no filter +CASE(test_optional_has_element_empty) + // no filter +CASE(test_or2d) + // no filter +CASE(test_or3d) + // no filter +CASE(test_or4d) + // no filter +CASE(test_or_bcast3v1d) + // no filter +CASE(test_or_bcast3v2d) + // no filter +CASE(test_or_bcast4v2d) + // no filter +CASE(test_or_bcast4v3d) + // no filter +CASE(test_or_bcast4v4d) + // no filter +CASE(test_pow) + // no filter +CASE(test_pow_bcast_array) + // no filter +CASE(test_pow_bcast_scalar) + // no filter +CASE(test_pow_example) + // no filter +CASE(test_pow_types_float) + // no filter +CASE(test_pow_types_float32_int32) + // no filter +CASE(test_pow_types_float32_int64) + // no filter +CASE(test_pow_types_float32_uint32) + // no filter +CASE(test_pow_types_float32_uint64) + // no filter +CASE(test_pow_types_int) + // no filter +CASE(test_pow_types_int32_float32) + // no filter +CASE(test_pow_types_int32_int32) + // no filter +CASE(test_pow_types_int64_float32) + // no filter +CASE(test_pow_types_int64_int64) + // no filter +CASE(test_prelu_broadcast) + // no filter +CASE(test_prelu_example) + // no filter +CASE(test_qlinearconv) + // no filter +CASE(test_qlinearmatmul_2D) + // no filter +CASE(test_qlinearmatmul_3D) + // no filter +CASE(test_quantizelinear) + // no filter +CASE(test_quantizelinear_axis) + // no filter +CASE(test_range_float_type_positive_delta) + // no filter +CASE(test_range_float_type_positive_delta_expanded) + // no filter +CASE(test_range_int32_type_negative_delta) + // no filter +CASE(test_range_int32_type_negative_delta_expanded) + // no filter +CASE(test_reciprocal) + // no filter +CASE(test_reciprocal_example) + // no filter +CASE(test_reduce_l1_default_axes_keepdims_example) + // no filter +CASE(test_reduce_l1_default_axes_keepdims_random) + // no filter +CASE(test_reduce_l1_do_not_keepdims_example) + // no filter +CASE(test_reduce_l1_do_not_keepdims_random) + // no filter +CASE(test_reduce_l1_keep_dims_example) + // no filter +CASE(test_reduce_l1_keep_dims_random) + // no filter +CASE(test_reduce_l1_negative_axes_keep_dims_example) + // no filter +CASE(test_reduce_l1_negative_axes_keep_dims_random) + // no filter +CASE(test_reduce_l2_default_axes_keepdims_example) + // no filter +CASE(test_reduce_l2_default_axes_keepdims_random) + // no filter +CASE(test_reduce_l2_do_not_keepdims_example) + // no filter +CASE(test_reduce_l2_do_not_keepdims_random) + // no filter +CASE(test_reduce_l2_keep_dims_example) + // no filter +CASE(test_reduce_l2_keep_dims_random) + // no filter +CASE(test_reduce_l2_negative_axes_keep_dims_example) + // no filter +CASE(test_reduce_l2_negative_axes_keep_dims_random) + // no filter +CASE(test_reduce_log_sum) + // no filter +CASE(test_reduce_log_sum_asc_axes) + // no filter +CASE(test_reduce_log_sum_default) + // no filter +CASE(test_reduce_log_sum_desc_axes) + // no filter +CASE(test_reduce_log_sum_exp_default_axes_keepdims_example) + // no filter +CASE(test_reduce_log_sum_exp_default_axes_keepdims_random) + // no filter +CASE(test_reduce_log_sum_exp_do_not_keepdims_example) + // no filter +CASE(test_reduce_log_sum_exp_do_not_keepdims_random) + // no filter +CASE(test_reduce_log_sum_exp_keepdims_example) + // no filter +CASE(test_reduce_log_sum_exp_keepdims_random) + // no filter +CASE(test_reduce_log_sum_exp_negative_axes_keepdims_example) + // no filter +CASE(test_reduce_log_sum_exp_negative_axes_keepdims_random) + // no filter +CASE(test_reduce_log_sum_negative_axes) + // no filter +CASE(test_reduce_max_default_axes_keepdim_example) + // no filter +CASE(test_reduce_max_default_axes_keepdims_random) + // no filter +CASE(test_reduce_max_do_not_keepdims_example) + // no filter +CASE(test_reduce_max_do_not_keepdims_random) + // no filter +CASE(test_reduce_max_keepdims_example) + // no filter +CASE(test_reduce_max_keepdims_random) + // no filter +CASE(test_reduce_max_negative_axes_keepdims_example) + // no filter +CASE(test_reduce_max_negative_axes_keepdims_random) + // no filter +CASE(test_reduce_mean_default_axes_keepdims_example) + // no filter +CASE(test_reduce_mean_default_axes_keepdims_random) + // no filter +CASE(test_reduce_mean_do_not_keepdims_example) + // no filter +CASE(test_reduce_mean_do_not_keepdims_random) + // no filter +CASE(test_reduce_mean_keepdims_example) + // no filter +CASE(test_reduce_mean_keepdims_random) + // no filter +CASE(test_reduce_mean_negative_axes_keepdims_example) + // no filter +CASE(test_reduce_mean_negative_axes_keepdims_random) + // no filter +CASE(test_reduce_min_default_axes_keepdims_example) + // no filter +CASE(test_reduce_min_default_axes_keepdims_random) + // no filter +CASE(test_reduce_min_do_not_keepdims_example) + // no filter +CASE(test_reduce_min_do_not_keepdims_random) + // no filter +CASE(test_reduce_min_keepdims_example) + // no filter +CASE(test_reduce_min_keepdims_random) + // no filter +CASE(test_reduce_min_negative_axes_keepdims_example) + // no filter +CASE(test_reduce_min_negative_axes_keepdims_random) + // no filter +CASE(test_reduce_prod_default_axes_keepdims_example) + // no filter +CASE(test_reduce_prod_default_axes_keepdims_random) + // no filter +CASE(test_reduce_prod_do_not_keepdims_example) + // no filter +CASE(test_reduce_prod_do_not_keepdims_random) + // no filter +CASE(test_reduce_prod_keepdims_example) + // no filter +CASE(test_reduce_prod_keepdims_random) + // no filter +CASE(test_reduce_prod_negative_axes_keepdims_example) + // no filter +CASE(test_reduce_prod_negative_axes_keepdims_random) + // no filter +CASE(test_reduce_sum_default_axes_keepdims_example) + // no filter +CASE(test_reduce_sum_default_axes_keepdims_random) + // no filter +CASE(test_reduce_sum_do_not_keepdims_example) + // no filter +CASE(test_reduce_sum_do_not_keepdims_random) + // no filter +CASE(test_reduce_sum_empty_axes_input_noop_example) + // no filter +CASE(test_reduce_sum_empty_axes_input_noop_random) + // no filter +CASE(test_reduce_sum_keepdims_example) + // no filter +CASE(test_reduce_sum_keepdims_random) + // no filter +CASE(test_reduce_sum_negative_axes_keepdims_example) + // no filter +CASE(test_reduce_sum_negative_axes_keepdims_random) + // no filter +CASE(test_reduce_sum_square_default_axes_keepdims_example) + // no filter +CASE(test_reduce_sum_square_default_axes_keepdims_random) + // no filter +CASE(test_reduce_sum_square_do_not_keepdims_example) + // no filter +CASE(test_reduce_sum_square_do_not_keepdims_random) + // no filter +CASE(test_reduce_sum_square_keepdims_example) + // no filter +CASE(test_reduce_sum_square_keepdims_random) + // no filter +CASE(test_reduce_sum_square_negative_axes_keepdims_example) + // no filter +CASE(test_reduce_sum_square_negative_axes_keepdims_random) + // no filter +CASE(test_reflect_pad) + // no filter +CASE(test_relu) + // no filter +CASE(test_reshape_allowzero_reordered) + // no filter +CASE(test_reshape_extended_dims) + // no filter +CASE(test_reshape_negative_dim) + // no filter +CASE(test_reshape_negative_extended_dims) + // no filter +CASE(test_reshape_one_dim) + // no filter +CASE(test_reshape_reduced_dims) + // no filter +CASE(test_reshape_reordered_all_dims) + // no filter +CASE(test_reshape_reordered_last_dims) + // no filter +CASE(test_reshape_zero_and_negative_dim) + // no filter +CASE(test_reshape_zero_dim) + // no filter +CASE(test_resize_downsample_scales_cubic) + // no filter +CASE(test_resize_downsample_scales_cubic_A_n0p5_exclude_outside) + // no filter +CASE(test_resize_downsample_scales_cubic_align_corners) + // no filter +CASE(test_resize_downsample_scales_linear) + // no filter +CASE(test_resize_downsample_scales_linear_align_corners) + // no filter +CASE(test_resize_downsample_scales_nearest) + // no filter +CASE(test_resize_downsample_sizes_cubic) + // no filter +CASE(test_resize_downsample_sizes_linear_pytorch_half_pixel) + // no filter +CASE(test_resize_downsample_sizes_nearest) + // no filter +CASE(test_resize_downsample_sizes_nearest_tf_half_pixel_for_nn) + // no filter +CASE(test_resize_tf_crop_and_resize) + // no filter +CASE(test_resize_upsample_scales_cubic) + // no filter +CASE(test_resize_upsample_scales_cubic_A_n0p5_exclude_outside) + // no filter +CASE(test_resize_upsample_scales_cubic_align_corners) + // no filter +CASE(test_resize_upsample_scales_cubic_asymmetric) + // no filter +CASE(test_resize_upsample_scales_linear) + // no filter +CASE(test_resize_upsample_scales_linear_align_corners) + // no filter +CASE(test_resize_upsample_scales_nearest) + // no filter +CASE(test_resize_upsample_sizes_cubic) + // no filter +CASE(test_resize_upsample_sizes_nearest) + // no filter +CASE(test_resize_upsample_sizes_nearest_ceil_half_pixel) + // no filter +CASE(test_resize_upsample_sizes_nearest_floor_align_corners) + // no filter +CASE(test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric) + // no filter +CASE(test_reversesequence_batch) + // no filter +CASE(test_reversesequence_time) + // no filter +CASE(test_rnn_seq_length) + // no filter +CASE(test_roialign_aligned_false) + // no filter +CASE(test_roialign_aligned_true) + // no filter +CASE(test_round) + // no filter +CASE(test_scan9_sum) + // no filter +CASE(test_scan_sum) + // no filter +CASE(test_scatter_elements_with_axis) + // no filter +CASE(test_scatter_elements_with_duplicate_indices) + // no filter +CASE(test_scatter_elements_with_negative_indices) + // no filter +CASE(test_scatter_elements_without_axis) + // no filter +CASE(test_scatter_with_axis) + // no filter +CASE(test_scatter_without_axis) + // no filter +CASE(test_scatternd) + // no filter +CASE(test_scatternd_add) + // no filter +CASE(test_scatternd_multiply) + // no filter +CASE(test_sce_NCd1_mean_weight_negative_ii) + // no filter +CASE(test_sce_NCd1_mean_weight_negative_ii_expanded) + // no filter +CASE(test_sce_NCd1_mean_weight_negative_ii_log_prob) + // no filter +CASE(test_sce_NCd1_mean_weight_negative_ii_log_prob_expanded) + // no filter +CASE(test_sce_NCd1d2d3_none_no_weight_negative_ii) + // no filter +CASE(test_sce_NCd1d2d3_none_no_weight_negative_ii_expanded) + // no filter +CASE(test_sce_NCd1d2d3_none_no_weight_negative_ii_log_prob) + // no filter +CASE(test_sce_NCd1d2d3_none_no_weight_negative_ii_log_prob_expanded) + // no filter +CASE(test_sce_NCd1d2d3_sum_weight_high_ii) + // no filter +CASE(test_sce_NCd1d2d3_sum_weight_high_ii_expanded) + // no filter +CASE(test_sce_NCd1d2d3_sum_weight_high_ii_log_prob) + // no filter +CASE(test_sce_NCd1d2d3_sum_weight_high_ii_log_prob_expanded) + // no filter +CASE(test_sce_NCd1d2d3d4d5_mean_weight) + // no filter +CASE(test_sce_NCd1d2d3d4d5_mean_weight_expanded) + // no filter +CASE(test_sce_NCd1d2d3d4d5_mean_weight_log_prob) + // no filter +CASE(test_sce_NCd1d2d3d4d5_mean_weight_log_prob_expanded) + // no filter +CASE(test_sce_NCd1d2d3d4d5_none_no_weight) + // no filter +CASE(test_sce_NCd1d2d3d4d5_none_no_weight_expanded) + // no filter +CASE(test_sce_NCd1d2d3d4d5_none_no_weight_log_prob) + // no filter +CASE(test_sce_NCd1d2d3d4d5_none_no_weight_log_prob_expanded) + // no filter +CASE(test_sce_mean) + // no filter +CASE(test_sce_mean_3d) + // no filter +CASE(test_sce_mean_3d_expanded) + // no filter +CASE(test_sce_mean_3d_log_prob) + // no filter +CASE(test_sce_mean_3d_log_prob_expanded) + // no filter +CASE(test_sce_mean_expanded) + // no filter +CASE(test_sce_mean_log_prob) + // no filter +CASE(test_sce_mean_log_prob_expanded) + // no filter +CASE(test_sce_mean_no_weight_ii) + // no filter +CASE(test_sce_mean_no_weight_ii_3d) + // no filter +CASE(test_sce_mean_no_weight_ii_3d_expanded) + // no filter +CASE(test_sce_mean_no_weight_ii_3d_log_prob) + // no filter +CASE(test_sce_mean_no_weight_ii_3d_log_prob_expanded) + // no filter +CASE(test_sce_mean_no_weight_ii_4d) + // no filter +CASE(test_sce_mean_no_weight_ii_4d_expanded) + // no filter +CASE(test_sce_mean_no_weight_ii_4d_log_prob) + // no filter +CASE(test_sce_mean_no_weight_ii_4d_log_prob_expanded) + // no filter +CASE(test_sce_mean_no_weight_ii_expanded) + // no filter +CASE(test_sce_mean_no_weight_ii_log_prob) + // no filter +CASE(test_sce_mean_no_weight_ii_log_prob_expanded) + // no filter +CASE(test_sce_mean_weight) + // no filter +CASE(test_sce_mean_weight_expanded) + // no filter +CASE(test_sce_mean_weight_ii) + // no filter +CASE(test_sce_mean_weight_ii_3d) + // no filter +CASE(test_sce_mean_weight_ii_3d_expanded) + // no filter +CASE(test_sce_mean_weight_ii_3d_log_prob) + // no filter +CASE(test_sce_mean_weight_ii_3d_log_prob_expanded) + // no filter +CASE(test_sce_mean_weight_ii_4d) + // no filter +CASE(test_sce_mean_weight_ii_4d_expanded) + // no filter +CASE(test_sce_mean_weight_ii_4d_log_prob) + // no filter +CASE(test_sce_mean_weight_ii_4d_log_prob_expanded) + // no filter +CASE(test_sce_mean_weight_ii_expanded) + // no filter +CASE(test_sce_mean_weight_ii_log_prob) + // no filter +CASE(test_sce_mean_weight_ii_log_prob_expanded) + // no filter +CASE(test_sce_mean_weight_log_prob) + // no filter +CASE(test_sce_mean_weight_log_prob_expanded) + // no filter +CASE(test_sce_none) + // no filter +CASE(test_sce_none_expanded) + // no filter +CASE(test_sce_none_log_prob) + // no filter +CASE(test_sce_none_log_prob_expanded) + // no filter +CASE(test_sce_none_weights) + // no filter +CASE(test_sce_none_weights_expanded) + // no filter +CASE(test_sce_none_weights_log_prob) + // no filter +CASE(test_sce_none_weights_log_prob_expanded) + // no filter +CASE(test_sce_sum) + // no filter +CASE(test_sce_sum_expanded) + // no filter +CASE(test_sce_sum_log_prob) + // no filter +CASE(test_sce_sum_log_prob_expanded) + // no filter +CASE(test_selu) + // no filter +CASE(test_selu_default) + // no filter +CASE(test_selu_example) + // no filter +CASE(test_sequence_insert_at_back) + // no filter +CASE(test_sequence_insert_at_front) + // no filter +CASE(test_shape) + // no filter +CASE(test_shape_clip_end) + // no filter +CASE(test_shape_clip_start) + // no filter +CASE(test_shape_end_1) + // no filter +CASE(test_shape_end_negative_1) + // no filter +CASE(test_shape_example) + // no filter +CASE(test_shape_start_1) + // no filter +CASE(test_shape_start_1_end_2) + // no filter +CASE(test_shape_start_1_end_negative_1) + // no filter +CASE(test_shape_start_negative_1) + // no filter +CASE(test_shrink_hard) + // no filter +CASE(test_shrink_soft) + // no filter +CASE(test_sigmoid) + // no filter +CASE(test_sigmoid_example) + // no filter +CASE(test_sign) + // no filter +CASE(test_simple_rnn_batchwise) + // no filter +CASE(test_simple_rnn_defaults) + // no filter +CASE(test_simple_rnn_with_initial_bias) + // no filter +CASE(test_sin) + // no filter +CASE(test_sin_example) + // no filter +CASE(test_sinh) + // no filter +CASE(test_sinh_example) + // no filter +CASE(test_size) + // no filter +CASE(test_size_example) + // no filter +CASE(test_slice) + // no filter +CASE(test_slice_default_axes) + // no filter +CASE(test_slice_default_steps) + // no filter +CASE(test_slice_end_out_of_bounds) + // no filter +CASE(test_slice_neg) + // no filter +CASE(test_slice_neg_steps) + // no filter +CASE(test_slice_negative_axes) + // no filter +CASE(test_slice_start_out_of_bounds) + // no filter +CASE(test_softmax_axis_0) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_OPENCL; + SKIP_OPENCL_FP16; +#endif +CASE(test_softmax_axis_0_expanded) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_OPENCL; + SKIP_OPENCL_FP16; +#endif +CASE(test_softmax_axis_1) + // no filter +CASE(test_softmax_axis_1_expanded) + // no filter +CASE(test_softmax_axis_2) + // no filter +CASE(test_softmax_axis_2_expanded) + // no filter +CASE(test_softmax_default_axis) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_softmax_default_axis_expanded) + // no filter +CASE(test_softmax_example) + // no filter +CASE(test_softmax_example_expanded) + // no filter +CASE(test_softmax_large_number) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_OPENCL_FP16; + SKIP_MYRIAD; +#endif +CASE(test_softmax_large_number_expanded) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_OPENCL_FP16; + SKIP_MYRIAD; +#endif +CASE(test_softmax_negative_axis) + // no filter +CASE(test_softmax_negative_axis_expanded) + // no filter +CASE(test_softplus) + // no filter +CASE(test_softplus_example) + // no filter +CASE(test_softsign) + // no filter +CASE(test_softsign_example) + // no filter +CASE(test_spacetodepth) + // no filter +CASE(test_spacetodepth_example) + // no filter +CASE(test_split_equal_parts_1d) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_CPU; + // MYRIAD is ok + SKIP_OPENCL; + SKIP_OPENCL_FP16; +#endif +CASE(test_split_equal_parts_2d) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_CPU; + // MYRIAD is ok + SKIP_OPENCL; + SKIP_OPENCL_FP16; +#endif +CASE(test_split_equal_parts_default_axis) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_CPU; + // MYRIAD is ok + SKIP_OPENCL; + SKIP_OPENCL_FP16; +#endif +CASE(test_split_variable_parts_1d) + // no filter +CASE(test_split_variable_parts_2d) + // no filter +CASE(test_split_variable_parts_default_axis) + // no filter +CASE(test_split_zero_size_splits) + // no filter +CASE(test_sqrt) + // no filter +CASE(test_sqrt_example) + // no filter +CASE(test_squeeze) + // no filter +CASE(test_squeeze_negative_axes) + // no filter +CASE(test_strnormalizer_export_monday_casesensintive_lower) + // no filter +CASE(test_strnormalizer_export_monday_casesensintive_nochangecase) + // no filter +CASE(test_strnormalizer_export_monday_casesensintive_upper) + // no filter +CASE(test_strnormalizer_export_monday_empty_output) + // no filter +CASE(test_strnormalizer_export_monday_insensintive_upper_twodim) + // no filter +CASE(test_strnormalizer_nostopwords_nochangecase) + // no filter +CASE(test_sub) + // no filter +CASE(test_sub_bcast) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_sub_example) + // no filter +CASE(test_sub_uint8) + // no filter +CASE(test_sum_example) + // no filter +CASE(test_sum_one_input) + // no filter +CASE(test_sum_two_inputs) + // no filter +CASE(test_tan) + // no filter +CASE(test_tan_example) + // no filter +CASE(test_tanh) + // no filter +CASE(test_tanh_example) + // no filter +CASE(test_tfidfvectorizer_tf_batch_onlybigrams_skip0) + // no filter +CASE(test_tfidfvectorizer_tf_batch_onlybigrams_skip5) + // no filter +CASE(test_tfidfvectorizer_tf_batch_uniandbigrams_skip5) + // no filter +CASE(test_tfidfvectorizer_tf_only_bigrams_skip0) + // no filter +CASE(test_tfidfvectorizer_tf_onlybigrams_levelempty) + // no filter +CASE(test_tfidfvectorizer_tf_onlybigrams_skip5) + // no filter +CASE(test_tfidfvectorizer_tf_uniandbigrams_skip5) + // no filter +CASE(test_thresholdedrelu) + // no filter +CASE(test_thresholdedrelu_default) + // no filter +CASE(test_thresholdedrelu_example) + // no filter +CASE(test_tile) + // no filter +CASE(test_tile_precomputed) + // no filter +CASE(test_top_k) + // no filter +CASE(test_top_k_negative_axis) + // no filter +CASE(test_top_k_smallest) + // no filter +CASE(test_training_dropout) + // no filter +CASE(test_training_dropout_default) + // no filter +CASE(test_training_dropout_default_mask) + // no filter +CASE(test_training_dropout_mask) + // no filter +CASE(test_training_dropout_zero_ratio) + // no filter +CASE(test_training_dropout_zero_ratio_mask) + // no filter +CASE(test_transpose_all_permutations_0) + // no filter +CASE(test_transpose_all_permutations_1) + // no filter +CASE(test_transpose_all_permutations_2) + // no filter +CASE(test_transpose_all_permutations_3) + // no filter +CASE(test_transpose_all_permutations_4) + // no filter +CASE(test_transpose_all_permutations_5) + // no filter +CASE(test_transpose_default) + // no filter +CASE(test_tril) + // no filter +CASE(test_tril_neg) + // no filter +CASE(test_tril_one_row_neg) + // no filter +CASE(test_tril_out_neg) + // no filter +CASE(test_tril_out_pos) + // no filter +CASE(test_tril_pos) + // no filter +CASE(test_tril_square) + // no filter +CASE(test_tril_square_neg) + // no filter +CASE(test_tril_zero) + // no filter +CASE(test_triu) + // no filter +CASE(test_triu_neg) + // no filter +CASE(test_triu_one_row) + // no filter +CASE(test_triu_out_neg_out) + // no filter +CASE(test_triu_out_pos) + // no filter +CASE(test_triu_pos) + // no filter +CASE(test_triu_square) + // no filter +CASE(test_triu_square_neg) + // no filter +CASE(test_triu_zero) + // no filter +CASE(test_unique_not_sorted_without_axis) + // no filter +CASE(test_unique_sorted_with_axis) + // no filter +CASE(test_unique_sorted_with_axis_3d) + // no filter +CASE(test_unique_sorted_with_negative_axis) + // no filter +CASE(test_unique_sorted_without_axis) + // no filter +CASE(test_unsqueeze_axis_0) + // no filter +CASE(test_unsqueeze_axis_1) + // no filter +CASE(test_unsqueeze_axis_2) + // no filter +CASE(test_unsqueeze_axis_3) + // no filter +CASE(test_unsqueeze_negative_axes) + // no filter +CASE(test_unsqueeze_three_axes) + // no filter +CASE(test_unsqueeze_two_axes) + // no filter +CASE(test_unsqueeze_unsorted_axes) + // no filter +CASE(test_upsample_nearest) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_where_example) + // no filter +CASE(test_where_long_example) + // no filter +CASE(test_xor2d) + // no filter +CASE(test_xor3d) + // no filter +CASE(test_xor4d) + // no filter +CASE(test_xor_bcast3v1d) + // no filter +CASE(test_xor_bcast3v2d) + // no filter +CASE(test_xor_bcast4v2d) + // no filter +CASE(test_xor_bcast4v3d) + // no filter +CASE(test_xor_bcast4v4d) + // no filter +END_SWITCH() +#undef EOF_LABEL +#undef BEGIN_SWITCH +#undef CASE +#undef END_SWITCH +if (!filterApplied) +{ + ADD_FAILURE() << "OpenVINO backend: unknown test='" << name << "'. Update filter configuration"; +} + +#undef SKIP_TAGS +#undef SKIP_ +#undef SKIP +#undef SKIP_CPU +#undef SKIP_NON_CPU +#undef SKIP_OPENCL +#undef SKIP_OPENCL_FP16 +#undef SKIP_MYRIAD + +#endif From b39ee54103b29cd9e835acc65ca3b07ea1509469 Mon Sep 17 00:00:00 2001 From: Maksim Shabunin Date: Thu, 16 Dec 2021 11:40:42 +0300 Subject: [PATCH 172/226] Do not force -O2 flag in hardening-enabled builds --- cmake/OpenCVCompilerDefenses.cmake | 8 -------- 1 file changed, 8 deletions(-) diff --git a/cmake/OpenCVCompilerDefenses.cmake b/cmake/OpenCVCompilerDefenses.cmake index 62029ea38b..9fffdb4d72 100644 --- a/cmake/OpenCVCompilerDefenses.cmake +++ b/cmake/OpenCVCompilerDefenses.cmake @@ -87,11 +87,3 @@ endif() set( CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${OPENCV_LINKER_DEFENSES_FLAGS_COMMON}" ) set( CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${OPENCV_LINKER_DEFENSES_FLAGS_COMMON}" ) set( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OPENCV_LINKER_DEFENSES_FLAGS_COMMON}" ) - -if(CV_GCC OR CV_CLANG) - foreach(flags - CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_RELEASE CMAKE_CXX_FLAGS_DEBUG - CMAKE_C_FLAGS CMAKE_C_FLAGS_RELEASE CMAKE_C_FLAGS_DEBUG) - string(REPLACE "-O3" "-O2" ${flags} "${${flags}}") - endforeach() -endif() From 3da17c42a4821bd7df9bf9d43e29f8d9e138d22c Mon Sep 17 00:00:00 2001 From: Vincent Rabaud Date: Thu, 16 Dec 2021 11:27:37 +0100 Subject: [PATCH 173/226] Use NaN-safe clip function. This is to prevent more NaN to int conversions like in #21111. --- modules/imgproc/src/color_lab.cpp | 12 ++++++------ modules/imgproc/test/test_color.cpp | 2 ++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/modules/imgproc/src/color_lab.cpp b/modules/imgproc/src/color_lab.cpp index 14df6473f4..c5ebe30fe1 100644 --- a/modules/imgproc/src/color_lab.cpp +++ b/modules/imgproc/src/color_lab.cpp @@ -2979,9 +2979,9 @@ struct RGB2Luvfloat for( ; i < n; i++, src += scn, dst += 3 ) { float R = src[0], G = src[1], B = src[2]; - R = std::min(std::max(R, 0.f), 1.f); - G = std::min(std::max(G, 0.f), 1.f); - B = std::min(std::max(B, 0.f), 1.f); + R = clip(R); + G = clip(G); + B = clip(B); if( gammaTab ) { R = splineInterpolate(R*gscale, gammaTab, GAMMA_TAB_SIZE); @@ -3205,9 +3205,9 @@ struct Luv2RGBfloat float G = X*C3 + Y*C4 + Z*C5; float B = X*C6 + Y*C7 + Z*C8; - R = std::min(std::max(R, 0.f), 1.f); - G = std::min(std::max(G, 0.f), 1.f); - B = std::min(std::max(B, 0.f), 1.f); + R = clip(R); + G = clip(G); + B = clip(B); if( gammaTab ) { diff --git a/modules/imgproc/test/test_color.cpp b/modules/imgproc/test/test_color.cpp index e7eab3e0fb..a5499c7cc3 100644 --- a/modules/imgproc/test/test_color.cpp +++ b/modules/imgproc/test/test_color.cpp @@ -3203,6 +3203,8 @@ TEST(ImgProc_RGB2Lab, NaN_21111) src(0, 1) = src(0, 28) = src(0, 82) = src(0, 109) = cv::Vec3f(0, kNaN, 0); src(0, 2) = src(0, 29) = src(0, 83) = src(0, 110) = cv::Vec3f(kNaN, 0, 0); EXPECT_NO_THROW(cvtColor(src, dst, COLOR_RGB2Lab)); + EXPECT_NO_THROW(cvtColor(src, dst, COLOR_RGB2Luv)); + EXPECT_NO_THROW(cvtColor(src, dst, COLOR_Luv2RGB)); #if 0 // no NaN propagation guarantee for (int i = 0; i < 20; ++i) From 6d677bbd6306e37f2d808e8e8a7754cd92984d23 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 15 Dec 2021 20:46:28 +0000 Subject: [PATCH 174/226] dnn(test): update ONNX conformance filters (4.x) --- modules/dnn/src/layers/convolution_layer.cpp | 10 ++ modules/dnn/src/layers/resize_layer.cpp | 1 + modules/dnn/test/test_onnx_conformance.cpp | 38 ++++- ...rmance_layer_filter__cuda_denylist.inl.hpp | 74 +++++++++ ...ance_layer_filter__halide_denylist.inl.hpp | 43 +++++ ...conformance_layer_filter__openvino.inl.hpp | 152 +++++++++++++----- ...ance_layer_filter__vulkan_denylist.inl.hpp | 65 ++++++++ ...e_layer_filter_opencv_all_denylist.inl.hpp | 40 +++++ ..._conformance_layer_parser_denylist.inl.hpp | 48 ------ 9 files changed, 381 insertions(+), 90 deletions(-) create mode 100644 modules/dnn/test/test_onnx_conformance_layer_filter__cuda_denylist.inl.hpp create mode 100644 modules/dnn/test/test_onnx_conformance_layer_filter__vulkan_denylist.inl.hpp diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index 50b06a19c0..bcc783d8a0 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -647,6 +647,7 @@ public: virtual Ptr initVkCom(const std::vector > &inputs) CV_OVERRIDE { #ifdef HAVE_VULKAN + CV_Assert(!blobs.empty()); int out_channel = blobs[0].size[0]; bool has_bias = hasBias() || fusedBias; int filter_size[2] = {kernel.height, kernel.width}; @@ -712,6 +713,7 @@ public: virtual Ptr initHalide(const std::vector > &inputs) CV_OVERRIDE { #ifdef HAVE_HALIDE + CV_Assert(!blobs.empty()); Halide::Buffer inputBuffer = halideBuffer(inputs[0]); const int inpCn = inputBuffer.channels(); @@ -760,6 +762,7 @@ public: #ifdef HAVE_DNN_IE_NN_BUILDER_2019 virtual Ptr initInfEngine(const std::vector > &inputs) CV_OVERRIDE { + CV_Assert(!blobs.empty()); InferenceEngine::DataPtr input = infEngineDataNode(inputs[0]); std::vector dims = input->getDims(); CV_Assert(dims.size() == 4 || dims.size() == 5); @@ -824,6 +827,7 @@ public: virtual Ptr initNgraph(const std::vector > &inputs, const std::vector >& nodes) CV_OVERRIDE { + CV_Assert(!blobs.empty()); CV_Assert_N(inputs.size() >= 1, nodes.size() >= 1); auto& ieInpNode = nodes[0].dynamicCast()->node; std::vector dims = ieInpNode->get_shape(); @@ -917,6 +921,7 @@ public: #ifdef HAVE_WEBNN virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE { + CV_Assert(!blobs.empty()); CV_Assert_N(inputs.size() >= 1, nodes.size() >= 1); Ptr node = nodes[0].dynamicCast(); auto& webnnInpOperand = node->operand; @@ -2156,6 +2161,7 @@ public: auto output_wrapper = outputs[0].dynamicCast(); auto output_shape = output_wrapper->getShape(); + CV_Assert(!blobs.empty()); const auto output_feature_maps = blobs[0].size[0]; const auto input_feature_maps = input_shape[1]; const auto input_feature_maps_per_group = blobs[0].size[1]; @@ -2917,6 +2923,7 @@ public: const std::vector>& outputs ) override { + CV_Assert(!blobs.empty()); auto context = reinterpret_cast(context_); CV_Assert(inputs.size() == 1); @@ -2974,6 +2981,7 @@ public: virtual Ptr initHalide(const std::vector > &inputs) CV_OVERRIDE { #ifdef HAVE_HALIDE + CV_Assert(!blobs.empty()); Halide::Buffer inputBuffer = halideBuffer(inputs[0]); int inW, inH, inC, inN; @@ -3027,6 +3035,7 @@ public: #ifdef HAVE_DNN_IE_NN_BUILDER_2019 virtual Ptr initInfEngine(const std::vector > &) CV_OVERRIDE { + CV_Assert(!blobs.empty()); InferenceEngine::Layout layout = blobs[0].dims == 5? InferenceEngine::Layout::NCDHW : InferenceEngine::Layout::OIHW; @@ -3086,6 +3095,7 @@ public: virtual Ptr initNgraph(const std::vector > &inputs, const std::vector >& nodes) CV_OVERRIDE { + CV_Assert(!blobs.empty()); const int outGroupCn = blobs[0].size[1]; const int group = numOutput / outGroupCn; CV_Assert(group == 1); diff --git a/modules/dnn/src/layers/resize_layer.cpp b/modules/dnn/src/layers/resize_layer.cpp index 47ff0719fe..42eb2e2331 100644 --- a/modules/dnn/src/layers/resize_layer.cpp +++ b/modules/dnn/src/layers/resize_layer.cpp @@ -64,6 +64,7 @@ public: outputs[0][2] = zoomFactorHeight > 0 ? (outputs[0][2] * zoomFactorHeight) : outHeight; outputs[0][3] = zoomFactorWidth > 0 ? (outputs[0][3] * zoomFactorWidth) : outWidth; } else { + CV_CheckGE(inputs[1].size(), (size_t)4, ""); outputs[0][2] = inputs[1][2]; outputs[0][3] = inputs[1][3]; } diff --git a/modules/dnn/test/test_onnx_conformance.cpp b/modules/dnn/test/test_onnx_conformance.cpp index ebdaf23726..1c3877b7b2 100644 --- a/modules/dnn/test/test_onnx_conformance.cpp +++ b/modules/dnn/test/test_onnx_conformance.cpp @@ -939,6 +939,12 @@ public: #ifdef HAVE_HALIDE static std::set halide_deny_list; #endif +#ifdef HAVE_VULKAN + static std::set vulkan_deny_list; +#endif +#ifdef HAVE_CUDA + static std::set cuda_deny_list; +#endif Test_ONNX_conformance() { @@ -1008,6 +1014,18 @@ public: #include "test_onnx_conformance_layer_filter__halide_denylist.inl.hpp" }; #endif + +#ifdef HAVE_VULKAN + vulkan_deny_list = { + #include "test_onnx_conformance_layer_filter__vulkan_denylist.inl.hpp" + }; +#endif + +#ifdef HAVE_CUDA + cuda_deny_list = { + #include "test_onnx_conformance_layer_filter__cuda_denylist.inl.hpp" + }; +#endif } }; @@ -1020,6 +1038,12 @@ std::set Test_ONNX_conformance::cpu_deny_list; #ifdef HAVE_HALIDE std::set Test_ONNX_conformance::halide_deny_list; #endif +#ifdef HAVE_VULKAN +std::set Test_ONNX_conformance::vulkan_deny_list; +#endif +#ifdef HAVE_CUDA +std::set Test_ONNX_conformance::cuda_deny_list; +#endif TEST_P(Test_ONNX_conformance, Layer_Test) { @@ -1068,16 +1092,22 @@ TEST_P(Test_ONNX_conformance, Layer_Test) #include "test_onnx_conformance_layer_filter__openvino.inl.hpp" } #endif -#if 0 //def HAVE_VULKAN +#ifdef HAVE_VULKAN else if (backend == DNN_BACKEND_VKCOM) { - #include "test_onnx_conformance_layer_filter__vulkan.inl.hpp" + if (vulkan_deny_list.find(name) != vulkan_deny_list.end()) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_VULKAN, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } } #endif -#if 0 //def HAVE_CUDA +#ifdef HAVE_CUDA else if (backend == DNN_BACKEND_CUDA) { - #include "test_onnx_conformance_layer_filter__cuda.inl.hpp" + if (cuda_deny_list.find(name) != cuda_deny_list.end()) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } } #endif else diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__cuda_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__cuda_denylist.inl.hpp new file mode 100644 index 0000000000..61a5843b35 --- /dev/null +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__cuda_denylist.inl.hpp @@ -0,0 +1,74 @@ +"test_add_bcast", +"test_add_uint8", +"test_argmax_default_axis_example", +"test_argmax_default_axis_example_select_last_index", +"test_argmax_default_axis_random", +"test_argmax_default_axis_random_select_last_index", +"test_argmax_keepdims_example", +"test_argmax_keepdims_example_select_last_index", +"test_argmax_keepdims_random", +"test_argmax_keepdims_random_select_last_index", +"test_argmax_negative_axis_keepdims_example", +"test_argmax_negative_axis_keepdims_example_select_last_index", +"test_argmax_negative_axis_keepdims_random", +"test_argmax_negative_axis_keepdims_random_select_last_index", +"test_argmax_no_keepdims_example", +"test_argmax_no_keepdims_example_select_last_index", +"test_argmax_no_keepdims_random", +"test_argmax_no_keepdims_random_select_last_index", +"test_argmin_default_axis_example", +"test_argmin_default_axis_example_select_last_index", +"test_argmin_default_axis_random", +"test_argmin_default_axis_random_select_last_index", +"test_argmin_keepdims_example", +"test_argmin_keepdims_example_select_last_index", +"test_argmin_keepdims_random", +"test_argmin_keepdims_random_select_last_index", +"test_argmin_negative_axis_keepdims_example", +"test_argmin_negative_axis_keepdims_example_select_last_index", +"test_argmin_negative_axis_keepdims_random", +"test_argmin_negative_axis_keepdims_random_select_last_index", +"test_argmin_no_keepdims_example", +"test_argmin_no_keepdims_example_select_last_index", +"test_argmin_no_keepdims_random", +"test_argmin_no_keepdims_random_select_last_index", +"test_averagepool_2d_pads_count_include_pad", +"test_averagepool_2d_precomputed_pads_count_include_pad", +"test_averagepool_2d_same_lower", +"test_basic_conv_with_padding", +"test_basic_conv_without_padding", +"test_cast_FLOAT_to_STRING", +"test_cast_STRING_to_FLOAT", +"test_castlike_FLOAT_to_STRING_expanded", +"test_castlike_STRING_to_FLOAT_expanded", +"test_concat_1d_axis_negative_1", +"test_conv_with_autopad_same", +"test_conv_with_strides_and_asymmetric_padding", +"test_conv_with_strides_no_padding", +"test_conv_with_strides_padding", +"test_div_bcast", +"test_div_uint8", +"test_dropout_default_ratio", +"test_flatten_axis0", +"test_flatten_axis2", +"test_flatten_axis3", +"test_flatten_negative_axis1", +"test_flatten_negative_axis2", +"test_flatten_negative_axis4", +"test_logsoftmax_default_axis", +"test_logsoftmax_large_number", +"test_logsoftmax_large_number_expanded", +"test_maxpool_2d_dilations", +"test_maxpool_2d_same_lower", +"test_maxpool_2d_uint8", +"test_maxpool_with_argmax_2d_precomputed_pads", +"test_maxpool_with_argmax_2d_precomputed_strides", +"test_maxunpool_export_with_output_shape", +"test_mul_bcast", +"test_mul_uint8", +"test_softmax_default_axis", +"test_softmax_large_number", // FP16 only +"test_softmax_large_number_expanded", // FP16 only +"test_sub_bcast", +"test_sub_uint8", +"test_upsample_nearest", diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__halide_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__halide_denylist.inl.hpp index dd0a249081..344e5604c7 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter__halide_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__halide_denylist.inl.hpp @@ -1,5 +1,39 @@ +"test_abs", "test_add", "test_add_bcast", +"test_add_uint8", +"test_argmax_default_axis_example", +"test_argmax_default_axis_example_select_last_index", +"test_argmax_default_axis_random", +"test_argmax_default_axis_random_select_last_index", +"test_argmax_keepdims_example", +"test_argmax_keepdims_example_select_last_index", +"test_argmax_keepdims_random", +"test_argmax_keepdims_random_select_last_index", +"test_argmax_negative_axis_keepdims_example", +"test_argmax_negative_axis_keepdims_example_select_last_index", +"test_argmax_negative_axis_keepdims_random", +"test_argmax_negative_axis_keepdims_random_select_last_index", +"test_argmax_no_keepdims_example", +"test_argmax_no_keepdims_example_select_last_index", +"test_argmax_no_keepdims_random", +"test_argmax_no_keepdims_random_select_last_index", +"test_argmin_default_axis_example", +"test_argmin_default_axis_example_select_last_index", +"test_argmin_default_axis_random", +"test_argmin_default_axis_random_select_last_index", +"test_argmin_keepdims_example", +"test_argmin_keepdims_example_select_last_index", +"test_argmin_keepdims_random", +"test_argmin_keepdims_random_select_last_index", +"test_argmin_negative_axis_keepdims_example", +"test_argmin_negative_axis_keepdims_example_select_last_index", +"test_argmin_negative_axis_keepdims_random", +"test_argmin_negative_axis_keepdims_random_select_last_index", +"test_argmin_no_keepdims_example", +"test_argmin_no_keepdims_example_select_last_index", +"test_argmin_no_keepdims_random", +"test_argmin_no_keepdims_random_select_last_index", "test_averagepool_2d_ceil", "test_averagepool_2d_pads_count_include_pad", "test_averagepool_2d_precomputed_pads_count_include_pad", @@ -10,10 +44,12 @@ "test_cast_STRING_to_FLOAT", "test_castlike_FLOAT_to_STRING_expanded", "test_castlike_STRING_to_FLOAT_expanded", +"test_ceil", "test_concat_1d_axis_negative_1", "test_concat_3d_axis_1", "test_div", "test_div_bcast", +"test_div_uint8", "test_elu", "test_elu_default", "test_exp", @@ -23,8 +59,10 @@ "test_flatten_negative_axis1", "test_flatten_negative_axis2", "test_flatten_negative_axis4", +"test_floor", "test_leakyrelu", "test_leakyrelu_default", +"test_log", "test_logsoftmax_axis_1", "test_logsoftmax_axis_1_expanded", "test_logsoftmax_default_axis", @@ -35,10 +73,13 @@ "test_matmul_4d", "test_maxpool_2d_dilations", "test_maxpool_2d_same_lower", +"test_maxpool_2d_uint8", "test_maxpool_with_argmax_2d_precomputed_pads", "test_maxpool_with_argmax_2d_precomputed_strides", +"test_maxunpool_export_with_output_shape", "test_mul", "test_mul_bcast", +"test_mul_uint8", "test_neg", "test_reduce_max_default_axes_keepdim_example", "test_reduce_max_default_axes_keepdims_random", @@ -54,7 +95,9 @@ "test_softmax_axis_1_expanded", "test_softmax_default_axis", "test_softmax_large_number", +"test_sqrt", "test_sub", "test_sub_bcast", +"test_sub_uint8", "test_tanh", "test_upsample_nearest", diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp index 25bb8dff9a..8938b00914 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp @@ -86,7 +86,9 @@ CASE(test_add_bcast) SKIP; #endif CASE(test_add_uint8) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_and2d) // no filter CASE(test_and3d) @@ -104,69 +106,133 @@ CASE(test_and_bcast4v3d) CASE(test_and_bcast4v4d) // no filter CASE(test_argmax_default_axis_example) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmax_default_axis_example_select_last_index) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmax_default_axis_random) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmax_default_axis_random_select_last_index) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmax_keepdims_example) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmax_keepdims_example_select_last_index) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmax_keepdims_random) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmax_keepdims_random_select_last_index) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmax_negative_axis_keepdims_example) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmax_negative_axis_keepdims_example_select_last_index) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmax_negative_axis_keepdims_random) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmax_negative_axis_keepdims_random_select_last_index) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmax_no_keepdims_example) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmax_no_keepdims_example_select_last_index) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmax_no_keepdims_random) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmax_no_keepdims_random_select_last_index) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmin_default_axis_example) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmin_default_axis_example_select_last_index) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmin_default_axis_random) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmin_default_axis_random_select_last_index) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmin_keepdims_example) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmin_keepdims_example_select_last_index) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmin_keepdims_random) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmin_keepdims_random_select_last_index) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmin_negative_axis_keepdims_example) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmin_negative_axis_keepdims_example_select_last_index) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmin_negative_axis_keepdims_random) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmin_negative_axis_keepdims_random_select_last_index) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmin_no_keepdims_example) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmin_no_keepdims_example_select_last_index) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmin_no_keepdims_random) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_argmin_no_keepdims_random_select_last_index) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_asin) // no filter CASE(test_asin_example) @@ -495,7 +561,9 @@ CASE(test_div_bcast) CASE(test_div_example) // no filter CASE(test_div_uint8) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_dropout_default) // no filter CASE(test_dropout_default_mask) @@ -895,7 +963,9 @@ CASE(test_maxpool_2d_strides) SKIP_MYRIAD; #endif CASE(test_maxpool_2d_uint8) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_maxpool_3d_default) #if INF_ENGINE_VER_MAJOR_EQ(2021040000) SKIP_NON_CPU; @@ -909,7 +979,9 @@ CASE(test_maxpool_with_argmax_2d_precomputed_strides) SKIP; #endif CASE(test_maxunpool_export_with_output_shape) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_maxunpool_export_without_output_shape) #if INF_ENGINE_VER_MAJOR_EQ(2021040000) SKIP; @@ -987,7 +1059,9 @@ CASE(test_mul_bcast) CASE(test_mul_example) // no filter CASE(test_mul_uint8) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_mvn) // no filter CASE(test_mvn_expanded) @@ -1766,7 +1840,9 @@ CASE(test_sub_bcast) CASE(test_sub_example) // no filter CASE(test_sub_uint8) - // no filter +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif CASE(test_sum_example) // no filter CASE(test_sum_one_input) diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__vulkan_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__vulkan_denylist.inl.hpp new file mode 100644 index 0000000000..101d44cbf0 --- /dev/null +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__vulkan_denylist.inl.hpp @@ -0,0 +1,65 @@ +"test_add_bcast", +"test_add_uint8", +"test_argmax_default_axis_example", +"test_argmax_default_axis_example_select_last_index", +"test_argmax_default_axis_random", +"test_argmax_default_axis_random_select_last_index", +"test_argmax_keepdims_example", +"test_argmax_keepdims_example_select_last_index", +"test_argmax_keepdims_random", +"test_argmax_keepdims_random_select_last_index", +"test_argmax_negative_axis_keepdims_example", +"test_argmax_negative_axis_keepdims_example_select_last_index", +"test_argmax_negative_axis_keepdims_random", +"test_argmax_negative_axis_keepdims_random_select_last_index", +"test_argmax_no_keepdims_example", +"test_argmax_no_keepdims_example_select_last_index", +"test_argmax_no_keepdims_random", +"test_argmax_no_keepdims_random_select_last_index", +"test_argmin_default_axis_example", +"test_argmin_default_axis_example_select_last_index", +"test_argmin_default_axis_random", +"test_argmin_default_axis_random_select_last_index", +"test_argmin_keepdims_example", +"test_argmin_keepdims_example_select_last_index", +"test_argmin_keepdims_random", +"test_argmin_keepdims_random_select_last_index", +"test_argmin_negative_axis_keepdims_example", +"test_argmin_negative_axis_keepdims_example_select_last_index", +"test_argmin_negative_axis_keepdims_random", +"test_argmin_negative_axis_keepdims_random_select_last_index", +"test_argmin_no_keepdims_example", +"test_argmin_no_keepdims_example_select_last_index", +"test_argmin_no_keepdims_random", +"test_argmin_no_keepdims_random_select_last_index", +"test_averagepool_2d_pads_count_include_pad", +"test_averagepool_2d_precomputed_pads_count_include_pad", +"test_averagepool_2d_same_lower", +"test_averagepool_3d_default", +"test_cast_FLOAT_to_STRING", +"test_cast_STRING_to_FLOAT", +"test_castlike_FLOAT_to_STRING_expanded", +"test_castlike_STRING_to_FLOAT_expanded", +"test_concat_1d_axis_negative_1", +"test_div_uint8", +"test_flatten_axis0", +"test_flatten_axis2", +"test_flatten_axis3", +"test_flatten_negative_axis1", +"test_flatten_negative_axis2", +"test_flatten_negative_axis4", +"test_logsoftmax_default_axis", +"test_maxpool_2d_dilations", +"test_maxpool_2d_same_lower", +"test_maxpool_2d_uint8", +"test_maxpool_3d_default", +"test_maxpool_with_argmax_2d_precomputed_pads", +"test_maxpool_with_argmax_2d_precomputed_strides", +"test_maxunpool_export_with_output_shape", +"test_maxunpool_export_without_output_shape", +"test_mul_uint8", +"test_softmax_default_axis", +"test_sub_bcast", +"test_sub_uint8", +"test_transpose_all_permutations_0", +"test_upsample_nearest", diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp index cff1e93aa0..9e4253ae7a 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp @@ -1,4 +1,39 @@ "test_add_bcast", +"test_add_uint8", // output type mismatch +#if 1 // output type mismatch CV_32F vs expected 32S +"test_argmax_default_axis_example", +"test_argmax_default_axis_example_select_last_index", +"test_argmax_default_axis_random", +"test_argmax_default_axis_random_select_last_index", +"test_argmax_keepdims_example", +"test_argmax_keepdims_example_select_last_index", +"test_argmax_keepdims_random", +"test_argmax_keepdims_random_select_last_index", +"test_argmax_negative_axis_keepdims_example", +"test_argmax_negative_axis_keepdims_example_select_last_index", +"test_argmax_negative_axis_keepdims_random", +"test_argmax_negative_axis_keepdims_random_select_last_index", +"test_argmax_no_keepdims_example", +"test_argmax_no_keepdims_example_select_last_index", +"test_argmax_no_keepdims_random", +"test_argmax_no_keepdims_random_select_last_index", +"test_argmin_default_axis_example", +"test_argmin_default_axis_example_select_last_index", +"test_argmin_default_axis_random", +"test_argmin_default_axis_random_select_last_index", +"test_argmin_keepdims_example", +"test_argmin_keepdims_example_select_last_index", +"test_argmin_keepdims_random", +"test_argmin_keepdims_random_select_last_index", +"test_argmin_negative_axis_keepdims_example", +"test_argmin_negative_axis_keepdims_example_select_last_index", +"test_argmin_negative_axis_keepdims_random", +"test_argmin_negative_axis_keepdims_random_select_last_index", +"test_argmin_no_keepdims_example", +"test_argmin_no_keepdims_example_select_last_index", +"test_argmin_no_keepdims_random", +"test_argmin_no_keepdims_random_select_last_index", +#endif "test_averagepool_2d_pads_count_include_pad", "test_averagepool_2d_precomputed_pads_count_include_pad", "test_averagepool_2d_same_lower", @@ -7,6 +42,7 @@ "test_castlike_FLOAT_to_STRING_expanded", "test_castlike_STRING_to_FLOAT_expanded", "test_concat_1d_axis_negative_1", +"test_div_uint8", // output type mismatch "test_flatten_axis0", "test_flatten_axis2", "test_flatten_axis3", @@ -16,8 +52,12 @@ "test_logsoftmax_default_axis", "test_maxpool_2d_dilations", "test_maxpool_2d_same_lower", +"test_maxpool_2d_uint8", // output type mismatch "test_maxpool_with_argmax_2d_precomputed_pads", "test_maxpool_with_argmax_2d_precomputed_strides", +"test_maxunpool_export_with_output_shape", // exception during net.forward() call +"test_mul_uint8", // output type mismatch "test_softmax_default_axis", "test_sub_bcast", +"test_sub_uint8", // output type mismatch "test_upsample_nearest", diff --git a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp index 2489105e91..b95cafd5d7 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp @@ -1,6 +1,5 @@ // The file is autogenerated // Update note: execute /testdata/dnn/onnx/generate_conformance_list.py -"test_abs", "test_acos", "test_acos_example", "test_acosh", @@ -9,7 +8,6 @@ "test_adagrad_multiple", "test_adam", "test_adam_multiple", -"test_add_uint8", "test_and2d", "test_and3d", "test_and4d", @@ -18,38 +16,6 @@ "test_and_bcast4v2d", "test_and_bcast4v3d", "test_and_bcast4v4d", -"test_argmax_default_axis_example", -"test_argmax_default_axis_example_select_last_index", -"test_argmax_default_axis_random", -"test_argmax_default_axis_random_select_last_index", -"test_argmax_keepdims_example", -"test_argmax_keepdims_example_select_last_index", -"test_argmax_keepdims_random", -"test_argmax_keepdims_random_select_last_index", -"test_argmax_negative_axis_keepdims_example", -"test_argmax_negative_axis_keepdims_example_select_last_index", -"test_argmax_negative_axis_keepdims_random", -"test_argmax_negative_axis_keepdims_random_select_last_index", -"test_argmax_no_keepdims_example", -"test_argmax_no_keepdims_example_select_last_index", -"test_argmax_no_keepdims_random", -"test_argmax_no_keepdims_random_select_last_index", -"test_argmin_default_axis_example", -"test_argmin_default_axis_example_select_last_index", -"test_argmin_default_axis_random", -"test_argmin_default_axis_random_select_last_index", -"test_argmin_keepdims_example", -"test_argmin_keepdims_example_select_last_index", -"test_argmin_keepdims_random", -"test_argmin_keepdims_random_select_last_index", -"test_argmin_negative_axis_keepdims_example", -"test_argmin_negative_axis_keepdims_example_select_last_index", -"test_argmin_negative_axis_keepdims_random", -"test_argmin_negative_axis_keepdims_random_select_last_index", -"test_argmin_no_keepdims_example", -"test_argmin_no_keepdims_example_select_last_index", -"test_argmin_no_keepdims_random", -"test_argmin_no_keepdims_random_select_last_index", "test_asin", "test_asin_example", "test_asinh", @@ -103,8 +69,6 @@ "test_castlike_FLOAT_to_FLOAT16_expanded", "test_castlike_FLOAT_to_STRING", "test_castlike_STRING_to_FLOAT", -"test_ceil", -"test_ceil_example", "test_celu", "test_clip", "test_clip_default_inbounds", @@ -158,7 +122,6 @@ "test_det_2d", "test_det_nd", "test_div_example", -"test_div_uint8", "test_dropout_default_mask", "test_dropout_default_mask_ratio", "test_dynamicquantizelinear", @@ -181,8 +144,6 @@ "test_eyelike_populate_off_main_diagonal", "test_eyelike_with_dtype", "test_eyelike_without_dtype", -"test_floor", -"test_floor_example", "test_gather_0", "test_gather_1", "test_gather_2d_indices", @@ -251,8 +212,6 @@ "test_less_equal_bcast", "test_less_equal_bcast_expanded", "test_less_equal_expanded", -"test_log", -"test_log_example", "test_loop11", "test_loop13_seq", "test_loop16_seq_none", @@ -275,8 +234,6 @@ "test_max_uint32", "test_max_uint64", "test_max_uint8", -"test_maxpool_2d_uint8", -"test_maxunpool_export_with_output_shape", "test_mean_example", "test_mean_one_input", "test_mean_two_inputs", @@ -310,7 +267,6 @@ "test_momentum", "test_momentum_multiple", "test_mul_example", -"test_mul_uint8", "test_mvn", "test_mvn_expanded", "test_nesterov_momentum", @@ -508,7 +464,6 @@ "test_rnn_seq_length", "test_roialign_aligned_false", "test_roialign_aligned_true", -"test_round", "test_scan9_sum", "test_scan_sum", "test_scatter_elements_with_axis", @@ -633,8 +588,6 @@ "test_split_variable_parts_2d", "test_split_variable_parts_default_axis", "test_split_zero_size_splits", -"test_sqrt", -"test_sqrt_example", "test_squeeze", "test_squeeze_negative_axes", "test_strnormalizer_export_monday_casesensintive_lower", @@ -644,7 +597,6 @@ "test_strnormalizer_export_monday_insensintive_upper_twodim", "test_strnormalizer_nostopwords_nochangecase", "test_sub_example", -"test_sub_uint8", "test_sum_example", "test_sum_two_inputs", "test_tan", From a079c2eb7cb74b4520c69b6f6c2ea79c7d410871 Mon Sep 17 00:00:00 2001 From: Maksim Shabunin Date: Wed, 15 Dec 2021 12:49:13 +0300 Subject: [PATCH 175/226] Fixed several issues found by static analysis --- modules/dnn/src/int8layers/eltwise_layer.cpp | 5 +++-- modules/dnn/src/onnx/onnx_importer.cpp | 4 +++- modules/objdetect/src/face_detect.cpp | 1 + modules/objdetect/src/face_recognize.cpp | 4 ++-- modules/objdetect/src/qrcode_encoder.cpp | 2 ++ 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/modules/dnn/src/int8layers/eltwise_layer.cpp b/modules/dnn/src/int8layers/eltwise_layer.cpp index be7a32b1ef..a522bc9031 100644 --- a/modules/dnn/src/int8layers/eltwise_layer.cpp +++ b/modules/dnn/src/int8layers/eltwise_layer.cpp @@ -238,7 +238,7 @@ public: EltwiseInvoker(EltwiseLayerInt8Impl& self_) : self(self_) - , nsrcs(0), dst(0), buf(0), nstripes(0), activ(0), channels(0) + , nsrcs(0), dst(0), buf(0), nstripes(0), activLUT(0), activ(0), channels(0) , planeSize(0), offset(0) {} @@ -345,7 +345,8 @@ public: int8_t* dstptr0 = dst->ptr(); float* bufptr0 = buf->ptr(); int blockSize0 = 1 << 12; - + CV_Assert(op != PROD || zeropointsptr); + CV_Assert((op != PROD && op != SUM) || coeffsptr); for (size_t ofs = stripeStart; ofs < stripeEnd; ) { int sampleIdx = (int)(ofs / planeSize); diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 85c4479c6f..8edc38979e 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -2023,7 +2023,9 @@ void ONNXImporter::parseUnsqueeze(LayerParams& layerParams, const opencv_onnx::N } CV_Assert(axes.getIntValue(axes.size()-1) <= dims.size()); for (int j = 0; j < axes.size(); j++) { - dims.insert(dims.begin() + axes.getIntValue(j), 1); + const int idx = axes.getIntValue(j); + CV_Assert(idx <= dims.size()); + dims.insert(dims.begin() + idx, 1); } Mat out = input.reshape(0, dims); diff --git a/modules/objdetect/src/face_detect.cpp b/modules/objdetect/src/face_detect.cpp index 4095745b7e..a9ca2d8957 100644 --- a/modules/objdetect/src/face_detect.cpp +++ b/modules/objdetect/src/face_detect.cpp @@ -142,6 +142,7 @@ private: {64.0f, 96.0f}, {128.0f, 192.0f, 256.0f} }; + CV_Assert(min_sizes.size() == feature_map_sizes.size()); // just to keep vectors in sync const std::vector steps = { 8, 16, 32, 64 }; // Generate priors diff --git a/modules/objdetect/src/face_recognize.cpp b/modules/objdetect/src/face_recognize.cpp index 6550a13b4b..66271068b2 100644 --- a/modules/objdetect/src/face_recognize.cpp +++ b/modules/objdetect/src/face_recognize.cpp @@ -45,8 +45,8 @@ public: double match(InputArray _face_feature1, InputArray _face_feature2, int dis_type) const override { Mat face_feature1 = _face_feature1.getMat(), face_feature2 = _face_feature2.getMat(); - face_feature1 /= norm(face_feature1); - face_feature2 /= norm(face_feature2); + normalize(face_feature1, face_feature1); + normalize(face_feature2, face_feature2); if(dis_type == DisType::FR_COSINE){ return sum(face_feature1.mul(face_feature2))[0]; diff --git a/modules/objdetect/src/qrcode_encoder.cpp b/modules/objdetect/src/qrcode_encoder.cpp index ae15c64f42..2b363b607d 100644 --- a/modules/objdetect/src/qrcode_encoder.cpp +++ b/modules/objdetect/src/qrcode_encoder.cpp @@ -881,6 +881,8 @@ void QRCodeEncoderImpl::findAutoMaskType() total_modules += 1; } } + if (total_modules == 0) + continue; // TODO: refactor, extract functions to reduce complexity int modules_percent = dark_modules * 100 / total_modules; int lower_bound = 45; int upper_bound = 55; From 792b7e062958c76da1d9ddba94507a28ad9cd891 Mon Sep 17 00:00:00 2001 From: Maksim Shabunin Date: Wed, 15 Dec 2021 12:49:13 +0300 Subject: [PATCH 176/226] (3.4) Fixed several issues found by static analysis original commit: a079c2eb7cb74b4520c69b6f6c2ea79c7d410871 --- modules/dnn/src/onnx/onnx_importer.cpp | 4 +++- modules/objdetect/src/qrcode_encoder.cpp | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 052b1b8529..df404a93d3 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -1821,7 +1821,9 @@ void ONNXImporter::parseUnsqueeze(LayerParams& layerParams, const opencv_onnx::N } CV_Assert(axes.getIntValue(axes.size()-1) <= dims.size()); for (int j = 0; j < axes.size(); j++) { - dims.insert(dims.begin() + axes.getIntValue(j), 1); + const int idx = axes.getIntValue(j); + CV_Assert(idx <= dims.size()); + dims.insert(dims.begin() + idx, 1); } Mat out = input.reshape(0, dims); diff --git a/modules/objdetect/src/qrcode_encoder.cpp b/modules/objdetect/src/qrcode_encoder.cpp index ae15c64f42..2b363b607d 100644 --- a/modules/objdetect/src/qrcode_encoder.cpp +++ b/modules/objdetect/src/qrcode_encoder.cpp @@ -881,6 +881,8 @@ void QRCodeEncoderImpl::findAutoMaskType() total_modules += 1; } } + if (total_modules == 0) + continue; // TODO: refactor, extract functions to reduce complexity int modules_percent = dark_modules * 100 / total_modules; int lower_bound = 45; int upper_bound = 55; From b4bb98ea60f63e99909f119aee27e76f7e26ddc3 Mon Sep 17 00:00:00 2001 From: Gruhuang <56301098+Crayon-new@users.noreply.github.com> Date: Fri, 17 Dec 2021 01:06:02 +0800 Subject: [PATCH 177/226] Merge pull request #21268 from pccvlab:tf_Arg add argmax and argmin parsing for tensorflow * add argmax and argmin for tf * remove whitespace * remove whitespace * remove static_cast Signed-off-by: Crayon-new <1349159541@qq.com> --- modules/dnn/src/tensorflow/tf_importer.cpp | 19 +++++++++++++++++++ modules/dnn/test/test_tf_importer.cpp | 8 ++++++++ 2 files changed, 27 insertions(+) diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index e37e888e35..efaedfaab1 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -599,6 +599,8 @@ private: void parseActivation (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); void parseExpandDims (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); void parseSquare (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseArg (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseCustomLayer (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); }; @@ -677,6 +679,7 @@ const TFImporter::DispatchMap TFImporter::buildDispatchMap() dispatch["Elu"] = dispatch["Exp"] = dispatch["Identity"] = dispatch["Relu6"] = &TFImporter::parseActivation; dispatch["ExpandDims"] = &TFImporter::parseExpandDims; dispatch["Square"] = &TFImporter::parseSquare; + dispatch["ArgMax"] = dispatch["ArgMin"] = &TFImporter::parseArg; return dispatch; } @@ -2624,6 +2627,22 @@ void TFImporter::parseActivation(tensorflow::GraphDef& net, const tensorflow::No connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); } +void TFImporter::parseArg(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const std::string& type = layer.op(); + + Mat dimension = getTensorContent(getConstBlob(layer, value_id, 1)); + CV_Assert(dimension.total() == 1 && dimension.type() == CV_32SC1); + layerParams.set("axis", *dimension.ptr()); + layerParams.set("op", type == "ArgMax" ? "max" : "min"); + layerParams.set("keepdims", false); //tensorflow doesn't have this atrr, the output's dims minus one(default); + + int id = dstNet.addLayer(name, "Arg", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); +} + void TFImporter::parseCustomLayer(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { // Importer does not know how to map this TensorFlow's operation onto OpenCV's layer. diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index 1c4beb7468..b40a604a6e 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -185,6 +185,14 @@ TEST_P(Test_TensorFlow_layers, reduce_sum_channel_keep_dims) runTensorFlowNet("reduce_sum_channel", false, 0.0, 0.0, false, "_keep_dims"); } +TEST_P(Test_TensorFlow_layers, ArgLayer) +{ + if (backend != DNN_BACKEND_OPENCV || target != DNN_TARGET_CPU) + throw SkipTestException("Only CPU is supported"); // FIXIT use tags + runTensorFlowNet("argmax"); + runTensorFlowNet("argmin"); +} + TEST_P(Test_TensorFlow_layers, conv_single_conv) { runTensorFlowNet("single_conv"); From 05bfdeab7ae1878b2082a29cba8736c297728d6f Mon Sep 17 00:00:00 2001 From: alessandro faria Date: Thu, 16 Dec 2021 15:07:53 -0300 Subject: [PATCH 178/226] New example for realsense --- doc/tutorials/app/intelperc.markdown | 2 +- samples/cpp/videocapture_intelperc.cpp | 352 ------------------------- samples/cpp/videocapture_realsense.cpp | 33 +++ 3 files changed, 34 insertions(+), 353 deletions(-) delete mode 100644 samples/cpp/videocapture_intelperc.cpp create mode 100644 samples/cpp/videocapture_realsense.cpp diff --git a/doc/tutorials/app/intelperc.markdown b/doc/tutorials/app/intelperc.markdown index 5c036a63c2..210cf17023 100644 --- a/doc/tutorials/app/intelperc.markdown +++ b/doc/tutorials/app/intelperc.markdown @@ -81,5 +81,5 @@ there are two flags that should be used to set/get property of the needed genera flag value is assumed by default if neither of the two possible values of the property is set. For more information please refer to the example of usage -[videocapture_intelperc.cpp](https://github.com/opencv/opencv/tree/master/samples/cpp/videocapture_intelperc.cpp) +[videocapture_realsense.cpp](https://github.com/opencv/opencv/tree/master/samples/cpp/videocapture_realsense.cpp) in opencv/samples/cpp folder. diff --git a/samples/cpp/videocapture_intelperc.cpp b/samples/cpp/videocapture_intelperc.cpp deleted file mode 100644 index 7ac3a7c184..0000000000 --- a/samples/cpp/videocapture_intelperc.cpp +++ /dev/null @@ -1,352 +0,0 @@ -#include "opencv2/videoio.hpp" -#include "opencv2/highgui.hpp" - -#include - -using namespace cv; -using namespace std; - -static bool g_printStreamSetting; -static int g_imageStreamProfileIdx; -static int g_depthStreamProfileIdx; -static bool g_irStreamShow; -static double g_imageBrightness; -static double g_imageContrast; -static bool g_printTiming; -static bool g_showClosedPoint; - - -static int g_closedDepthPoint[2]; - -static void printUsage(const char *arg0) -{ - const char *filename = arg0; - while (*filename) - filename++; - while ((arg0 <= filename) && ('\\' != *filename) && ('/' != *filename)) - filename--; - filename++; - - cout << "This program demonstrates usage of camera supported\nby Intel Perceptual computing SDK." << endl << endl; - cout << "usage: " << filename << "[-ps] [-isp=IDX] [-dsp=IDX]\n [-ir] [-imb=VAL] [-imc=VAL]" << endl << endl; - cout << " -ps, print streams setting and profiles" << endl; - cout << " -isp=IDX, set profile index of the image stream" << endl; - cout << " -dsp=IDX, set profile index of the depth stream" << endl; - cout << " -ir, show data from IR stream" << endl; - cout << " -imb=VAL, set brightness value for an image stream" << endl; - cout << " -imc=VAL, set contrast value for a image stream" << endl; - cout << " -pts, print frame index and frame time" << endl; - cout << " --show-closed, print frame index and frame time" << endl; - cout << endl; -} - -static void parseCMDLine(int argc, char* argv[]) -{ - cv::CommandLineParser parser(argc, argv, - "{ h help | | }" - "{ ps print-streams | | }" - "{ isp image-stream-prof | -1 | }" - "{ dsp depth-stream-prof | -1 | }" - "{ir||}{imb||}{imc||}{pts||}{show-closed||}"); - if (parser.has("h")) - { - printUsage(argv[0]); - exit(0); - } - g_printStreamSetting = parser.has("ps"); - g_imageStreamProfileIdx = parser.get("isp"); - g_depthStreamProfileIdx = parser.get("dsp"); - g_irStreamShow = parser.has("ir"); - if (parser.has("imb")) - g_imageBrightness = parser.get("imb"); - else - g_imageBrightness = -DBL_MAX; - if (parser.has("imc")) - g_imageContrast = parser.get("imc"); - else - g_imageContrast = -DBL_MAX; - g_printTiming = parser.has("pts"); - g_showClosedPoint = parser.has("show-closed"); - if (!parser.check()) - { - parser.printErrors(); - exit(-1); - } - if (g_showClosedPoint && (-1 == g_depthStreamProfileIdx)) - { - cerr << "For --show-closed depth profile has be selected" << endl; - exit(-1); - } -} - -static void printStreamProperties(VideoCapture &capture) -{ - size_t profilesCount = (size_t)capture.get(CAP_INTELPERC_IMAGE_GENERATOR | CAP_PROP_INTELPERC_PROFILE_COUNT); - cout << "Image stream." << endl; - cout << " Brightness = " << capture.get(CAP_INTELPERC_IMAGE_GENERATOR | CAP_PROP_BRIGHTNESS) << endl; - cout << " Contrast = " << capture.get(CAP_INTELPERC_IMAGE_GENERATOR | CAP_PROP_CONTRAST) << endl; - cout << " Saturation = " << capture.get(CAP_INTELPERC_IMAGE_GENERATOR | CAP_PROP_SATURATION) << endl; - cout << " Hue = " << capture.get(CAP_INTELPERC_IMAGE_GENERATOR | CAP_PROP_HUE) << endl; - cout << " Gamma = " << capture.get(CAP_INTELPERC_IMAGE_GENERATOR | CAP_PROP_GAMMA) << endl; - cout << " Sharpness = " << capture.get(CAP_INTELPERC_IMAGE_GENERATOR | CAP_PROP_SHARPNESS) << endl; - cout << " Gain = " << capture.get(CAP_INTELPERC_IMAGE_GENERATOR | CAP_PROP_GAIN) << endl; - cout << " Backligh = " << capture.get(CAP_INTELPERC_IMAGE_GENERATOR | CAP_PROP_BACKLIGHT) << endl; - cout << "Image streams profiles:" << endl; - for (size_t i = 0; i < profilesCount; i++) - { - capture.set(CAP_INTELPERC_IMAGE_GENERATOR | CAP_PROP_INTELPERC_PROFILE_IDX, (double)i); - cout << " Profile[" << i << "]: "; - cout << "width = " << - (int)capture.get(CAP_INTELPERC_IMAGE_GENERATOR | CAP_PROP_FRAME_WIDTH); - cout << ", height = " << - (int)capture.get(CAP_INTELPERC_IMAGE_GENERATOR | CAP_PROP_FRAME_HEIGHT); - cout << ", fps = " << - capture.get(CAP_INTELPERC_IMAGE_GENERATOR | CAP_PROP_FPS); - cout << endl; - } - - profilesCount = (size_t)capture.get(CAP_INTELPERC_DEPTH_GENERATOR | CAP_PROP_INTELPERC_PROFILE_COUNT); - cout << "Depth stream." << endl; - cout << " Low confidence value = " << capture.get(CAP_INTELPERC_DEPTH_GENERATOR | CAP_PROP_INTELPERC_DEPTH_LOW_CONFIDENCE_VALUE) << endl; - cout << " Saturation value = " << capture.get(CAP_INTELPERC_DEPTH_GENERATOR | CAP_PROP_INTELPERC_DEPTH_SATURATION_VALUE) << endl; - cout << " Confidence threshold = " << capture.get(CAP_INTELPERC_DEPTH_GENERATOR | CAP_PROP_INTELPERC_DEPTH_CONFIDENCE_THRESHOLD) << endl; - cout << " Focal length = (" << capture.get(CAP_INTELPERC_DEPTH_GENERATOR | CAP_PROP_INTELPERC_DEPTH_FOCAL_LENGTH_HORZ) << ", " - << capture.get(CAP_INTELPERC_DEPTH_GENERATOR | CAP_PROP_INTELPERC_DEPTH_FOCAL_LENGTH_VERT) << ")" << endl; - cout << "Depth streams profiles:" << endl; - for (size_t i = 0; i < profilesCount; i++) - { - capture.set(CAP_INTELPERC_DEPTH_GENERATOR | CAP_PROP_INTELPERC_PROFILE_IDX, (double)i); - cout << " Profile[" << i << "]: "; - cout << "width = " << - (int)capture.get(CAP_INTELPERC_DEPTH_GENERATOR | CAP_PROP_FRAME_WIDTH); - cout << ", height = " << - (int)capture.get(CAP_INTELPERC_DEPTH_GENERATOR | CAP_PROP_FRAME_HEIGHT); - cout << ", fps = " << - capture.get(CAP_INTELPERC_DEPTH_GENERATOR | CAP_PROP_FPS); - cout << endl; - } -} - -static void imshowImage(const char *winname, Mat &image, VideoCapture &capture) -{ - if (g_showClosedPoint) - { - Mat uvMap; - if (capture.retrieve(uvMap, CAP_INTELPERC_UVDEPTH_MAP)) - { - float *uvmap = (float *)uvMap.ptr() + 2 * (g_closedDepthPoint[0] * uvMap.cols + g_closedDepthPoint[1]); - int x = (int)((*uvmap) * image.cols); uvmap++; - int y = (int)((*uvmap) * image.rows); - - if ((0 <= x) && (0 <= y)) - { - static const int pointSize = 4; - for (int row = y; row < min(y + pointSize, image.rows); row++) - { - uchar* ptrDst = image.ptr(row) + x * 3 + 2;//+2 -> Red - for (int col = 0; col < min(pointSize, image.cols - x); col++, ptrDst+=3) - { - *ptrDst = 255; - } - } - } - } - } - imshow(winname, image); -} -static void imshowIR(const char *winname, Mat &ir) -{ - Mat image; - if (g_showClosedPoint) - { - image.create(ir.rows, ir.cols, CV_8UC3); - for (int row = 0; row < ir.rows; row++) - { - uchar* ptrDst = image.ptr(row); - short* ptrSrc = (short*)ir.ptr(row); - for (int col = 0; col < ir.cols; col++, ptrSrc++) - { - uchar val = (uchar) ((*ptrSrc) >> 2); - *ptrDst = val; ptrDst++; - *ptrDst = val; ptrDst++; - *ptrDst = val; ptrDst++; - } - } - - static const int pointSize = 4; - for (int row = g_closedDepthPoint[0]; row < min(g_closedDepthPoint[0] + pointSize, image.rows); row++) - { - uchar* ptrDst = image.ptr(row) + g_closedDepthPoint[1] * 3 + 2;//+2 -> Red - for (int col = 0; col < min(pointSize, image.cols - g_closedDepthPoint[1]); col++, ptrDst+=3) - { - *ptrDst = 255; - } - } - } - else - { - image.create(ir.rows, ir.cols, CV_8UC1); - for (int row = 0; row < ir.rows; row++) - { - uchar* ptrDst = image.ptr(row); - short* ptrSrc = (short*)ir.ptr(row); - for (int col = 0; col < ir.cols; col++, ptrSrc++, ptrDst++) - { - *ptrDst = (uchar) ((*ptrSrc) >> 2); - } - } - } - - imshow(winname, image); -} -static void imshowDepth(const char *winname, Mat &depth, VideoCapture &capture) -{ - short lowValue = (short)capture.get(CAP_INTELPERC_DEPTH_GENERATOR | CAP_PROP_INTELPERC_DEPTH_LOW_CONFIDENCE_VALUE); - short saturationValue = (short)capture.get(CAP_INTELPERC_DEPTH_GENERATOR | CAP_PROP_INTELPERC_DEPTH_SATURATION_VALUE); - - Mat image; - if (g_showClosedPoint) - { - image.create(depth.rows, depth.cols, CV_8UC3); - for (int row = 0; row < depth.rows; row++) - { - uchar* ptrDst = image.ptr(row); - short* ptrSrc = (short*)depth.ptr(row); - for (int col = 0; col < depth.cols; col++, ptrSrc++) - { - if ((lowValue == (*ptrSrc)) || (saturationValue == (*ptrSrc))) - { - *ptrDst = 0; ptrDst++; - *ptrDst = 0; ptrDst++; - *ptrDst = 0; ptrDst++; - } - else - { - uchar val = (uchar) ((*ptrSrc) >> 2); - *ptrDst = val; ptrDst++; - *ptrDst = val; ptrDst++; - *ptrDst = val; ptrDst++; - } - } - } - - static const int pointSize = 4; - for (int row = g_closedDepthPoint[0]; row < min(g_closedDepthPoint[0] + pointSize, image.rows); row++) - { - uchar* ptrDst = image.ptr(row) + g_closedDepthPoint[1] * 3 + 2;//+2 -> Red - for (int col = 0; col < min(pointSize, image.cols - g_closedDepthPoint[1]); col++, ptrDst+=3) - { - *ptrDst = 255; - } - } - } - else - { - image.create(depth.rows, depth.cols, CV_8UC1); - for (int row = 0; row < depth.rows; row++) - { - uchar* ptrDst = image.ptr(row); - short* ptrSrc = (short*)depth.ptr(row); - for (int col = 0; col < depth.cols; col++, ptrSrc++, ptrDst++) - { - if ((lowValue == (*ptrSrc)) || (saturationValue == (*ptrSrc))) - *ptrDst = 0; - else - *ptrDst = (uchar) ((*ptrSrc) >> 2); - } - } - } - imshow(winname, image); -} - -int main(int argc, char* argv[]) -{ - parseCMDLine(argc, argv); - - VideoCapture capture; - capture.open(CAP_INTELPERC); - if (!capture.isOpened()) - { - cerr << "Can not open a capture object." << endl; - return -1; - } - - if (g_printStreamSetting) - printStreamProperties(capture); - - if (-1 != g_imageStreamProfileIdx) - { - if (!capture.set(CAP_INTELPERC_IMAGE_GENERATOR | CAP_PROP_INTELPERC_PROFILE_IDX, (double)g_imageStreamProfileIdx)) - { - cerr << "Can not setup a image stream." << endl; - return -1; - } - } - if (-1 != g_depthStreamProfileIdx) - { - if (!capture.set(CAP_INTELPERC_DEPTH_GENERATOR | CAP_PROP_INTELPERC_PROFILE_IDX, (double)g_depthStreamProfileIdx)) - { - cerr << "Can not setup a depth stream." << endl; - return -1; - } - } - else if (g_irStreamShow) - { - if (!capture.set(CAP_INTELPERC_DEPTH_GENERATOR | CAP_PROP_INTELPERC_PROFILE_IDX, 0.0)) - { - cerr << "Can not setup a IR stream." << endl; - return -1; - } - } - else - { - cout << "Streams not selected" << endl; - return 0; - } - - //Setup additional properties only after set profile of the stream - if ( (-10000.0 < g_imageBrightness) && (g_imageBrightness < 10000.0)) - capture.set(CAP_INTELPERC_IMAGE_GENERATOR | CAP_PROP_BRIGHTNESS, g_imageBrightness); - if ( (0 < g_imageContrast) && (g_imageContrast < 10000.0)) - capture.set(CAP_INTELPERC_IMAGE_GENERATOR | CAP_PROP_BRIGHTNESS, g_imageContrast); - - int frame = 0; - for(;;frame++) - { - Mat bgrImage; - Mat depthImage; - Mat irImage; - - if (!capture.grab()) - { - cout << "Can not grab images." << endl; - return -1; - } - - if ((-1 != g_depthStreamProfileIdx) && (capture.retrieve(depthImage, CAP_INTELPERC_DEPTH_MAP))) - { - if (g_showClosedPoint) - { - double minVal = 0.0; double maxVal = 0.0; - minMaxIdx(depthImage, &minVal, &maxVal, g_closedDepthPoint); - } - imshowDepth("depth image", depthImage, capture); - } - if ((g_irStreamShow) && (capture.retrieve(irImage, CAP_INTELPERC_IR_MAP))) - imshowIR("ir image", irImage); - if ((-1 != g_imageStreamProfileIdx) && (capture.retrieve(bgrImage, CAP_INTELPERC_IMAGE))) - imshowImage("color image", bgrImage, capture); - - if (g_printTiming) - { - cout << "Image frame: " << capture.get(CAP_INTELPERC_IMAGE_GENERATOR | CAP_PROP_POS_FRAMES) - << ", Depth(IR) frame: " << capture.get(CAP_INTELPERC_DEPTH_GENERATOR | CAP_PROP_POS_FRAMES) << endl; - cout << "Image frame: " << capture.get(CAP_INTELPERC_IMAGE_GENERATOR | CAP_PROP_POS_MSEC) - << ", Depth(IR) frame: " << capture.get(CAP_INTELPERC_DEPTH_GENERATOR | CAP_PROP_POS_MSEC) << endl; - } - if( waitKey(30) >= 0 ) - break; - } - - return 0; -} diff --git a/samples/cpp/videocapture_realsense.cpp b/samples/cpp/videocapture_realsense.cpp new file mode 100644 index 0000000000..352353eb2b --- /dev/null +++ b/samples/cpp/videocapture_realsense.cpp @@ -0,0 +1,33 @@ +#include "opencv2/videoio.hpp" +#include "opencv2/highgui.hpp" +#include "opencv2/imgproc.hpp" + +using namespace cv; +using namespace std; + +int main() +{ + VideoCapture capture(CAP_INTELPERC); + for(;;) + { + Mat depthMap; + Mat image; + Mat irImage; + Mat adjMap; + + capture.grab(); + capture.retrieve(depthMap,CAP_INTELPERC_DEPTH_MAP); + capture.retrieve(image,CAP_INTELPERC_IMAGE); + capture.retrieve(irImage,CAP_INTELPERC_IR_MAP); + + normalize(depthMap, adjMap, 0, 255, NORM_MINMAX, CV_8UC1); + applyColorMap(adjMap, adjMap, COLORMAP_JET); + + imshow("RGB", image); + imshow("IR", irImage); + imshow("DEPTH", adjMap); + if( waitKey( 30 ) >= 0 ) + break; + } + return 0; +} From db4ab1c936d8dd1046c28741137510aab8f6537e Mon Sep 17 00:00:00 2001 From: Maxim Milashchenko <67949029+MaximMilashchenko@users.noreply.github.com> Date: Thu, 16 Dec 2021 22:43:02 +0300 Subject: [PATCH 179/226] Merge pull request #21264 from MaximMilashchenko:AudioGStreamer Audio GStreamer: added support .wav .flac audio formats * added support .wav, lossless compressed audio formats * fixed docs * fixes * videoio(gstreamer-audio): extra tests, improve error handling Co-authored-by: Alexander Alekhin --- modules/videoio/src/cap_gstreamer.cpp | 529 +++++++++++++++++++++----- modules/videoio/test/test_audio.cpp | 76 +++- 2 files changed, 485 insertions(+), 120 deletions(-) diff --git a/modules/videoio/src/cap_gstreamer.cpp b/modules/videoio/src/cap_gstreamer.cpp index dd91bf1db4..adbc94fc95 100644 --- a/modules/videoio/src/cap_gstreamer.cpp +++ b/modules/videoio/src/cap_gstreamer.cpp @@ -59,6 +59,7 @@ #include #include #include +#include #include #include #include @@ -308,6 +309,8 @@ private: GSafePtr sample; GSafePtr caps; + gint videoStream; + gint audioStream; gint64 duration; gint width; gint height; @@ -315,6 +318,12 @@ private: bool isPosFramesSupported; bool isPosFramesEmulated; gint64 emulatedFrameNumber; + gint outputAudioFormat; + gint audioBaseIndex; + gint nAudioChannels; + gint audioSamplesPerSecond; + + Mat audioFrame; VideoAccelerationType va_type; int hw_device; @@ -323,6 +332,9 @@ public: virtual ~GStreamerCapture() CV_OVERRIDE; virtual bool grabFrame() CV_OVERRIDE; virtual bool retrieveFrame(int /*unused*/, OutputArray dst) CV_OVERRIDE; + bool grabAudioFrame(); + bool retrieveVideoFrame(int /*unused*/, OutputArray dst); + bool retrieveAudioFrame(int /*unused*/, OutputArray dst); virtual double getProperty(int propId) const CV_OVERRIDE; virtual bool setProperty(int propId, double value) CV_OVERRIDE; virtual bool isOpened() const CV_OVERRIDE { return (bool)pipeline; } @@ -330,6 +342,9 @@ public: bool open(int id, const cv::VideoCaptureParameters& params); bool open(const String &filename_, const cv::VideoCaptureParameters& params); static void newPad(GstElement * /*elem*/, GstPad *pad, gpointer data); + bool configureHW(const cv::VideoCaptureParameters&); + bool configureStreams(const cv::VideoCaptureParameters&); + bool setAudioProperties(const cv::VideoCaptureParameters&); protected: bool isPipelinePlaying(); @@ -341,10 +356,16 @@ protected: }; GStreamerCapture::GStreamerCapture() : + videoStream(0), + audioStream(-1), duration(-1), width(-1), height(-1), fps(-1), isPosFramesSupported(false), isPosFramesEmulated(false), - emulatedFrameNumber(-1) + emulatedFrameNumber(-1), + outputAudioFormat(CV_16S), + audioBaseIndex(1), + nAudioChannels(0), + audioSamplesPerSecond(44100) , va_type(VIDEO_ACCELERATION_NONE) , hw_device(-1) { @@ -365,6 +386,92 @@ GStreamerCapture::~GStreamerCapture() } } +bool GStreamerCapture::configureHW(const cv::VideoCaptureParameters& params) +{ + if (params.has(CAP_PROP_HW_ACCELERATION)) + { + va_type = params.get(CAP_PROP_HW_ACCELERATION); + } + if (params.has(CAP_PROP_HW_DEVICE)) + { + hw_device = params.get(CAP_PROP_HW_DEVICE); + if (va_type == VIDEO_ACCELERATION_NONE && hw_device != -1) + { + CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: Invalid usage of CAP_PROP_HW_DEVICE without requested H/W acceleration. Bailout"); + return false; + } + if (va_type == VIDEO_ACCELERATION_ANY && hw_device != -1) + { + CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: Invalid usage of CAP_PROP_HW_DEVICE with 'ANY' H/W acceleration. Bailout"); + return false; + } + if (hw_device != -1) + { + CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: CAP_PROP_HW_DEVICE is not supported. Specify -1 (auto) value. Bailout"); + return false; + } + } + return true; +} + +bool GStreamerCapture::configureStreams(const cv::VideoCaptureParameters& params) +{ + if (params.has(CAP_PROP_VIDEO_STREAM)) + { + double value = params.get(CAP_PROP_VIDEO_STREAM); + if (value == -1 || value == 0) + videoStream = static_cast(value); + else + { + CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: CAP_PROP_VIDEO_STREAM parameter value is invalid/unsupported: " << value); + return false; + } + } + if (params.has(CAP_PROP_AUDIO_STREAM)) + { + double value = params.get(CAP_PROP_AUDIO_STREAM); + if (value == -1 || value > -1) + audioStream = static_cast(value); + else + { + CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: CAP_PROP_AUDIO_STREAM parameter value is invalid/unsupported: " << value); + return false; + } + } + return true; +} + +bool GStreamerCapture::setAudioProperties(const cv::VideoCaptureParameters& params) +{ + if (params.has(CAP_PROP_AUDIO_DATA_DEPTH)) + { + gint value = static_cast(params.get(CAP_PROP_AUDIO_DATA_DEPTH)); + if (value != CV_8S && value != CV_16S && value != CV_32S && value != CV_32F) + { + CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: CAP_PROP_AUDIO_DATA_DEPTH parameter value is invalid/unsupported: " << value); + return false; + } + else + { + outputAudioFormat = value; + } + } + if (params.has(CAP_PROP_AUDIO_SAMPLES_PER_SECOND)) + { + int value = static_cast(params.get(CAP_PROP_AUDIO_SAMPLES_PER_SECOND)); + if (value < 0) + { + CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: CAP_PROP_AUDIO_SAMPLES_PER_SECOND parameter can't be negative: " << value); + return false; + } + else + { + audioSamplesPerSecond = value; + } + } + return true; +} + /*! * \brief CvCapture_GStreamer::grabFrame * \return @@ -391,21 +498,137 @@ bool GStreamerCapture::grabFrame() if (isPosFramesEmulated) emulatedFrameNumber++; + if (audioStream >= 0) + return grabAudioFrame(); + return true; } -/*! - * \brief CvCapture_GStreamer::retrieveFrame - * \return IplImage pointer. [Transfer Full] - * Retrieve the previously grabbed buffer, and wrap it in an IPLImage structure - */ -bool GStreamerCapture::retrieveFrame(int, OutputArray dst) +bool GStreamerCapture::grabAudioFrame() { - if (!sample) + GstCaps* frame_caps = gst_sample_get_caps(sample); // no lifetime transfer + if (!frame_caps) { + CV_LOG_ERROR(NULL, "GStreamer: gst_sample_get_caps() returns NULL"); return false; } + if (!GST_CAPS_IS_SIMPLE(frame_caps)) + { + // bail out in no caps + CV_LOG_ERROR(NULL, "GStreamer: GST_CAPS_IS_SIMPLE(frame_caps) check is failed"); + return false; + } + + GstAudioInfo info = {}; + gboolean audio_info_res = gst_audio_info_from_caps(&info, frame_caps); + if (!audio_info_res) + { + CV_Error(Error::StsError, "GStreamer: gst_audio_info_from_caps() is failed. Can't handle unknown layout"); + } + int bpf = GST_AUDIO_INFO_BPF(&info); + + GstStructure* structure = gst_caps_get_structure(frame_caps, 0); // no lifetime transfer + if (!structure) + { + CV_LOG_ERROR(NULL, "GStreamer: Can't query 'structure'-0 from GStreamer sample"); + return false; + } + + const gchar* name_ = gst_structure_get_name(structure); + if (!name_) + { + CV_LOG_ERROR(NULL, "GStreamer: Can't query 'name' from GStreamer sample"); + return false; + } + std::string name = toLowerCase(std::string(name_)); + + GstBuffer* buf = gst_sample_get_buffer(sample); + if (!buf) + return false; + GstMapInfo map_info = {}; + if (!gst_buffer_map(buf, &map_info, GST_MAP_READ)) + { + CV_LOG_ERROR(NULL, "GStreamer: Failed to map GStreamer buffer to system memory"); + return false; + } + ScopeGuardGstMapInfo map_guard(buf, &map_info); + if (name == "audio/x-raw") + { + const gchar* format_ = gst_structure_get_string(structure, "format"); + if (!format_) + { + CV_LOG_ERROR(NULL, "GStreamer: Can't query 'format' of 'video/x-raw'"); + return false; + } + std::string format = toUpperCase(std::string(format_)); + cv::Mat data; + if (format == "S8") + { + Mat(map_info.size/bpf, nAudioChannels, CV_8S, map_info.data).copyTo(audioFrame); + return true; + } + if (format == "S16LE") + { + Mat(map_info.size/bpf, nAudioChannels, CV_16S, map_info.data).copyTo(audioFrame); + return true; + } + if (format == "S32LE") + { + Mat(map_info.size/bpf, nAudioChannels, CV_32S, map_info.data).copyTo(audioFrame); + return true; + } + if (format == "F32LE") + { + Mat(map_info.size/bpf, nAudioChannels, CV_32F, map_info.data).copyTo(audioFrame); + return true; + } + CV_Error_(Error::StsNotImplemented, ("Unsupported GStreamer audio format: %s", format.c_str())); + } + + CV_Error_(Error::StsNotImplemented, ("Unsupported GStreamer layer type: %s", name.c_str())); +} + +bool GStreamerCapture::retrieveAudioFrame(int index, OutputArray dst) +{ + CV_Check(index, index >= audioBaseIndex && index < audioBaseIndex + nAudioChannels, ""); + index -= audioBaseIndex; + + CV_CheckType(outputAudioFormat, + outputAudioFormat == CV_8S || + outputAudioFormat == CV_16S || + outputAudioFormat == CV_32S || + outputAudioFormat == CV_32F, + ""); + + dst.create(1, audioFrame.rows, outputAudioFormat); + Mat data = dst.getMat(); + switch (outputAudioFormat) + { + case CV_8S: + for (int i = 0; i < audioFrame.rows; i++) + data.at(i) = audioFrame.at(i, index); + return true; + case CV_16S: + for (int i = 0; i < audioFrame.rows; i++) + data.at(i) = audioFrame.at(i, index); + return true; + case CV_32S: + for (int i = 0; i < audioFrame.rows; i++) + data.at(i) = audioFrame.at(i, index); + return true; + case CV_32F: + for (int i = 0; i < audioFrame.rows; i++) + data.at(i) = audioFrame.at(i, index); + return true; + } + + dst.release(); + return false; +} + +bool GStreamerCapture::retrieveVideoFrame(int, OutputArray dst) +{ GstCaps* frame_caps = gst_sample_get_caps(sample); // no lifetime transfer if (!frame_caps) { @@ -609,6 +832,28 @@ bool GStreamerCapture::retrieveFrame(int, OutputArray dst) CV_Error_(Error::StsNotImplemented, ("Unsupported GStreamer layer type: %s", name.c_str())); } +bool GStreamerCapture::retrieveFrame(int index, OutputArray dst) +{ + if (index < 0) + return false; + if (!sample) + return false; + + if (index == 0) + { + CV_CheckGE(videoStream, 0, "No video stream configured"); + return retrieveVideoFrame(index, dst); + } + else if (index >= audioBaseIndex) + { + CV_CheckGE(audioStream, 0, "No audio stream configured"); + return retrieveAudioFrame(index, dst); + } + + CV_LOG_ERROR(NULL, "GStreamer(retrive): unrecognized index=" << index); + return false; +} + bool GStreamerCapture::isPipelinePlaying() { if (!pipeline || !GST_IS_ELEMENT(pipeline.get())) @@ -810,28 +1055,33 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam { gst_initializer::init(); - if (params.has(CAP_PROP_HW_ACCELERATION)) + if (!configureHW(params)) { - va_type = params.get(CAP_PROP_HW_ACCELERATION); + CV_LOG_WARNING(NULL, "GStreamer: can't configure HW encoding/decoding support"); + return false; } - if (params.has(CAP_PROP_HW_DEVICE)) + + if (!configureStreams(params)) { - hw_device = params.get(CAP_PROP_HW_DEVICE); - if (va_type == VIDEO_ACCELERATION_NONE && hw_device != -1) - { - CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: Invalid usage of CAP_PROP_HW_DEVICE without requested H/W acceleration. Bailout"); - return false; - } - if (va_type == VIDEO_ACCELERATION_ANY && hw_device != -1) - { - CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: Invalid usage of CAP_PROP_HW_DEVICE with 'ANY' H/W acceleration. Bailout"); - return false; - } - if (hw_device != -1) - { - CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: CAP_PROP_HW_DEVICE is not supported. Specify -1 (auto) value. Bailout"); - return false; - } + CV_LOG_WARNING(NULL, "GStreamer: can't configure streams"); + return false; + } + + if ((videoStream >= 0 && audioStream >= 0) || (videoStream < 0 && audioStream < 0)) + { + CV_LOG_ERROR(NULL, "GStreamer backend supports audio-only or video-only capturing. Only one of the properties CAP_PROP_AUDIO_STREAM=" << audioStream << " and CAP_PROP_VIDEO_STREAM=" << videoStream << " should be >= 0"); + return false; + } + if (audioStream > 0) + { + CV_LOG_ERROR(NULL, "GStreamer backend supports the first audio stream only. CAP_PROP_AUDIO_STREAM=" << audioStream); + return false; + } + + if (!setAudioProperties(params)) + { + CV_LOG_WARNING(NULL, "GStreamer: can't configure audio properties"); + return false; } const gchar* filename = filename_.c_str(); @@ -843,6 +1093,9 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam GSafePtr color; GstStateChangeReturn status; + GSafePtr convert; + GSafePtr resample; + // test if we have a valid uri. If so, open it with an uridecodebin // else, we might have a file or a manual pipeline. // if gstreamer cannot parse the manual pipeline, we assume we were given and @@ -884,20 +1137,29 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam bool element_from_uri = false; if (!uridecodebin) { - // At this writing, the v4l2 element (and maybe others too) does not support caps renegotiation. - // This means that we cannot use an uridecodebin when dealing with v4l2, since setting - // capture properties will not work. - // The solution (probably only until gstreamer 1.2) is to make an element from uri when dealing with v4l2. - GSafePtr protocol_; protocol_.attach(gst_uri_get_protocol(uri)); - CV_Assert(protocol_); - std::string protocol = toLowerCase(std::string(protocol_.get())); - if (protocol == "v4l2") + if (videoStream >= 0) { - uridecodebin.reset(gst_element_make_from_uri(GST_URI_SRC, uri.get(), "src", NULL)); - CV_Assert(uridecodebin); - element_from_uri = true; + // At this writing, the v4l2 element (and maybe others too) does not support caps renegotiation. + // This means that we cannot use an uridecodebin when dealing with v4l2, since setting + // capture properties will not work. + // The solution (probably only until gstreamer 1.2) is to make an element from uri when dealing with v4l2. + GSafePtr protocol_; protocol_.attach(gst_uri_get_protocol(uri)); + CV_Assert(protocol_); + std::string protocol = toLowerCase(std::string(protocol_.get())); + if (protocol == "v4l2") + { + uridecodebin.reset(gst_element_make_from_uri(GST_URI_SRC, uri.get(), "src", NULL)); + CV_Assert(uridecodebin); + element_from_uri = true; + } + else + { + uridecodebin.reset(gst_element_factory_make("uridecodebin", NULL)); + CV_Assert(uridecodebin); + g_object_set(G_OBJECT(uridecodebin.get()), "uri", uri.get(), NULL); + } } - else + else if (audioStream >= 0) { uridecodebin.reset(gst_element_factory_make("uridecodebin", NULL)); CV_Assert(uridecodebin); @@ -971,35 +1233,51 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam pipeline.reset(gst_pipeline_new(NULL)); CV_Assert(pipeline); - // videoconvert (in 0.10: ffmpegcolorspace, in 1.x autovideoconvert) - //automatically selects the correct colorspace conversion based on caps. - color.reset(gst_element_factory_make(COLOR_ELEM, NULL)); - CV_Assert(color); - sink.reset(gst_element_factory_make("appsink", NULL)); CV_Assert(sink); - - gst_bin_add_many(GST_BIN(pipeline.get()), uridecodebin.get(), color.get(), sink.get(), NULL); - - if (element_from_uri) + if (videoStream >= 0) { - if(!gst_element_link(uridecodebin, color.get())) + // videoconvert (in 0.10: ffmpegcolorspace, in 1.x autovideoconvert) + //automatically selects the correct colorspace conversion based on caps. + color.reset(gst_element_factory_make(COLOR_ELEM, NULL)); + CV_Assert(color); + + gst_bin_add_many(GST_BIN(pipeline.get()), uridecodebin.get(), color.get(), sink.get(), NULL); + + if (element_from_uri) { - CV_WARN("cannot link color -> sink"); + if(!gst_element_link(uridecodebin, color.get())) + { + CV_WARN("GStreamer(video): cannot link color -> sink"); + pipeline.release(); + return false; + } + } + else + { + g_signal_connect(uridecodebin, "pad-added", G_CALLBACK(newPad), color.get()); + } + + if (!gst_element_link(color.get(), sink.get())) + { + CV_WARN("GStreamer(video): cannot link color -> sink"); pipeline.release(); return false; } } - else + else if (audioStream >= 0) { - g_signal_connect(uridecodebin, "pad-added", G_CALLBACK(newPad), color.get()); - } + convert.reset(gst_element_factory_make("audioconvert", NULL)); + resample.reset(gst_element_factory_make("audioresample", NULL)); - if (!gst_element_link(color.get(), sink.get())) - { - CV_WARN("GStreamer: cannot link color -> sink"); - pipeline.release(); - return false; + gst_bin_add_many (GST_BIN (pipeline.get()), uridecodebin.get(), convert.get(), resample.get(), sink.get(), NULL); + if (!gst_element_link_many (convert.get(), resample.get(), sink.get(), NULL)) + { + CV_WARN("GStreamer(audio): cannot link convert -> resample -> sink"); + pipeline.release(); + return false; + } + g_signal_connect (uridecodebin, "pad-added", G_CALLBACK (newPad), convert.get()); } } @@ -1008,17 +1286,41 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam //TODO: is 1 single buffer really high enough? gst_app_sink_set_max_buffers(GST_APP_SINK(sink.get()), 1); } - if (!manualpipeline) { gst_base_sink_set_sync(GST_BASE_SINK(sink.get()), FALSE); } - //do not emit signals: all calls will be synchronous and blocking gst_app_sink_set_emit_signals (GST_APP_SINK(sink.get()), FALSE); - - caps.attach(gst_caps_from_string("video/x-raw, format=(string){BGR, GRAY8}; video/x-bayer,format=(string){rggb,bggr,grbg,gbrg}; image/jpeg")); + if (videoStream >= 0) + { + caps.attach(gst_caps_from_string("video/x-raw, format=(string){BGR, GRAY8}; video/x-bayer,format=(string){rggb,bggr,grbg,gbrg}; image/jpeg")); + } + else if (audioStream >= 0) + { + std::string audioFormat; + switch (outputAudioFormat) + { + case CV_8S: + audioFormat = "S8"; + break; + case CV_16S: + audioFormat = "S16LE"; + break; + case CV_32S: + audioFormat = "S32LE"; + break; + case CV_32F: + audioFormat = "F32LE"; + break; + default: + audioFormat = "S16LE"; + break; + } + std::string stringCaps = "audio/x-raw, format=(string)" + audioFormat + ", rate=(int)" + std::to_string(audioSamplesPerSecond) + ", channels=(int){1, 2}, layout=(string)interleaved"; + caps.attach(gst_caps_from_string(stringCaps.c_str())); + } if (manualpipeline) { @@ -1054,58 +1356,73 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam return false; } - GstFormat format; - - format = GST_FORMAT_DEFAULT; - if(!gst_element_query_duration(sink, format, &duration)) - { - handleMessage(pipeline); - CV_WARN("unable to query duration of stream"); - duration = -1; - } - - handleMessage(pipeline); - GSafePtr pad; pad.attach(gst_element_get_static_pad(sink, "sink")); GSafePtr buffer_caps; buffer_caps.attach(gst_pad_get_current_caps(pad)); - const GstStructure *structure = gst_caps_get_structure(buffer_caps, 0); // no lifetime transfer - if (!gst_structure_get_int (structure, "width", &width) || - !gst_structure_get_int (structure, "height", &height)) + if (videoStream >= 0) { - CV_WARN("cannot query video width/height"); - } + GstFormat format; - gint num = 0, denom=1; - if (!gst_structure_get_fraction(structure, "framerate", &num, &denom)) - { - CV_WARN("cannot query video fps"); - } - - fps = (double)num/(double)denom; - - { - GstFormat format_; - gint64 value_ = -1; - gboolean status_; - - format_ = GST_FORMAT_DEFAULT; - - status_ = gst_element_query_position(sink, CV_GST_FORMAT(format_), &value_); - if (!status_ || value_ != 0 || duration < 0) + format = GST_FORMAT_DEFAULT; + if(!gst_element_query_duration(sink, format, &duration)) { - CV_WARN("Cannot query video position: status=" << status_ << ", value=" << value_ << ", duration=" << duration); - isPosFramesSupported = false; - isPosFramesEmulated = true; - emulatedFrameNumber = 0; + handleMessage(pipeline); + CV_WARN("unable to query duration of stream"); + duration = -1; + } + + handleMessage(pipeline); + + const GstStructure *structure = gst_caps_get_structure(buffer_caps, 0); // no lifetime transfer + if (!gst_structure_get_int (structure, "width", &width) || + !gst_structure_get_int (structure, "height", &height)) + { + CV_WARN("cannot query video width/height"); + } + + gint num = 0, denom=1; + if (!gst_structure_get_fraction(structure, "framerate", &num, &denom)) + { + CV_WARN("cannot query video fps"); + } + + fps = (double)num/(double)denom; + + { + GstFormat format_; + gint64 value_ = -1; + gboolean status_; + + format_ = GST_FORMAT_DEFAULT; + + status_ = gst_element_query_position(sink, CV_GST_FORMAT(format_), &value_); + if (!status_ || value_ != 0 || duration < 0) + { + CV_WARN("Cannot query video position: status=" << status_ << ", value=" << value_ << ", duration=" << duration); + isPosFramesSupported = false; + isPosFramesEmulated = true; + emulatedFrameNumber = 0; + } + else + isPosFramesSupported = true; + } + } + else if (audioStream >= 0) + { + GstAudioInfo info = {}; + if (gst_audio_info_from_caps(&info, buffer_caps)) + { + nAudioChannels = GST_AUDIO_INFO_CHANNELS(&info); + audioSamplesPerSecond = GST_AUDIO_INFO_RATE(&info); } else - isPosFramesSupported = true; + { + CV_WARN("cannot query audio nChannels and SamplesPerSecond"); + } } - GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(pipeline.get()), GST_DEBUG_GRAPH_SHOW_ALL, "pipeline"); } @@ -1232,6 +1549,14 @@ double GStreamerCapture::getProperty(int propId) const return 0; } return gst_app_sink_get_max_buffers(GST_APP_SINK(sink.get())); + case CAP_PROP_AUDIO_TOTAL_CHANNELS: + return nAudioChannels; + case CAP_PROP_AUDIO_SAMPLES_PER_SECOND: + return audioSamplesPerSecond; + case CAP_PROP_AUDIO_DATA_DEPTH: + return outputAudioFormat; + case CAP_PROP_AUDIO_BASE_INDEX: + return audioBaseIndex; default: CV_WARN("unhandled property: " << propId); break; diff --git a/modules/videoio/test/test_audio.cpp b/modules/videoio/test/test_audio.cpp index 7c66b83e34..b1eb0ed4b7 100644 --- a/modules/videoio/test/test_audio.cpp +++ b/modules/videoio/test/test_audio.cpp @@ -92,7 +92,7 @@ public: { for (int nCh = 0; nCh < numberOfChannels; nCh++) { - ASSERT_TRUE(cap.retrieve(audioFrame, audioBaseIndex)); + ASSERT_TRUE(cap.retrieve(audioFrame, audioBaseIndex + nCh)); ASSERT_EQ(CV_16SC1, audioFrame.type()) << audioData[nCh].size(); for (int i = 0; i < audioFrame.cols; i++) { @@ -111,10 +111,14 @@ public: const param audioParams[] = { +#ifdef _WIN32 param("test_audio.wav", 1, 132300, 0.0001, cv::CAP_MSMF), param("test_mono_audio.mp3", 1, 133104, 0.12, cv::CAP_MSMF), param("test_stereo_audio.mp3", 2, 133104, 0.12, cv::CAP_MSMF), - param("test_audio.mp4", 1, 133104, 0.15, cv::CAP_MSMF) + param("test_audio.mp4", 1, 133104, 0.15, cv::CAP_MSMF), +#endif + param("test_audio.wav", 1, 132300, 0.0001, cv::CAP_GSTREAMER), + param("test_audio.mp4", 1, 132522, 0.15, cv::CAP_GSTREAMER), }; class Audio : public AudioTestFixture{}; @@ -249,15 +253,6 @@ protected: const double psnrThreshold; }; -const paramCombination mediaParams[] = -{ - paramCombination("test_audio.mp4", 1, 0.15, CV_8UC3, 240, 320, 90, 131819, 30, 30., cv::CAP_MSMF) -#if 0 - // https://filesamples.com/samples/video/mp4/sample_960x400_ocean_with_audio.mp4 - , paramCombination("sample_960x400_ocean_with_audio.mp4", 2, -1/*eplsilon*/, CV_8UC3, 400, 960, 1116, 2056588, 30, 30., cv::CAP_MSMF) -#endif -}; - class Media : public MediaTestFixture{}; TEST_P(Media, audio) @@ -268,27 +263,72 @@ TEST_P(Media, audio) doTest(); } +#ifdef _WIN32 +const paramCombination mediaParams[] = +{ +#ifdef _WIN32 + paramCombination("test_audio.mp4", 1, 0.15, CV_8UC3, 240, 320, 90, 131819, 30, 30., cv::CAP_MSMF) +#if 0 + // https://filesamples.com/samples/video/mp4/sample_960x400_ocean_with_audio.mp4 + , paramCombination("sample_960x400_ocean_with_audio.mp4", 2, -1/*eplsilon*/, CV_8UC3, 400, 960, 1116, 2056588, 30, 30., cv::CAP_MSMF) +#endif +#endif // _WIN32 +}; + INSTANTIATE_TEST_CASE_P(/**/, Media, testing::ValuesIn(mediaParams)); +#endif // _WIN32 TEST(AudioOpenCheck, bad_arg_invalid_audio_stream) +{ + std::string fileName = "audio/test_audio.wav"; + std::vector params { + CAP_PROP_AUDIO_STREAM, 1, + CAP_PROP_VIDEO_STREAM, -1, // disabled + CAP_PROP_AUDIO_DATA_DEPTH, CV_16S + }; + VideoCapture cap; + cap.open(findDataFile(fileName), cv::CAP_ANY, params); + ASSERT_FALSE(cap.isOpened()); +} + +TEST(AudioOpenCheck, bad_arg_invalid_audio_stream_video) { std::string fileName = "audio/test_audio.mp4"; - std::vector params { CAP_PROP_AUDIO_STREAM, 1, - CAP_PROP_VIDEO_STREAM, 0, - CAP_PROP_AUDIO_DATA_DEPTH, CV_16S }; + std::vector params { + CAP_PROP_AUDIO_STREAM, 1, + CAP_PROP_VIDEO_STREAM, 0, + CAP_PROP_AUDIO_DATA_DEPTH, CV_16S + }; + VideoCapture cap; + cap.open(findDataFile(fileName), cv::CAP_ANY, params); + ASSERT_FALSE(cap.isOpened()); +} + +#ifdef _WIN32 +TEST(AudioOpenCheck, MSMF_bad_arg_invalid_audio_sample_per_second) +{ + std::string fileName = "audio/test_audio.mp4"; + std::vector params { + CAP_PROP_AUDIO_STREAM, 0, + CAP_PROP_VIDEO_STREAM, -1, // disabled + CAP_PROP_AUDIO_SAMPLES_PER_SECOND, (int)1e9 + }; VideoCapture cap; cap.open(findDataFile(fileName), cv::CAP_MSMF, params); ASSERT_FALSE(cap.isOpened()); } +#endif TEST(AudioOpenCheck, bad_arg_invalid_audio_sample_per_second) { std::string fileName = "audio/test_audio.mp4"; - std::vector params { CAP_PROP_AUDIO_STREAM, 0, - CAP_PROP_VIDEO_STREAM, -1, - CAP_PROP_AUDIO_SAMPLES_PER_SECOND, (int)1e9 }; + std::vector params { + CAP_PROP_AUDIO_STREAM, 0, + CAP_PROP_VIDEO_STREAM, -1, // disabled + CAP_PROP_AUDIO_SAMPLES_PER_SECOND, -1000 + }; VideoCapture cap; - cap.open(findDataFile(fileName), cv::CAP_MSMF, params); + cap.open(findDataFile(fileName), cv::CAP_ANY, params); ASSERT_FALSE(cap.isOpened()); } From 60c093f086893f765c2a8b3cbaa64f23f41300ee Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 17 Dec 2021 10:05:52 +0000 Subject: [PATCH 180/226] pre: OpenCV 3.4.17 (version++) --- .../cross_referencing/tutorial_cross_referencing.markdown | 4 ++-- modules/core/include/opencv2/core/version.hpp | 4 ++-- modules/dnn/include/opencv2/dnn/dnn.hpp | 4 ++-- modules/python/package/setup.py | 2 +- platforms/android/build_sdk.py | 2 +- platforms/android/service/readme.txt | 2 +- platforms/maven/opencv-it/pom.xml | 2 +- platforms/maven/opencv/pom.xml | 2 +- platforms/maven/pom.xml | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown b/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown index 27dfc5588a..3e4f222ff6 100644 --- a/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown +++ b/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown @@ -39,14 +39,14 @@ Open your Doxyfile using your favorite text editor and search for the key `TAGFILES`. Change it as follows: @code -TAGFILES = ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/3.4.16 +TAGFILES = ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/3.4.17 @endcode If you had other definitions already, you can append the line using a `\`: @code TAGFILES = ./docs/doxygen-tags/libstdc++.tag=https://gcc.gnu.org/onlinedocs/libstdc++/latest-doxygen \ - ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/3.4.16 + ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/3.4.17 @endcode Doxygen can now use the information from the tag file to link to the OpenCV diff --git a/modules/core/include/opencv2/core/version.hpp b/modules/core/include/opencv2/core/version.hpp index 0584b2001b..b62d6ed00d 100644 --- a/modules/core/include/opencv2/core/version.hpp +++ b/modules/core/include/opencv2/core/version.hpp @@ -7,8 +7,8 @@ #define CV_VERSION_MAJOR 3 #define CV_VERSION_MINOR 4 -#define CV_VERSION_REVISION 16 -#define CV_VERSION_STATUS "-dev" +#define CV_VERSION_REVISION 17 +#define CV_VERSION_STATUS "-pre" #define CVAUX_STR_EXP(__A) #__A #define CVAUX_STR(__A) CVAUX_STR_EXP(__A) diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index a2a48b7f3e..34afc4147f 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -47,9 +47,9 @@ #include "opencv2/core/async.hpp" #if !defined CV_DOXYGEN && !defined CV_STATIC_ANALYSIS && !defined CV_DNN_DONT_ADD_EXPERIMENTAL_NS -#define CV__DNN_EXPERIMENTAL_NS_BEGIN namespace experimental_dnn_34_v23 { +#define CV__DNN_EXPERIMENTAL_NS_BEGIN namespace experimental_dnn_34_v24 { #define CV__DNN_EXPERIMENTAL_NS_END } -namespace cv { namespace dnn { namespace experimental_dnn_34_v23 { } using namespace experimental_dnn_34_v23; }} +namespace cv { namespace dnn { namespace experimental_dnn_34_v24 { } using namespace experimental_dnn_34_v24; }} #else #define CV__DNN_EXPERIMENTAL_NS_BEGIN #define CV__DNN_EXPERIMENTAL_NS_END diff --git a/modules/python/package/setup.py b/modules/python/package/setup.py index 47a0629dc7..bd69de8fde 100644 --- a/modules/python/package/setup.py +++ b/modules/python/package/setup.py @@ -9,7 +9,7 @@ def main(): os.chdir(SCRIPT_DIR) package_name = 'opencv' - package_version = os.environ.get('OPENCV_VERSION', '3.4.16') # TODO + package_version = os.environ.get('OPENCV_VERSION', '3.4.17') # TODO long_description = 'Open Source Computer Vision Library Python bindings' # TODO diff --git a/platforms/android/build_sdk.py b/platforms/android/build_sdk.py index 54d6f8fb4a..75fc680615 100755 --- a/platforms/android/build_sdk.py +++ b/platforms/android/build_sdk.py @@ -269,7 +269,7 @@ class Builder: # Add extra data apkxmldest = check_dir(os.path.join(apkdest, "res", "xml"), create=True) apklibdest = check_dir(os.path.join(apkdest, "libs", abi.name), create=True) - for ver, d in self.extra_packs + [("3.4.16", os.path.join(self.libdest, "lib"))]: + for ver, d in self.extra_packs + [("3.4.17", os.path.join(self.libdest, "lib"))]: r = ET.Element("library", attrib={"version": ver}) log.info("Adding libraries from %s", d) diff --git a/platforms/android/service/readme.txt b/platforms/android/service/readme.txt index c3d3244a18..7abc897b03 100644 --- a/platforms/android/service/readme.txt +++ b/platforms/android/service/readme.txt @@ -12,7 +12,7 @@ manually using adb tool: adb install /apk/OpenCV__Manager__.apk -Example: OpenCV_3.4.16-dev_Manager_3.49_armeabi-v7a.apk +Example: OpenCV_3.4.17-dev_Manager_3.49_armeabi-v7a.apk Use the list of platforms below to determine proper OpenCV Manager package for your device: diff --git a/platforms/maven/opencv-it/pom.xml b/platforms/maven/opencv-it/pom.xml index 039f951160..ae9b61da4c 100644 --- a/platforms/maven/opencv-it/pom.xml +++ b/platforms/maven/opencv-it/pom.xml @@ -4,7 +4,7 @@ org.opencv opencv-parent - 3.4.16 + 3.4.17 org.opencv opencv-it diff --git a/platforms/maven/opencv/pom.xml b/platforms/maven/opencv/pom.xml index 64f3f5656f..a974019d64 100644 --- a/platforms/maven/opencv/pom.xml +++ b/platforms/maven/opencv/pom.xml @@ -4,7 +4,7 @@ org.opencv opencv-parent - 3.4.16 + 3.4.17 org.opencv opencv diff --git a/platforms/maven/pom.xml b/platforms/maven/pom.xml index f3fadc106f..ac554de429 100644 --- a/platforms/maven/pom.xml +++ b/platforms/maven/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.opencv opencv-parent - 3.4.16 + 3.4.17 pom OpenCV Parent POM From 07dca8cc03582309cfaad906cf1c20ff1469211a Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 17 Dec 2021 10:12:11 +0000 Subject: [PATCH 181/226] pre: OpenCV 4.5.5 (version++) --- .../cross_referencing/tutorial_cross_referencing.markdown | 4 ++-- modules/core/include/opencv2/core/version.hpp | 4 ++-- modules/dnn/include/opencv2/dnn/version.hpp | 2 +- modules/python/package/setup.py | 2 +- platforms/maven/opencv-it/pom.xml | 2 +- platforms/maven/opencv/pom.xml | 2 +- platforms/maven/pom.xml | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown b/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown index da3047aa87..dfd244fbed 100644 --- a/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown +++ b/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown @@ -46,14 +46,14 @@ Open your Doxyfile using your favorite text editor and search for the key `TAGFILES`. Change it as follows: @code -TAGFILES = ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/4.5.4 +TAGFILES = ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/4.5.5 @endcode If you had other definitions already, you can append the line using a `\`: @code TAGFILES = ./docs/doxygen-tags/libstdc++.tag=https://gcc.gnu.org/onlinedocs/libstdc++/latest-doxygen \ - ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/4.5.4 + ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/4.5.5 @endcode Doxygen can now use the information from the tag file to link to the OpenCV diff --git a/modules/core/include/opencv2/core/version.hpp b/modules/core/include/opencv2/core/version.hpp index 5c69c612d6..d3dde3414c 100644 --- a/modules/core/include/opencv2/core/version.hpp +++ b/modules/core/include/opencv2/core/version.hpp @@ -7,8 +7,8 @@ #define CV_VERSION_MAJOR 4 #define CV_VERSION_MINOR 5 -#define CV_VERSION_REVISION 4 -#define CV_VERSION_STATUS "-dev" +#define CV_VERSION_REVISION 5 +#define CV_VERSION_STATUS "-pre" #define CVAUX_STR_EXP(__A) #__A #define CVAUX_STR(__A) CVAUX_STR_EXP(__A) diff --git a/modules/dnn/include/opencv2/dnn/version.hpp b/modules/dnn/include/opencv2/dnn/version.hpp index 63e70d2d6e..a0a1754901 100644 --- a/modules/dnn/include/opencv2/dnn/version.hpp +++ b/modules/dnn/include/opencv2/dnn/version.hpp @@ -6,7 +6,7 @@ #define OPENCV_DNN_VERSION_HPP /// Use with major OpenCV version only. -#define OPENCV_DNN_API_VERSION 20211004 +#define OPENCV_DNN_API_VERSION 20211220 #if !defined CV_DOXYGEN && !defined CV_STATIC_ANALYSIS && !defined CV_DNN_DONT_ADD_INLINE_NS #define CV__DNN_INLINE_NS __CV_CAT(dnn4_v, OPENCV_DNN_API_VERSION) diff --git a/modules/python/package/setup.py b/modules/python/package/setup.py index 030523182a..191ab4e77d 100644 --- a/modules/python/package/setup.py +++ b/modules/python/package/setup.py @@ -9,7 +9,7 @@ def main(): os.chdir(SCRIPT_DIR) package_name = 'opencv' - package_version = os.environ.get('OPENCV_VERSION', '4.5.4') # TODO + package_version = os.environ.get('OPENCV_VERSION', '4.5.5') # TODO long_description = 'Open Source Computer Vision Library Python bindings' # TODO diff --git a/platforms/maven/opencv-it/pom.xml b/platforms/maven/opencv-it/pom.xml index 0209abc3a1..a7dc090f38 100644 --- a/platforms/maven/opencv-it/pom.xml +++ b/platforms/maven/opencv-it/pom.xml @@ -4,7 +4,7 @@ org.opencv opencv-parent - 4.5.4 + 4.5.5 org.opencv opencv-it diff --git a/platforms/maven/opencv/pom.xml b/platforms/maven/opencv/pom.xml index 3551ffc09f..9b081e4d1f 100644 --- a/platforms/maven/opencv/pom.xml +++ b/platforms/maven/opencv/pom.xml @@ -4,7 +4,7 @@ org.opencv opencv-parent - 4.5.4 + 4.5.5 org.opencv opencv diff --git a/platforms/maven/pom.xml b/platforms/maven/pom.xml index 66aa22c8ec..453b79f4ec 100644 --- a/platforms/maven/pom.xml +++ b/platforms/maven/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.opencv opencv-parent - 4.5.4 + 4.5.5 pom OpenCV Parent POM From fec2c7e715d3eaeca20e7051afd4476c4412aef6 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Thu, 16 Dec 2021 22:41:47 +0300 Subject: [PATCH 182/226] fix Flatten layer --- modules/dnn/src/layers/flatten_layer.cpp | 1 - modules/dnn/src/onnx/onnx_importer.cpp | 57 +++++++++++++++++-- ...ance_layer_filter__halide_denylist.inl.hpp | 6 -- ...conformance_layer_filter__openvino.inl.hpp | 24 ++------ ...e_layer_filter_opencv_all_denylist.inl.hpp | 6 -- 5 files changed, 58 insertions(+), 36 deletions(-) diff --git a/modules/dnn/src/layers/flatten_layer.cpp b/modules/dnn/src/layers/flatten_layer.cpp index c59b71248e..1e0b010167 100644 --- a/modules/dnn/src/layers/flatten_layer.cpp +++ b/modules/dnn/src/layers/flatten_layer.cpp @@ -100,7 +100,6 @@ public: { outputShapeVec.push_back(inputs[0][i]); } - CV_Assert(outputShapeVec.size() <= 4); outputs.resize(inputs.size(), outputShapeVec); diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index df404a93d3..5d88844e61 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -1781,20 +1781,67 @@ void ONNXImporter::parseSqueeze(LayerParams& layerParams, const opencv_onnx::Nod addLayer(layerParams, node_proto); } -void ONNXImporter::parseFlatten(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +void ONNXImporter::parseFlatten(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) { + opencv_onnx::NodeProto node_proto = node_proto_; CV_CheckEQ(node_proto.input_size(), 1, ""); + int axis_ = layerParams.get("axis", 1); if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) { Mat input = getBlob(node_proto, 0); - int axis = normalize_axis(layerParams.get("axis", 1), input.dims); + int axis = normalize_axis(axis_, input.dims); - std::vector out_size(&input.size[0], &input.size[0] + axis); - out_size.push_back(input.total(axis)); - Mat output = input.reshape(1, out_size); + int out_size[2] = {1, 1}; + for (int i = 0; i < axis; ++i) + { + out_size[0] *= input.size[i]; + } + for (int i = axis; i < input.dims; ++i) + { + out_size[1] *= input.size[i]; + } + + Mat output = input.reshape(1, 2, out_size); addConstant(layerParams.name, output); return; } + IterShape_t shapeIt = outShapes.find(node_proto.input(0)); + CV_Assert(shapeIt != outShapes.end()); + MatShape inpShape = shapeIt->second; + int axis = normalize_axis(axis_, inpShape.size()); + + if (axis == 0 || axis == inpShape.size()) + { + LayerParams reshapeLp; + reshapeLp.name = layerParams.name + "/reshape"; + reshapeLp.type = "Reshape"; + CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); + + inpShape.insert(axis == 0 ? inpShape.begin() : inpShape.end(), 1); + reshapeLp.set("dim", DictValue::arrayInt(&inpShape[0], inpShape.size())); + + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(reshapeLp.name); + addLayer(reshapeLp, proto); + node_proto.set_input(0, reshapeLp.name); + axis += 1; + } + + LayerParams first_pass; + first_pass.name = layerParams.name + "/flatten"; + CV_Assert(layer_id.find(first_pass.name) == layer_id.end()); + first_pass.type = "Flatten"; + first_pass.set("axis", 0); + first_pass.set("end_axis", axis - 1); + + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(first_pass.name); + addLayer(first_pass, proto); + + layerParams.set("axis", 1); + node_proto.set_input(0, first_pass.name); addLayer(layerParams, node_proto); } diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__halide_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__halide_denylist.inl.hpp index dd0a249081..08938c9ad7 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter__halide_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__halide_denylist.inl.hpp @@ -17,12 +17,6 @@ "test_elu", "test_elu_default", "test_exp", -"test_flatten_axis0", -"test_flatten_axis2", -"test_flatten_axis3", -"test_flatten_negative_axis1", -"test_flatten_negative_axis2", -"test_flatten_negative_axis4", "test_leakyrelu", "test_leakyrelu_default", "test_logsoftmax_axis_1", diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp index 25bb8dff9a..c3b62a7f75 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp @@ -561,35 +561,23 @@ CASE(test_eyelike_with_dtype) CASE(test_eyelike_without_dtype) // no filter CASE(test_flatten_axis0) -#if INF_ENGINE_VER_MAJOR_EQ(2021040000) - SKIP; -#endif + // no filter CASE(test_flatten_axis1) // no filter CASE(test_flatten_axis2) -#if INF_ENGINE_VER_MAJOR_EQ(2021040000) - SKIP; -#endif + // no filter CASE(test_flatten_axis3) -#if INF_ENGINE_VER_MAJOR_EQ(2021040000) - SKIP; -#endif + // no filter CASE(test_flatten_default_axis) // no filter CASE(test_flatten_negative_axis1) -#if INF_ENGINE_VER_MAJOR_EQ(2021040000) - SKIP; -#endif + // no filter CASE(test_flatten_negative_axis2) -#if INF_ENGINE_VER_MAJOR_EQ(2021040000) - SKIP; -#endif + // no filter CASE(test_flatten_negative_axis3) // no filter CASE(test_flatten_negative_axis4) -#if INF_ENGINE_VER_MAJOR_EQ(2021040000) - SKIP; -#endif + // no filter CASE(test_floor) // no filter CASE(test_floor_example) diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp index cff1e93aa0..c9966f11d5 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp @@ -7,12 +7,6 @@ "test_castlike_FLOAT_to_STRING_expanded", "test_castlike_STRING_to_FLOAT_expanded", "test_concat_1d_axis_negative_1", -"test_flatten_axis0", -"test_flatten_axis2", -"test_flatten_axis3", -"test_flatten_negative_axis1", -"test_flatten_negative_axis2", -"test_flatten_negative_axis4", "test_logsoftmax_default_axis", "test_maxpool_2d_dilations", "test_maxpool_2d_same_lower", From 175bcb17345ed290c0c32af82491ece115676d36 Mon Sep 17 00:00:00 2001 From: eplankin Date: Fri, 17 Dec 2021 16:31:37 +0300 Subject: [PATCH 183/226] Merge pull request #21258 from eplankin:fix_threshold_to_zero_ipp_bug Fixed threshold(THRESH_TOZERO) at imgproc(IPP) * Fixed #16085: imgproc(IPP): wrong result from threshold(THRESH_TOZERO) * 1. Added test cases with float where all bits of mantissa equal 1, min and max float as inputs 2. Used nextafterf instead of cast to hex * Used float value in test instead of hex and casts * Changed input value in test --- modules/imgproc/src/thresh.cpp | 4 +--- modules/imgproc/test/test_thresh.cpp | 30 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/modules/imgproc/src/thresh.cpp b/modules/imgproc/src/thresh.cpp index 2e6690e55b..c3799d83a1 100644 --- a/modules/imgproc/src/thresh.cpp +++ b/modules/imgproc/src/thresh.cpp @@ -774,16 +774,14 @@ thresh_32f( const Mat& _src, Mat& _dst, float thresh, float maxval, int type ) } setIppErrorStatus(); break; -#if 0 // details: https://github.com/opencv/opencv/pull/16085 case THRESH_TOZERO: - if (0 <= CV_INSTRUMENT_FUN_IPP(ippiThreshold_LTVal_32f_C1R, src, (int)src_step*sizeof(src[0]), dst, (int)dst_step*sizeof(dst[0]), sz, thresh + FLT_EPSILON, 0)) + if (0 <= CV_INSTRUMENT_FUN_IPP(ippiThreshold_LTVal_32f_C1R, src, (int)src_step*sizeof(src[0]), dst, (int)dst_step*sizeof(dst[0]), sz, nextafterf(thresh, std::numeric_limits::infinity()), 0)) { CV_IMPL_ADD(CV_IMPL_IPP); return; } setIppErrorStatus(); break; -#endif case THRESH_TOZERO_INV: if (0 <= CV_INSTRUMENT_FUN_IPP(ippiThreshold_GTVal_32f_C1R, src, (int)src_step*sizeof(src[0]), dst, (int)dst_step*sizeof(dst[0]), sz, thresh, 0)) { diff --git a/modules/imgproc/test/test_thresh.cpp b/modules/imgproc/test/test_thresh.cpp index a8e961675d..f51afa12bd 100644 --- a/modules/imgproc/test/test_thresh.cpp +++ b/modules/imgproc/test/test_thresh.cpp @@ -443,4 +443,34 @@ TEST(Imgproc_Threshold, regression_THRESH_TOZERO_IPP_16085) EXPECT_EQ(0, cv::norm(result, NORM_INF)); } +TEST(Imgproc_Threshold, regression_THRESH_TOZERO_IPP_21258) +{ + Size sz(16, 16); + float val = nextafterf(16.0f, 0.0f); // 0x417fffff, all bits in mantissa are 1 + Mat input(sz, CV_32F, Scalar::all(val)); + Mat result; + cv::threshold(input, result, val, 0.0, THRESH_TOZERO); + EXPECT_EQ(0, cv::norm(result, NORM_INF)); +} + +TEST(Imgproc_Threshold, regression_THRESH_TOZERO_IPP_21258_Min) +{ + Size sz(16, 16); + float min_val = -std::numeric_limits::max(); + Mat input(sz, CV_32F, Scalar::all(min_val)); + Mat result; + cv::threshold(input, result, min_val, 0.0, THRESH_TOZERO); + EXPECT_EQ(0, cv::norm(result, NORM_INF)); +} + +TEST(Imgproc_Threshold, regression_THRESH_TOZERO_IPP_21258_Max) +{ + Size sz(16, 16); + float max_val = std::numeric_limits::max(); + Mat input(sz, CV_32F, Scalar::all(max_val)); + Mat result; + cv::threshold(input, result, max_val, 0.0, THRESH_TOZERO); + EXPECT_EQ(0, cv::norm(result, NORM_INF)); +} + }} // namespace From 04b27525faa51941de054526c069bd3b7f9e4af3 Mon Sep 17 00:00:00 2001 From: Trutnev Aleksei Date: Fri, 17 Dec 2021 16:42:47 +0300 Subject: [PATCH 184/226] Merge pull request #21231 from alexgiving:atrutnev/SIMD_SubRC_fluid GAPI FLUID: SIMD for SubRC kernel * SIMD for SubRC * Reverse subrc --- modules/gapi/include/opencv2/gapi/core.hpp | 2 +- .../gapi/src/backends/fluid/gfluidcore.cpp | 190 +++++------------- .../fluid/gfluidcore_func.dispatch.cpp | 27 +++ .../src/backends/fluid/gfluidcore_func.hpp | 27 ++- .../backends/fluid/gfluidcore_func.simd.hpp | 70 ++++++- 5 files changed, 171 insertions(+), 145 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/core.hpp b/modules/gapi/include/opencv2/gapi/core.hpp index 052c6a944c..791aa4ce56 100644 --- a/modules/gapi/include/opencv2/gapi/core.hpp +++ b/modules/gapi/include/opencv2/gapi/core.hpp @@ -707,7 +707,7 @@ GAPI_EXPORTS GMat subC(const GMat& src, const GScalar& c, int ddepth = -1); /** @brief Calculates the per-element difference between given scalar and the matrix. The function can be replaced with matrix expressions: - \f[\texttt{dst} = \texttt{val} - \texttt{src}\f] + \f[\texttt{dst} = \texttt{c} - \texttt{src}\f] Depth of the output matrix is determined by the ddepth parameter. If ddepth is set to default -1, the depth of output matrix will be the same as the depth of input matrix. diff --git a/modules/gapi/src/backends/fluid/gfluidcore.cpp b/modules/gapi/src/backends/fluid/gfluidcore.cpp index 8342a26d0d..5f2dbe37de 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore.cpp +++ b/modules/gapi/src/backends/fluid/gfluidcore.cpp @@ -75,7 +75,7 @@ static inline DST sub(SRC1 x, SRC2 y) template static inline DST subr(SRC1 x, SRC2 y) { - return saturate(y - x, roundf); // reverse: y - x + return saturate(y - x, roundf); // reverse sub } template @@ -844,110 +844,6 @@ GAPI_FLUID_KERNEL(GFluidAbsDiff, cv::gapi::core::GAbsDiff, false) // //-------------------------------------- -static inline v_uint16x8 v_subr_16u(const v_uint16x8 &x, const v_uint16x8 &y) { return y - x; } - -static inline v_float32x4 v_subr_32f(const v_float32x4 &x, const v_float32x4 &y) { return y - x; } - -static inline int s_subr_8u(uchar x, uchar y) { return y - x; } - -static inline float s_subr_32f(float x, float y) { return y - x; } - -// manual SIMD if important case 8UC3 -static void run_arithm_s3(uchar out[], const uchar in[], int width, const uchar scalar[], - v_uint16x8 (*v_op)(const v_uint16x8&, const v_uint16x8&), - int (*s_op)(uchar, uchar)) -{ - int w = 0; - -#if CV_SIMD128 - for (; w <= width-16; w+=16) - { - v_uint8x16 x, y, z; - v_load_deinterleave(&in[3*w], x, y, z); - - v_uint16x8 r0, r1; - - v_expand(x, r0, r1); - r0 = v_op(r0, v_setall_u16(scalar[0])); // x + scalar[0] - r1 = v_op(r1, v_setall_u16(scalar[0])); - x = v_pack(r0, r1); - - v_expand(y, r0, r1); - r0 = v_op(r0, v_setall_u16(scalar[1])); // y + scalar[1] - r1 = v_op(r1, v_setall_u16(scalar[1])); - y = v_pack(r0, r1); - - v_expand(z, r0, r1); - r0 = v_op(r0, v_setall_u16(scalar[2])); // z + scalar[2] - r1 = v_op(r1, v_setall_u16(scalar[2])); - z = v_pack(r0, r1); - - v_store_interleave(&out[3*w], x, y, z); - } -#endif - cv::util::suppress_unused_warning(v_op); - for (; w < width; w++) - { - out[3*w ] = saturate( s_op(in[3*w ], scalar[0]) ); - out[3*w + 1] = saturate( s_op(in[3*w + 1], scalar[1]) ); - out[3*w + 2] = saturate( s_op(in[3*w + 2], scalar[2]) ); - } -} - -// manually SIMD if rounding 32F into 8U, single channel -static void run_arithm_s1(uchar out[], const float in[], int width, const float scalar[], - v_float32x4 (*v_op)(const v_float32x4&, const v_float32x4&), - float (*s_op)(float, float)) -{ - int w = 0; - -#if CV_SIMD128 - for (; w <= width-16; w+=16) - { - v_float32x4 r0, r1, r2, r3; - r0 = v_load(&in[w ]); - r1 = v_load(&in[w + 4]); - r2 = v_load(&in[w + 8]); - r3 = v_load(&in[w + 12]); - - r0 = v_op(r0, v_setall_f32(scalar[0])); // r + scalar[0] - r1 = v_op(r1, v_setall_f32(scalar[0])); - r2 = v_op(r2, v_setall_f32(scalar[0])); - r3 = v_op(r3, v_setall_f32(scalar[0])); - - v_int32x4 i0, i1, i2, i3; - i0 = v_round(r0); - i1 = v_round(r1); - i2 = v_round(r2); - i3 = v_round(r3); - - v_uint16x8 us0, us1; - us0 = v_pack_u(i0, i1); - us1 = v_pack_u(i2, i3); - - v_uint8x16 uc; - uc = v_pack(us0, us1); - - v_store(&out[w], uc); - } -#endif - cv::util::suppress_unused_warning(v_op); - for (; w < width; w++) - { - out[w] = saturate(s_op(in[w], scalar[0]), roundf); - } -} - -static void run_arithm_s_subr3(uchar out[], const uchar in[], int width, const uchar scalar[]) -{ - run_arithm_s3(out, in, width, scalar, v_subr_16u, s_subr_8u); // reverse: subr -} - -static void run_arithm_s_subr1(uchar out[], const float in[], int width, const float scalar[]) -{ - run_arithm_s1(out, in, width, scalar, v_subr_32f, s_subr_32f); // reverse: subr -} - // manually unroll the inner cycle by channels template static void run_arithm_s(DST out[], const SRC in[], int width, int chan, @@ -1076,32 +972,20 @@ static void run_arithm_rs(Buffer &dst, const View &src, const float scalar[4], A int width = dst.length(); int chan = dst.meta().chan; - - // What if we cast the scalar into the SRC type? - const SRC myscal[4] = { static_cast(scalar[0]), static_cast(scalar[1]), - static_cast(scalar[2]), static_cast(scalar[3]) }; - bool usemyscal = (myscal[0] == scalar[0]) && (myscal[1] == scalar[1]) && - (myscal[2] == scalar[2]) && (myscal[3] == scalar[3]); + const int length = width * chan; switch (arithm) { case ARITHM_SUBTRACT: - if (usemyscal) - { - if (std::is_same::value && - std::is_same::value && - chan == 3) - run_arithm_s_subr3((uchar*)out, (const uchar*)in, width, (const uchar*)myscal); - else if (std::is_same::value && - std::is_same::value && - chan == 1) - run_arithm_s_subr1((uchar*)out, (const float*)in, width, (const float*)myscal); - else - run_arithm_s(out, in, width, chan, myscal, subr); - } - else - run_arithm_s(out, in, width, chan, scalar, subr); + { + int w = 0; +#if CV_SIMD + w = subrc_simd(scalar, in, out, length, chan); +#endif + for (; w < length; ++w) + out[w] = subr(in[w], scalar[w % chan]); break; + } // TODO: optimize division case ARITHM_DIVIDE: for (int w=0; w < width; w++) @@ -1274,30 +1158,54 @@ GAPI_FLUID_KERNEL(GFluidSubC, cv::gapi::core::GSubC, true) } }; -GAPI_FLUID_KERNEL(GFluidSubRC, cv::gapi::core::GSubRC, false) +GAPI_FLUID_KERNEL(GFluidSubRC, cv::gapi::core::GSubRC, true) { static const int Window = 1; - static void run(const cv::Scalar &_scalar, const View &src, int /*dtype*/, Buffer &dst) + static void run(const cv::Scalar& _scalar, const View& src, int /*dtype*/, Buffer& dst, Buffer& scratch) { - const float scalar[4] = { - static_cast(_scalar[0]), - static_cast(_scalar[1]), - static_cast(_scalar[2]), - static_cast(_scalar[3]) - }; + GAPI_Assert(src.meta().chan <= 4); + + if (dst.y() == 0) + { + const int chan = src.meta().chan; + float* sc = scratch.OutLine(); + + for (int i = 0; i < scratch.length(); ++i) + sc[i] = static_cast(_scalar[i % chan]); + } + + const float* scalar = scratch.OutLine(); // DST SRC OP __VA_ARGS__ - UNARY_(uchar , uchar , run_arithm_rs, dst, src, scalar, ARITHM_SUBTRACT); - UNARY_(uchar , short, run_arithm_rs, dst, src, scalar, ARITHM_SUBTRACT); - UNARY_(uchar , float, run_arithm_rs, dst, src, scalar, ARITHM_SUBTRACT); - UNARY_( short, short, run_arithm_rs, dst, src, scalar, ARITHM_SUBTRACT); - UNARY_( float, uchar , run_arithm_rs, dst, src, scalar, ARITHM_SUBTRACT); - UNARY_( float, short, run_arithm_rs, dst, src, scalar, ARITHM_SUBTRACT); - UNARY_( float, float, run_arithm_rs, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(uchar, uchar, run_arithm_rs, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(uchar, ushort, run_arithm_rs, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(uchar, short, run_arithm_rs, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(uchar, float, run_arithm_rs, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(ushort, ushort, run_arithm_rs, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(ushort, short, run_arithm_rs, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(ushort, uchar, run_arithm_rs, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(ushort, float, run_arithm_rs, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(short, short, run_arithm_rs, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(short, ushort, run_arithm_rs, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(short, uchar, run_arithm_rs, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(short, float, run_arithm_rs, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(float, uchar , run_arithm_rs, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(float, ushort, run_arithm_rs, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(float, short, run_arithm_rs, dst, src, scalar, ARITHM_SUBTRACT); + UNARY_(float, float, run_arithm_rs, dst, src, scalar, ARITHM_SUBTRACT); CV_Error(cv::Error::StsBadArg, "unsupported combination of types"); } + + static void initScratch(const GScalarDesc&, const GMatDesc&, int, Buffer& scratch) + { + initScratchBuffer(scratch); + } + + static void resetScratch(Buffer& /*scratch*/) + { + } }; GAPI_FLUID_KERNEL(GFluidMulC, cv::gapi::core::GMulC, true) diff --git a/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp b/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp index ab6b013694..348c00ed12 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp +++ b/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp @@ -138,6 +138,33 @@ SUBC_SIMD(float, float) #undef SUBC_SIMD +#define SUBRC_SIMD(SRC, DST) \ +int subrc_simd(const float scalar[], const SRC in[], DST out[], \ + const int length, const int chan) \ +{ \ + CV_CPU_DISPATCH(subrc_simd, (scalar, in, out, length, chan), \ + CV_CPU_DISPATCH_MODES_ALL); \ +} + +SUBRC_SIMD(uchar, uchar) +SUBRC_SIMD(ushort, uchar) +SUBRC_SIMD(short, uchar) +SUBRC_SIMD(float, uchar) +SUBRC_SIMD(short, short) +SUBRC_SIMD(ushort, short) +SUBRC_SIMD(uchar, short) +SUBRC_SIMD(float, short) +SUBRC_SIMD(ushort, ushort) +SUBRC_SIMD(uchar, ushort) +SUBRC_SIMD(short, ushort) +SUBRC_SIMD(float, ushort) +SUBRC_SIMD(uchar, float) +SUBRC_SIMD(ushort, float) +SUBRC_SIMD(short, float) +SUBRC_SIMD(float, float) + +#undef SUBRC_SIMD + #define MULC_SIMD(SRC, DST) \ int mulc_simd(const SRC in[], const float scalar[], DST out[], \ const int length, const int chan, const float scale) \ diff --git a/modules/gapi/src/backends/fluid/gfluidcore_func.hpp b/modules/gapi/src/backends/fluid/gfluidcore_func.hpp index 522d7b8b44..6023a879d9 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore_func.hpp +++ b/modules/gapi/src/backends/fluid/gfluidcore_func.hpp @@ -106,8 +106,31 @@ SUBC_SIMD(float, float) #undef SUBC_SIMD -#define MULC_SIMD(SRC, DST) \ -int mulc_simd(const SRC in[], const float scalar[], DST out[], \ +#define SUBRC_SIMD(SRC, DST) \ +int subrc_simd(const float scalar[], const SRC in[], DST out[], \ + const int length, const int chan); + +SUBRC_SIMD(uchar, uchar) +SUBRC_SIMD(ushort, uchar) +SUBRC_SIMD(short, uchar) +SUBRC_SIMD(float, uchar) +SUBRC_SIMD(short, short) +SUBRC_SIMD(ushort, short) +SUBRC_SIMD(uchar, short) +SUBRC_SIMD(float, short) +SUBRC_SIMD(ushort, ushort) +SUBRC_SIMD(uchar, ushort) +SUBRC_SIMD(short, ushort) +SUBRC_SIMD(float, ushort) +SUBRC_SIMD(uchar, float) +SUBRC_SIMD(ushort, float) +SUBRC_SIMD(short, float) +SUBRC_SIMD(float, float) + +#undef SUBRC_SIMD + +#define MULC_SIMD(SRC, DST) \ +int mulc_simd(const SRC in[], const float scalar[], DST out[], \ const int length, const int chan, const float scale); MULC_SIMD(uchar, uchar) diff --git a/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp b/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp index 12b74f8f67..38c47072f4 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp +++ b/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp @@ -127,6 +127,28 @@ SUBC_SIMD(float, float) #undef SUBC_SIMD +#define SUBRC_SIMD(SRC, DST) \ +int subrc_simd(const float scalar[], const SRC in[], DST out[], \ + const int length, const int chan); + +SUBRC_SIMD(uchar, uchar) +SUBRC_SIMD(ushort, uchar) +SUBRC_SIMD(short, uchar) +SUBRC_SIMD(float, uchar) +SUBRC_SIMD(short, short) +SUBRC_SIMD(ushort, short) +SUBRC_SIMD(uchar, short) +SUBRC_SIMD(float, short) +SUBRC_SIMD(ushort, ushort) +SUBRC_SIMD(uchar, ushort) +SUBRC_SIMD(short, ushort) +SUBRC_SIMD(float, ushort) +SUBRC_SIMD(uchar, float) +SUBRC_SIMD(ushort, float) +SUBRC_SIMD(short, float) +SUBRC_SIMD(float, float) + +#undef SUBRC_SIMD #define MULC_SIMD(SRC, DST) \ int mulc_simd(const SRC in[], const float scalar[], DST out[], \ @@ -905,12 +927,13 @@ MUL_SIMD(float, float) //------------------------- // -// Fluid kernels: AddC, SubC +// Fluid kernels: AddC, SubC, SubRC // //------------------------- struct add_tag {}; struct sub_tag {}; +struct subr_tag {}; struct mul_tag {}; struct absdiff_tag {}; @@ -946,6 +969,11 @@ CV_ALWAYS_INLINE v_float32 oper(sub_tag, const v_float32& a, const v_float32& sc return a - sc; } +CV_ALWAYS_INLINE v_float32 oper(subr_tag, const v_float32& a, const v_float32& sc) +{ + return sc - a; +} + CV_ALWAYS_INLINE v_float32 oper(mul_tag, const v_float32& a, const v_float32& sc) { return a * sc; @@ -1218,6 +1246,46 @@ SUBC_SIMD(float, float) #undef SUBC_SIMD +//------------------------------------------------------------------------------------------------- + +#define SUBRC_SIMD(SRC, DST) \ +int subrc_simd(const float scalar[], const SRC in[], DST out[], \ + const int length, const int chan) \ +{ \ + switch (chan) \ + { \ + case 1: \ + case 2: \ + case 4: \ + return arithmOpScalar_simd_common(subr_tag{}, in, scalar, out, length); \ + case 3: \ + return arithmOpScalar_simd_c3(subr_tag{}, in, scalar, out, length); \ + default: \ + GAPI_Assert(chan <= 4); \ + break; \ + } \ + return 0; \ +} + +SUBRC_SIMD(uchar, uchar) +SUBRC_SIMD(ushort, uchar) +SUBRC_SIMD(short, uchar) +SUBRC_SIMD(float, uchar) +SUBRC_SIMD(short, short) +SUBRC_SIMD(ushort, short) +SUBRC_SIMD(uchar, short) +SUBRC_SIMD(float, short) +SUBRC_SIMD(ushort, ushort) +SUBRC_SIMD(uchar, ushort) +SUBRC_SIMD(short, ushort) +SUBRC_SIMD(float, ushort) +SUBRC_SIMD(uchar, float) +SUBRC_SIMD(ushort, float) +SUBRC_SIMD(short, float) +SUBRC_SIMD(float, float) + +#undef SUBRC_SIMD + //------------------------- // // Fluid kernels: MulC From 249c5081262ae258f92f9e2a047d9eaac6f65499 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 17 Dec 2021 13:28:34 +0000 Subject: [PATCH 185/226] dnn(test): skip failed tests on Vulkan backend --- modules/dnn/test/test_misc.cpp | 10 +++++++-- modules/dnn/test/test_onnx_importer.cpp | 27 +++++++++++++++++++++++++ modules/dnn/test/test_tf_importer.cpp | 6 ++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/modules/dnn/test/test_misc.cpp b/modules/dnn/test/test_misc.cpp index e60eb969aa..39bb73a918 100644 --- a/modules/dnn/test/test_misc.cpp +++ b/modules/dnn/test/test_misc.cpp @@ -842,6 +842,12 @@ TEST_P(Test_two_inputs, basic) Backend backendId = get<0>(get<2>(GetParam())); Target targetId = get<1>(get<2>(GetParam())); + int type1 = get<0>(GetParam()); + int type2 = get<1>(GetParam()); + + if (backendId == DNN_BACKEND_VKCOM && !(type1 == CV_32F && type2 == CV_32F)) + applyTestTag(CV_TEST_TAG_DNN_SKIP_VULKAN); + Net net; LayerParams lp; lp.type = "Eltwise"; @@ -851,8 +857,8 @@ TEST_P(Test_two_inputs, basic) net.connect(0, 1, eltwiseId, 1); // connect to a second input int inpSize[] = {1, 2, 3, 4}; - Mat firstInp(4, &inpSize[0], get<0>(GetParam())); - Mat secondInp(4, &inpSize[0], get<1>(GetParam())); + Mat firstInp(4, &inpSize[0], type1); + Mat secondInp(4, &inpSize[0], type2); randu(firstInp, 0, 100); randu(secondInp, 0, 100); diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index bfea8550c0..55339462c8 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -242,6 +242,9 @@ TEST_P(Test_ONNX_layers, Deconvolution3D) if (backend == DNN_BACKEND_OPENCV) throw SkipTestException("OpenCV backend is not supported"); // FIXIT use tags + if (backend == DNN_BACKEND_VKCOM) + applyTestTag(CV_TEST_TAG_DNN_SKIP_VULKAN); + testONNXModels("deconv3d"); } @@ -260,6 +263,9 @@ TEST_P(Test_ONNX_layers, Deconvolution3D_bias) if (backend == DNN_BACKEND_OPENCV) throw SkipTestException("OpenCV backend is not supported"); // FIXIT use tags + if (backend == DNN_BACKEND_VKCOM) + applyTestTag(CV_TEST_TAG_DNN_SKIP_VULKAN); + testONNXModels("deconv3d_bias"); } @@ -278,6 +284,9 @@ TEST_P(Test_ONNX_layers, Deconvolution3D_pad) if (backend == DNN_BACKEND_OPENCV) throw SkipTestException("OpenCV backend is not supported"); // FIXIT use tags + if (backend == DNN_BACKEND_VKCOM) + applyTestTag(CV_TEST_TAG_DNN_SKIP_VULKAN); + testONNXModels("deconv3d_pad"); } @@ -296,6 +305,9 @@ TEST_P(Test_ONNX_layers, Deconvolution3D_adjpad) if (backend == DNN_BACKEND_OPENCV) throw SkipTestException("OpenCV backend is not supported"); // FIXIT use tags + if (backend == DNN_BACKEND_VKCOM) + applyTestTag(CV_TEST_TAG_DNN_SKIP_VULKAN); + testONNXModels("deconv3d_adjpad"); } @@ -384,6 +396,9 @@ TEST_P(Test_ONNX_layers, ReduceMean3D) if (backend == DNN_BACKEND_OPENCV && target != DNN_TARGET_CPU) throw SkipTestException("Only CPU is supported"); // FIXIT use tags + if (backend == DNN_BACKEND_VKCOM) + applyTestTag(CV_TEST_TAG_DNN_SKIP_VULKAN); + testONNXModels("reduce_mean3d"); } @@ -671,6 +686,9 @@ TEST_P(Test_ONNX_layers, MaxPooling3D) if (backend == DNN_BACKEND_OPENCV && target != DNN_TARGET_CPU) throw SkipTestException("Only CPU is supported"); // FIXIT use tags + if (backend == DNN_BACKEND_VKCOM) + applyTestTag(CV_TEST_TAG_DNN_SKIP_VULKAN); + testONNXModels("max_pool3d", npy, 0, 0, false, false); } @@ -685,6 +703,9 @@ TEST_P(Test_ONNX_layers, AvePooling3D) if (backend == DNN_BACKEND_OPENCV && target != DNN_TARGET_CPU) throw SkipTestException("Only CPU is supported"); // FIXIT use tags + if (backend == DNN_BACKEND_VKCOM) + applyTestTag(CV_TEST_TAG_DNN_SKIP_VULKAN); + testONNXModels("ave_pool3d"); } @@ -699,6 +720,9 @@ TEST_P(Test_ONNX_layers, PoolConv3D) if (backend == DNN_BACKEND_OPENCV && target != DNN_TARGET_CPU) throw SkipTestException("Only CPU is supported"); // FIXIT use tags + if (backend == DNN_BACKEND_VKCOM) + applyTestTag(CV_TEST_TAG_DNN_SKIP_VULKAN); + testONNXModels("pool_conv_3d"); } @@ -1777,6 +1801,9 @@ TEST_P(Test_ONNX_nets, Resnet34_kinetics) if (backend == DNN_BACKEND_OPENCV && target != DNN_TARGET_CPU) throw SkipTestException("Only CPU is supported"); // FIXIT use tags + if (backend == DNN_BACKEND_VKCOM) + applyTestTag(CV_TEST_TAG_DNN_SKIP_VULKAN); + String onnxmodel = findDataFile("dnn/resnet-34_kinetics.onnx", false); Mat image0 = imread(findDataFile("dnn/dog416.png")); Mat image1 = imread(findDataFile("dnn/street.png")); diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index b40a604a6e..7a4bfc96f2 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -627,6 +627,9 @@ TEST_P(Test_TensorFlow_layers, MaxPooling3D) if (backend == DNN_BACKEND_OPENCV && target != DNN_TARGET_CPU) throw SkipTestException("Only CPU is supported"); // FIXIT use tags + if (backend == DNN_BACKEND_VKCOM) + applyTestTag(CV_TEST_TAG_DNN_SKIP_VULKAN); + runTensorFlowNet("max_pool3d"); } @@ -641,6 +644,9 @@ TEST_P(Test_TensorFlow_layers, AvePooling3D) if (backend == DNN_BACKEND_OPENCV && target != DNN_TARGET_CPU) throw SkipTestException("Only CPU is supported"); // FIXIT use tags + if (backend == DNN_BACKEND_VKCOM) + applyTestTag(CV_TEST_TAG_DNN_SKIP_VULKAN); + runTensorFlowNet("ave_pool3d"); } From 1bd382c1d0dc7ce65b352ffb0628b5e9bb431f28 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Thu, 2 Dec 2021 15:45:27 +0300 Subject: [PATCH 186/226] Add acos, acosh, asin, asinh, atan, atanh, cos, cosh, erf, hardswish, sin, sinh, softplus, softsign, tan layers --- .../dnn/include/opencv2/dnn/all_layers.hpp | 90 +++ modules/dnn/src/cuda/activations.cu | 105 ++++ modules/dnn/src/cuda/functors.hpp | 225 +++++++ modules/dnn/src/cuda/math.hpp | 84 +++ .../dnn/src/cuda4dnn/kernels/activations.hpp | 45 ++ .../src/cuda4dnn/primitives/activation.hpp | 210 +++++++ modules/dnn/src/init.cpp | 15 + modules/dnn/src/layers/elementwise_layers.cpp | 552 ++++++++++++++++++ .../dnn/src/onnx/onnx_graph_simplifier.cpp | 37 ++ modules/dnn/src/opencl/activations.cl | 84 +++ ...rmance_layer_filter__cuda_denylist.inl.hpp | 1 + ...er_filter_opencv_ocl_fp16_denylist.inl.hpp | 1 + ..._conformance_layer_parser_denylist.inl.hpp | 29 - 13 files changed, 1449 insertions(+), 29 deletions(-) diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index 45935279f5..26d7a9b069 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -648,6 +648,96 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + class CV_EXPORTS AcosLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS AcoshLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS AsinLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS AsinhLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS AtanLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS AtanhLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS CosLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS CoshLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS ErfLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS HardSwishLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS SinLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS SinhLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS SoftplusLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS SoftsignLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS TanLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + class CV_EXPORTS ActivationLayerInt8 : public ActivationLayer { public: diff --git a/modules/dnn/src/cuda/activations.cu b/modules/dnn/src/cuda/activations.cu index 0980b5dd46..3d99a03ae3 100644 --- a/modules/dnn/src/cuda/activations.cu +++ b/modules/dnn/src/cuda/activations.cu @@ -158,6 +158,81 @@ void not_k(const Stream& stream, Span output, View input) { generic_op>(stream, output, input); } +template +void acos(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + +template +void acosh(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + +template +void asin(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + +template +void asinh(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + +template +void atan(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + +template +void atanh(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + +template +void cos(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + +template +void cosh(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + +template +void erf(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + +template +void hardswish(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + +template +void sin(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + +template +void sinh(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + +template +void softplus(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + +template +void softsign(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + +template +void tan(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + template void abs(const Stream& stream, Span output, View input) { generic_op>(stream, output, input); @@ -196,6 +271,21 @@ template void log<__half>(const Stream&, Span<__half>, View<__half>); template void rint<__half>(const Stream&, Span<__half>, View<__half>); template void sqrt<__half>(const Stream&, Span<__half>, View<__half>); template void not_k<__half>(const Stream&, Span<__half>, View<__half>); +template void acos<__half>(const Stream&, Span<__half>, View<__half>); +template void acosh<__half>(const Stream&, Span<__half>, View<__half>); +template void asin<__half>(const Stream&, Span<__half>, View<__half>); +template void asinh<__half>(const Stream&, Span<__half>, View<__half>); +template void atan<__half>(const Stream&, Span<__half>, View<__half>); +template void atanh<__half>(const Stream&, Span<__half>, View<__half>); +template void cos<__half>(const Stream&, Span<__half>, View<__half>); +template void cosh<__half>(const Stream&, Span<__half>, View<__half>); +template void erf<__half>(const Stream&, Span<__half>, View<__half>); +template void hardswish<__half>(const Stream&, Span<__half>, View<__half>); +template void sin<__half>(const Stream&, Span<__half>, View<__half>); +template void sinh<__half>(const Stream&, Span<__half>, View<__half>); +template void softplus<__half>(const Stream&, Span<__half>, View<__half>); +template void softsign<__half>(const Stream&, Span<__half>, View<__half>); +template void tan<__half>(const Stream&, Span<__half>, View<__half>); template void power<__half>(const Stream&, Span<__half>, View<__half>, __half, __half, __half); template void exp<__half>(const Stream&, Span<__half>, View<__half>, __half, __half); #endif @@ -216,6 +306,21 @@ template void log(const Stream&, Span, View); template void rint(const Stream&, Span, View); template void sqrt(const Stream&, Span, View); template void not_k(const Stream&, Span, View); +template void acos(const Stream&, Span, View); +template void acosh(const Stream&, Span, View); +template void asin(const Stream&, Span, View); +template void asinh(const Stream&, Span, View); +template void atan(const Stream&, Span, View); +template void atanh(const Stream&, Span, View); +template void cos(const Stream&, Span, View); +template void cosh(const Stream&, Span, View); +template void erf(const Stream&, Span, View); +template void hardswish(const Stream&, Span, View); +template void sin(const Stream&, Span, View); +template void sinh(const Stream&, Span, View); +template void softplus(const Stream&, Span, View); +template void softsign(const Stream&, Span, View); +template void tan(const Stream&, Span, View); template void power(const Stream&, Span, View, float, float, float); template void exp(const Stream&, Span, View, float, float); diff --git a/modules/dnn/src/cuda/functors.hpp b/modules/dnn/src/cuda/functors.hpp index 98ae175ce8..c3d1669344 100644 --- a/modules/dnn/src/cuda/functors.hpp +++ b/modules/dnn/src/cuda/functors.hpp @@ -303,6 +303,231 @@ struct NotFunctor { } }; +template +struct AcosFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE AcosFunctor() { } + CUDA4DNN_DEVICE AcosFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::acos; + return acos(value); + } +}; + +template +struct AcoshFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE AcoshFunctor() { } + CUDA4DNN_DEVICE AcoshFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::acosh; + return acosh(value); + } +}; + +template +struct AsinFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE AsinFunctor() { } + CUDA4DNN_DEVICE AsinFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::asin; + return asin(value); + } +}; + +template +struct AsinhFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE AsinhFunctor() { } + CUDA4DNN_DEVICE AsinhFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::asinh; + return asinh(value); + } +}; + +template +struct AtanFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE AtanFunctor() { } + CUDA4DNN_DEVICE AtanFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::atan; + return atan(value); + } +}; + +template +struct AtanhFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE AtanhFunctor() { } + CUDA4DNN_DEVICE AtanhFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::atanh; + return atanh(value); + } +}; + +template +struct CosFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE CosFunctor() { } + CUDA4DNN_DEVICE CosFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::cos; + return cos(value); + } +}; + +template +struct CoshFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE CoshFunctor() { } + CUDA4DNN_DEVICE CoshFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::cosh; + return cosh(value); + } +}; + +template +struct ErfFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE ErfFunctor() { } + CUDA4DNN_DEVICE ErfFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::erf; + return erf(value); + } +}; + +template +struct HardSwishFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE HardSwishFunctor() { } + CUDA4DNN_DEVICE HardSwishFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::clamp; // saturate? + return value * clamp(value / static_cast(6.f) + static_cast(0.5f), static_cast(0.f), static_cast(1.f)); + } +}; + +template +struct SinFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE SinFunctor() { } + CUDA4DNN_DEVICE SinFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::sin; + return sin(value); + } +}; + +template +struct SinhFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE SinhFunctor() { } + CUDA4DNN_DEVICE SinhFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::sinh; + return sinh(value); + } +}; + +template +struct SoftplusFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE SoftplusFunctor() { } + CUDA4DNN_DEVICE SoftplusFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::log1pexp; + return log1pexp(value); + } +}; + +template +struct SoftsignFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE SoftsignFunctor() { } + CUDA4DNN_DEVICE SoftsignFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::abs; + return value / (static_cast(1.f) + abs(value)); + } +}; + +template +struct TanFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE TanFunctor() { } + CUDA4DNN_DEVICE TanFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::tan; + return tan(value); + } +}; + template struct PowerFunctor { struct Params { diff --git a/modules/dnn/src/cuda/math.hpp b/modules/dnn/src/cuda/math.hpp index 0da584197d..0a312a250d 100644 --- a/modules/dnn/src/cuda/math.hpp +++ b/modules/dnn/src/cuda/math.hpp @@ -140,6 +140,90 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace csl { namespace de template <> inline __device__ __half rint(__half value) { return hrint(value); } #endif + template __device__ T acos(T value); + template <> inline __device__ double acos(double value) { return ::acos(value); } + template <> inline __device__ float acos(float value) { return acosf(value); } +#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) + template <> inline __device__ __half acos(__half value) { return acosf(value); } +#endif + + template __device__ T acosh(T value); + template <> inline __device__ double acosh(double value) { return ::acosh(value); } + template <> inline __device__ float acosh(float value) { return acoshf(value); } +#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) + template <> inline __device__ __half acosh(__half value) { return acoshf(value); } +#endif + + template __device__ T asin(T value); + template <> inline __device__ double asin(double value) { return ::asin(value); } + template <> inline __device__ float asin(float value) { return asinf(value); } +#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) + template <> inline __device__ __half asin(__half value) { return asinf(value); } +#endif + + template __device__ T asinh(T value); + template <> inline __device__ double asinh(double value) { return ::asinh(value); } + template <> inline __device__ float asinh(float value) { return asinhf(value); } +#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) + template <> inline __device__ __half asinh(__half value) { return asinhf(value); } +#endif + + template __device__ T atan(T value); + template <> inline __device__ double atan(double value) { return ::atan(value); } + template <> inline __device__ float atan(float value) { return atanf(value); } +#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) + template <> inline __device__ __half atan(__half value) { return atanf(value); } +#endif + + template __device__ T atanh(T value); + template <> inline __device__ double atanh(double value) { return ::atanh(value); } + template <> inline __device__ float atanh(float value) { return atanhf(value); } +#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) + template <> inline __device__ __half atanh(__half value) { return atanhf(value); } +#endif + + template __device__ T cos(T value); + template <> inline __device__ double cos(double value) { return ::cos(value); } + template <> inline __device__ float cos(float value) { return cosf(value); } +#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) + template <> inline __device__ __half cos(__half value) { return hcos(value); } +#endif + + template __device__ T cosh(T value); + template <> inline __device__ double cosh(double value) { return ::cosh(value); } + template <> inline __device__ float cosh(float value) { return coshf(value); } +#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) + template <> inline __device__ __half cosh(__half value) { return coshf(value); } +#endif + + template __device__ T erf(T value); + template <> inline __device__ double erf(double value) { return ::erf(value); } + template <> inline __device__ float erf(float value) { return erff(value); } +#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) + template <> inline __device__ __half erf(__half value) { return erff(value); } +#endif + + template __device__ T sin(T value); + template <> inline __device__ double sin(double value) { return ::sin(value); } + template <> inline __device__ float sin(float value) { return sinf(value); } +#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) + template <> inline __device__ __half sin(__half value) { return hsin(value); } +#endif + + template __device__ T sinh(T value); + template <> inline __device__ double sinh(double value) { return ::sinh(value); } + template <> inline __device__ float sinh(float value) { return sinhf(value); } +#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) + template <> inline __device__ __half sinh(__half value) { return sinhf(value); } +#endif + + template __device__ T tan(T value); + template <> inline __device__ double tan(double value) { return ::tan(value); } + template <> inline __device__ float tan(float value) { return tanf(value); } +#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) + template <> inline __device__ __half tan(__half value) { return tanf(value); } +#endif + template __device__ T ceil(T value); template <> inline __device__ double ceil(double value) { return ::ceil(value); } template <> inline __device__ float ceil(float value) { return ceilf(value); } diff --git a/modules/dnn/src/cuda4dnn/kernels/activations.hpp b/modules/dnn/src/cuda4dnn/kernels/activations.hpp index d7c471a5ec..854bc8ac0c 100644 --- a/modules/dnn/src/cuda4dnn/kernels/activations.hpp +++ b/modules/dnn/src/cuda4dnn/kernels/activations.hpp @@ -60,6 +60,51 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { template void not_k(const csl::Stream& stream, csl::Span output, csl::View input); + template + void acos(const csl::Stream& stream, csl::Span output, csl::View input); + + template + void acosh(const csl::Stream& stream, csl::Span output, csl::View input); + + template + void asin(const csl::Stream& stream, csl::Span output, csl::View input); + + template + void asinh(const csl::Stream& stream, csl::Span output, csl::View input); + + template + void atan(const csl::Stream& stream, csl::Span output, csl::View input); + + template + void atanh(const csl::Stream& stream, csl::Span output, csl::View input); + + template + void cos(const csl::Stream& stream, csl::Span output, csl::View input); + + template + void cosh(const csl::Stream& stream, csl::Span output, csl::View input); + + template + void erf(const csl::Stream& stream, csl::Span output, csl::View input); + + template + void hardswish(const csl::Stream& stream, csl::Span output, csl::View input); + + template + void sin(const csl::Stream& stream, csl::Span output, csl::View input); + + template + void sinh(const csl::Stream& stream, csl::Span output, csl::View input); + + template + void softplus(const csl::Stream& stream, csl::Span output, csl::View input); + + template + void softsign(const csl::Stream& stream, csl::Span output, csl::View input); + + template + void tan(const csl::Stream& stream, csl::Span output, csl::View input); + template void power(const csl::Stream& stream, csl::Span output, csl::View input, T exp, T scale, T shift); diff --git a/modules/dnn/src/cuda4dnn/primitives/activation.hpp b/modules/dnn/src/cuda4dnn/primitives/activation.hpp index 77a79703fe..4691996d4e 100644 --- a/modules/dnn/src/cuda4dnn/primitives/activation.hpp +++ b/modules/dnn/src/cuda4dnn/primitives/activation.hpp @@ -280,6 +280,216 @@ namespace cv { namespace dnn { namespace cuda4dnn { csl::Stream stream; }; + template + class AcosOp final : public BaseOp { + public: + AcosOp(csl::Stream stream_) : stream(std::move(stream_)) { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::acos(stream, output, input); + } + + private: + csl::Stream stream; + }; + + template + class AcoshOp final : public BaseOp { + public: + AcoshOp(csl::Stream stream_) : stream(std::move(stream_)) { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::acosh(stream, output, input); + } + + private: + csl::Stream stream; + }; + + template + class AsinOp final : public BaseOp { + public: + AsinOp(csl::Stream stream_) : stream(std::move(stream_)) { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::asin(stream, output, input); + } + + private: + csl::Stream stream; + }; + + template + class AsinhOp final : public BaseOp { + public: + AsinhOp(csl::Stream stream_) : stream(std::move(stream_)) { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::asinh(stream, output, input); + } + + private: + csl::Stream stream; + }; + + template + class AtanOp final : public BaseOp { + public: + AtanOp(csl::Stream stream_) : stream(std::move(stream_)) { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::atan(stream, output, input); + } + + private: + csl::Stream stream; + }; + + template + class AtanhOp final : public BaseOp { + public: + AtanhOp(csl::Stream stream_) : stream(std::move(stream_)) { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::atanh(stream, output, input); + } + + private: + csl::Stream stream; + }; + + template + class CosOp final : public BaseOp { + public: + CosOp(csl::Stream stream_) : stream(std::move(stream_)) { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::cos(stream, output, input); + } + + private: + csl::Stream stream; + }; + + template + class CoshOp final : public BaseOp { + public: + CoshOp(csl::Stream stream_) : stream(std::move(stream_)) { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::cosh(stream, output, input); + } + + private: + csl::Stream stream; + }; + + template + class ErfOp final : public BaseOp { + public: + ErfOp(csl::Stream stream_) : stream(std::move(stream_)) { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::erf(stream, output, input); + } + + private: + csl::Stream stream; + }; + + template + class HardSwishOp final : public BaseOp { + public: + HardSwishOp(csl::Stream stream_) : stream(std::move(stream_)) { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::hardswish(stream, output, input); + } + + private: + csl::Stream stream; + }; + + template + class SinOp final : public BaseOp { + public: + SinOp(csl::Stream stream_) : stream(std::move(stream_)) { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::sin(stream, output, input); + } + + private: + csl::Stream stream; + }; + + template + class SinhOp final : public BaseOp { + public: + SinhOp(csl::Stream stream_) : stream(std::move(stream_)) { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::sinh(stream, output, input); + } + + private: + csl::Stream stream; + }; + + template + class SoftplusOp final : public BaseOp { + public: + SoftplusOp(csl::Stream stream_) : stream(std::move(stream_)) { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::softplus(stream, output, input); + } + + private: + csl::Stream stream; + }; + + template + class SoftsignOp final : public BaseOp { + public: + SoftsignOp(csl::Stream stream_) : stream(std::move(stream_)) { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::softsign(stream, output, input); + } + + private: + csl::Stream stream; + }; + + template + class TanOp final : public BaseOp { + public: + TanOp(csl::Stream stream_) : stream(std::move(stream_)) { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::tan(stream, output, input); + } + + private: + csl::Stream stream; + }; + template class PowerOp final : public BaseOp { public: diff --git a/modules/dnn/src/init.cpp b/modules/dnn/src/init.cpp index 443d1eaef4..89a91e17ae 100644 --- a/modules/dnn/src/init.cpp +++ b/modules/dnn/src/init.cpp @@ -117,6 +117,21 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(Round, RoundLayer); CV_DNN_REGISTER_LAYER_CLASS(Sqrt, SqrtLayer); CV_DNN_REGISTER_LAYER_CLASS(Not, NotLayer); + CV_DNN_REGISTER_LAYER_CLASS(Acos, AcosLayer); + CV_DNN_REGISTER_LAYER_CLASS(Acosh, AcoshLayer); + CV_DNN_REGISTER_LAYER_CLASS(Asin, AsinLayer); + CV_DNN_REGISTER_LAYER_CLASS(Asinh, AsinhLayer); + CV_DNN_REGISTER_LAYER_CLASS(Atan, AtanLayer); + CV_DNN_REGISTER_LAYER_CLASS(Atanh, AtanhLayer); + CV_DNN_REGISTER_LAYER_CLASS(Cos, CosLayer); + CV_DNN_REGISTER_LAYER_CLASS(Cosh, CoshLayer); + CV_DNN_REGISTER_LAYER_CLASS(Erf, ErfLayer); + CV_DNN_REGISTER_LAYER_CLASS(HardSwish, HardSwishLayer); + CV_DNN_REGISTER_LAYER_CLASS(Sin, SinLayer); + CV_DNN_REGISTER_LAYER_CLASS(Sinh, SinhLayer); + CV_DNN_REGISTER_LAYER_CLASS(Softplus, SoftplusLayer); + CV_DNN_REGISTER_LAYER_CLASS(Softsign, SoftsignLayer); + CV_DNN_REGISTER_LAYER_CLASS(Tan, TanLayer); CV_DNN_REGISTER_LAYER_CLASS(BatchNorm, BatchNormLayer); CV_DNN_REGISTER_LAYER_CLASS(MaxUnpool, MaxUnpoolLayer); CV_DNN_REGISTER_LAYER_CLASS(Dropout, BlankLayer); diff --git a/modules/dnn/src/layers/elementwise_layers.cpp b/modules/dnn/src/layers/elementwise_layers.cpp index 7cec0d5f7b..772dfca602 100644 --- a/modules/dnn/src/layers/elementwise_layers.cpp +++ b/modules/dnn/src/layers/elementwise_layers.cpp @@ -76,8 +76,21 @@ using std::pow; using std::ceil; using std::floor; using std::log; +using std::log1p; using std::sqrt; using std::round; +using std::acos; +using std::acosh; +using std::asin; +using std::asinh; +using std::atan; +using std::atanh; +using std::cos; +using std::cosh; +using std::erf; +using std::sin; +using std::sinh; +using std::tan; template class ElementWiseLayer : public Func::Layer @@ -746,6 +759,20 @@ struct BaseDefaultFunctor : public BaseFunctor } #endif // HAVE_VULKAN +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + CV_Error(Error::StsNotImplemented, ""); + } +#endif + +#ifdef HAVE_HALIDE + void attachHalide(const Halide::Expr& input, Halide::Func& top) + { + CV_Error(Error::StsNotImplemented, ""); + } +#endif // HAVE_HALIDE + private: static const char* const ocl_kernel_name; }; @@ -1390,6 +1417,411 @@ struct NotFunctor : public BaseDefaultFunctor template<> const char* const BaseDefaultFunctor::ocl_kernel_name = "NotForward"; +struct AcosFunctor : public BaseDefaultFunctor +{ + typedef AcosLayer Layer; + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA; + } + + inline float calculate(float x) const + { + return acos(x); + } + +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream); + } +#endif + + int64 getFLOPSPerElement() const { return 1; } +}; + +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "AcosForward"; + +struct AcoshFunctor : public BaseDefaultFunctor +{ + typedef AcoshLayer Layer; + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA; + } + + inline float calculate(float x) const + { + return acosh(x); + } + +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream); + } +#endif + + int64 getFLOPSPerElement() const { return 1; } +}; + +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "AcoshForward"; + +struct AsinFunctor : public BaseDefaultFunctor +{ + typedef AsinLayer Layer; + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA; + } + + inline float calculate(float x) const + { + return asin(x); + } + +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream); + } +#endif + + int64 getFLOPSPerElement() const { return 1; } +}; + +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "AsinForward"; + +struct AsinhFunctor : public BaseDefaultFunctor +{ + typedef AsinhLayer Layer; + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA; + } + + inline float calculate(float x) const + { + return asinh(x); + } + +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream); + } +#endif + + int64 getFLOPSPerElement() const { return 1; } +}; + +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "AsinhForward"; + +struct AtanFunctor : public BaseDefaultFunctor +{ + typedef AtanLayer Layer; + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA; + } + + inline float calculate(float x) const + { + return atan(x); + } + +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream); + } +#endif + + int64 getFLOPSPerElement() const { return 1; } +}; + +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "AtanForward"; + +struct AtanhFunctor : public BaseDefaultFunctor +{ + typedef AtanhLayer Layer; + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA; + } + + inline float calculate(float x) const + { + return atanh(x); + } + +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream); + } +#endif + + int64 getFLOPSPerElement() const { return 1; } +}; + +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "AtanhForward"; + +struct CosFunctor : public BaseDefaultFunctor +{ + typedef CosLayer Layer; + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA; + } + + inline float calculate(float x) const + { + return cos(x); + } + +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream); + } +#endif + + int64 getFLOPSPerElement() const { return 1; } +}; + +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "CosForward"; + +struct CoshFunctor : public BaseDefaultFunctor +{ + typedef CoshLayer Layer; + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA; + } + + inline float calculate(float x) const + { + return cosh(x); + } + +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream); + } +#endif + + int64 getFLOPSPerElement() const { return 1; } +}; + +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "CoshForward"; + +struct ErfFunctor : public BaseDefaultFunctor +{ + typedef ErfLayer Layer; + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA; + } + + inline float calculate(float x) const + { + return erf(x); + } + +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream); + } +#endif + + int64 getFLOPSPerElement() const { return 1; } +}; + +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "ErfForward"; + +struct HardSwishFunctor : public BaseDefaultFunctor +{ + typedef HardSwishLayer Layer; + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA; + } + + inline float calculate(float x) const + { + return x * max(0.f, min(1.f, x / 6.f + 0.5f)); + } + +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream); + } +#endif + + int64 getFLOPSPerElement() const { return 1; } +}; + +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "HardSwishForward"; + +struct SinFunctor : public BaseDefaultFunctor +{ + typedef SinLayer Layer; + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA; + } + + inline float calculate(float x) const + { + return sin(x); + } + +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream); + } +#endif + + int64 getFLOPSPerElement() const { return 1; } +}; + +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "SinForward"; + +struct SinhFunctor : public BaseDefaultFunctor +{ + typedef SinhLayer Layer; + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA; + } + + inline float calculate(float x) const + { + return sinh(x); + } + +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream); + } +#endif + + int64 getFLOPSPerElement() const { return 1; } +}; + +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "SinhForward"; + +struct SoftplusFunctor : public BaseDefaultFunctor +{ + typedef SoftplusLayer Layer; + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA; + } + + inline float calculate(float x) const + { + return log1p(exp(x)); + } + +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream); + } +#endif + + int64 getFLOPSPerElement() const { return 1; } +}; + +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "SoftplusForward"; + +struct SoftsignFunctor : public BaseDefaultFunctor +{ + typedef SoftsignLayer Layer; + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA; + } + + inline float calculate(float x) const + { + return x / (1.f + abs(x)); + } + +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream); + } +#endif + + int64 getFLOPSPerElement() const { return 1; } +}; + +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "SoftsignForward"; + +struct TanFunctor : public BaseDefaultFunctor +{ + typedef TanLayer Layer; + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA; + } + + inline float calculate(float x) const + { + return tan(x); + } + +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream); + } +#endif + + int64 getFLOPSPerElement() const { return 1; } +}; + +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "TanForward"; + struct PowerFunctor : public BaseFunctor { typedef PowerLayer Layer; @@ -1937,6 +2369,126 @@ Ptr NotLayer::create(const LayerParams& params) return l; } +Ptr AcosLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + +Ptr AcoshLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + +Ptr AsinLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + +Ptr AsinhLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + +Ptr AtanLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + +Ptr AtanhLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + +Ptr CosLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + +Ptr CoshLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + +Ptr ErfLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + +Ptr HardSwishLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + +Ptr SinLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + +Ptr SinhLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + +Ptr SoftplusLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + +Ptr SoftsignLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + +Ptr TanLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + Ptr PowerLayer::create(const LayerParams& params) { float power = params.get("power", 1.0f); diff --git a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp index 15ce9624c4..81a5df1a28 100644 --- a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp +++ b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp @@ -206,6 +206,42 @@ public: } }; +class HardSwishSubgraph : public Subgraph +{ +public: + HardSwishSubgraph() + { + int input = addNodeToMatch(""); + int hardSigmoid = addNodeToMatch("HardSigmoid", input); + addNodeToMatch("Mul", input, hardSigmoid); + setFusedNode("HardSwish", input); + } + + virtual bool match(const Ptr& net, int nodeId, + std::vector& matchedNodesIds, + std::vector& targetNodesIds) CV_OVERRIDE + { + if (Subgraph::match(net, nodeId, matchedNodesIds, targetNodesIds)) + { + Ptr hardSigmoid = net->getNode(matchedNodesIds[0]); + opencv_onnx::NodeProto* node = hardSigmoid.dynamicCast()->node; + + uint8_t matched = 0; + for (int i = 0; i < node->attribute_size(); i++) + { + opencv_onnx::AttributeProto attr = node->attribute(i); + if ((attr.name() == "alpha" && attr.f() == 1.f / 6.f) || + (attr.name() == "beta" && attr.f() == 0.5f)) + { + ++matched; + } + } + return matched == 2; + } + return false; + } +}; + class NormalizeSubgraphBase : public Subgraph { public: @@ -625,6 +661,7 @@ void simplifySubgraphs(opencv_onnx::GraphProto& net) subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); + subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); diff --git a/modules/dnn/src/opencl/activations.cl b/modules/dnn/src/opencl/activations.cl index e110160c06..02ed9345c3 100644 --- a/modules/dnn/src/opencl/activations.cl +++ b/modules/dnn/src/opencl/activations.cl @@ -188,3 +188,87 @@ __kernel void NotForward(const int n, __global T* in, __global T* out) { if(index < n) out[index] = floor(1.0f - in[index]); } + +__kernel void AcosForward(const int n, __global T* in, __global T* out) { + int index = get_global_id(0); + if(index < n) + out[index] = acos(in[index]); +} + +__kernel void AcoshForward(const int n, __global T* in, __global T* out) { + int index = get_global_id(0); + if(index < n) + out[index] = acosh(in[index]); +} + +__kernel void AsinForward(const int n, __global T* in, __global T* out) { + int index = get_global_id(0); + if(index < n) + out[index] = asin(in[index]); +} + +__kernel void AsinhForward(const int n, __global T* in, __global T* out) { + int index = get_global_id(0); + if(index < n) + out[index] = asinh(in[index]); +} + +__kernel void AtanForward(const int n, __global T* in, __global T* out) { + int index = get_global_id(0); + if(index < n) + out[index] = atan(in[index]); +} + +__kernel void AtanhForward(const int n, __global T* in, __global T* out) { + int index = get_global_id(0); + if(index < n) + out[index] = atanh(in[index]); +} + +__kernel void CosForward(const int n, __global T* in, __global T* out) { + int index = get_global_id(0); + if(index < n) + out[index] = cos(in[index]); +} + +__kernel void CoshForward(const int n, __global T* in, __global T* out) { + int index = get_global_id(0); + if(index < n) + out[index] = cosh(in[index]); +} + +__kernel void HardSwishForward(const int n, __global T* in, __global T* out) { + int index = get_global_id(0); + if(index < n) + out[index] = in[index] * max(0.f, min(1.f, in[index] / 6.f + 0.5f)); +} + +__kernel void SinForward(const int n, __global T* in, __global T* out) { + int index = get_global_id(0); + if(index < n) + out[index] = sin(in[index]); +} + +__kernel void SinhForward(const int n, __global T* in, __global T* out) { + int index = get_global_id(0); + if(index < n) + out[index] = sinh(in[index]); +} + +__kernel void SoftplusForward(const int n, __global T* in, __global T* out) { + int index = get_global_id(0); + if(index < n) + out[index] = log1p(exp(in[index])); +} + +__kernel void SoftsignForward(const int n, __global T* in, __global T* out) { + int index = get_global_id(0); + if(index < n) + out[index] = in[index] / (1.f + fabs(in[index])); +} + +__kernel void TanForward(const int n, __global T* in, __global T* out) { + int index = get_global_id(0); + if(index < n) + out[index] = tan(in[index]); +} diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__cuda_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__cuda_denylist.inl.hpp index 61a5843b35..0f5f387132 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter__cuda_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__cuda_denylist.inl.hpp @@ -71,4 +71,5 @@ "test_softmax_large_number_expanded", // FP16 only "test_sub_bcast", "test_sub_uint8", +"test_tan", // FP16 only "test_upsample_nearest", diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp16_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp16_denylist.inl.hpp index 573d847985..ccd1568845 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp16_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp16_denylist.inl.hpp @@ -19,3 +19,4 @@ "test_split_equal_parts_1d", "test_split_equal_parts_2d", "test_split_equal_parts_default_axis", +"test_tan", diff --git a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp index b95cafd5d7..a69ace0d14 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp @@ -1,9 +1,5 @@ // The file is autogenerated // Update note: execute /testdata/dnn/onnx/generate_conformance_list.py -"test_acos", -"test_acos_example", -"test_acosh", -"test_acosh_example", "test_adagrad", "test_adagrad_multiple", "test_adam", @@ -16,14 +12,6 @@ "test_and_bcast4v2d", "test_and_bcast4v3d", "test_and_bcast4v4d", -"test_asin", -"test_asin_example", -"test_asinh", -"test_asinh_example", -"test_atan", -"test_atan_example", -"test_atanh", -"test_atanh_example", "test_basic_convinteger", "test_batchnorm_epsilon", "test_batchnorm_epsilon_training_mode", @@ -102,10 +90,6 @@ "test_convtranspose_pad", "test_convtranspose_pads", "test_convtranspose_with_kernel", -"test_cos", -"test_cos_example", -"test_cosh", -"test_cosh_example", "test_cumsum_1d", "test_cumsum_1d_exclusive", "test_cumsum_1d_reverse", @@ -138,7 +122,6 @@ "test_einsum_transpose", "test_equal", "test_equal_bcast", -"test_erf", "test_expand_dim_changed", "test_expand_dim_unchanged", "test_eyelike_populate_off_main_diagonal", @@ -193,8 +176,6 @@ "test_hardsigmoid", "test_hardsigmoid_default", "test_hardsigmoid_example", -"test_hardswish", -"test_hardswish_expanded", "test_identity_opt", "test_identity_sequence", "test_if", @@ -564,10 +545,6 @@ "test_simple_rnn_batchwise", "test_simple_rnn_defaults", "test_simple_rnn_with_initial_bias", -"test_sin", -"test_sin_example", -"test_sinh", -"test_sinh_example", "test_size", "test_size_example", "test_slice", @@ -578,10 +555,6 @@ "test_slice_neg_steps", "test_slice_negative_axes", "test_slice_start_out_of_bounds", -"test_softplus", -"test_softplus_example", -"test_softsign", -"test_softsign_example", "test_spacetodepth", "test_spacetodepth_example", "test_split_variable_parts_1d", @@ -599,8 +572,6 @@ "test_sub_example", "test_sum_example", "test_sum_two_inputs", -"test_tan", -"test_tan_example", "test_tfidfvectorizer_tf_batch_onlybigrams_skip0", "test_tfidfvectorizer_tf_batch_onlybigrams_skip5", "test_tfidfvectorizer_tf_batch_uniandbigrams_skip5", From 91817bffe178fcbb93c59c8f1d6c931faece6e77 Mon Sep 17 00:00:00 2001 From: Stefano Allegretti Date: Fri, 17 Dec 2021 20:36:57 +0100 Subject: [PATCH 187/226] Merge pull request #21275 from stal12:CCL_improvements Improve CCL with new algorithms and tests * Improve CCL with new algorithms and tests * Split CCL test into dedicated tests cases --- doc/opencv.bib | 8 + modules/imgproc/include/opencv2/imgproc.hpp | 18 +- modules/imgproc/src/connectedcomponents.cpp | 1393 ++++++++++++++++- .../imgproc/test/test_connectedcomponents.cpp | 501 +++++- 4 files changed, 1852 insertions(+), 68 deletions(-) diff --git a/doc/opencv.bib b/doc/opencv.bib index 4a5cafd2d8..bace4c68c9 100644 --- a/doc/opencv.bib +++ b/doc/opencv.bib @@ -1278,3 +1278,11 @@ pages={281--305}, year={1987} } +@article{Bolelli2021, + title={One DAG to Rule Them All}, + author={Bolelli, Federico and Allegretti, Stefano and Grana, Costantino}, + journal={IEEE Transactions on Pattern Analysis and Machine Intelligence}, + year={2021}, + publisher={IEEE}, + doi = {10.1109/TPAMI.2021.3055337} +} diff --git a/modules/imgproc/include/opencv2/imgproc.hpp b/modules/imgproc/include/opencv2/imgproc.hpp index c669f2cdef..98b074d46d 100644 --- a/modules/imgproc/include/opencv2/imgproc.hpp +++ b/modules/imgproc/include/opencv2/imgproc.hpp @@ -406,10 +406,10 @@ enum ConnectedComponentsTypes { //! connected components algorithm enum ConnectedComponentsAlgorithmsTypes { - CCL_DEFAULT = -1, //!< BBDT @cite Grana2010 algorithm for 8-way connectivity, SAUF algorithm for 4-way connectivity. The parallel implementation described in @cite Bolelli2017 is available for both BBDT and SAUF. + CCL_DEFAULT = -1, //!< Spaghetti @cite Bolelli2019 algorithm for 8-way connectivity, Spaghetti4C @cite Bolelli2021 algorithm for 4-way connectivity. CCL_WU = 0, //!< SAUF @cite Wu2009 algorithm for 8-way connectivity, SAUF algorithm for 4-way connectivity. The parallel implementation described in @cite Bolelli2017 is available for SAUF. CCL_GRANA = 1, //!< BBDT @cite Grana2010 algorithm for 8-way connectivity, SAUF algorithm for 4-way connectivity. The parallel implementation described in @cite Bolelli2017 is available for both BBDT and SAUF. - CCL_BOLELLI = 2, //!< Spaghetti @cite Bolelli2019 algorithm for 8-way connectivity, SAUF algorithm for 4-way connectivity. + CCL_BOLELLI = 2, //!< Spaghetti @cite Bolelli2019 algorithm for 8-way connectivity, Spaghetti4C @cite Bolelli2021 algorithm for 4-way connectivity. The parallel implementation described in @cite Bolelli2017 is available for both Spaghetti and Spaghetti4C. CCL_SAUF = 3, //!< Same as CCL_WU. It is preferable to use the flag with the name of the algorithm (CCL_SAUF) rather than the one with the name of the first author (CCL_WU). CCL_BBDT = 4, //!< Same as CCL_GRANA. It is preferable to use the flag with the name of the algorithm (CCL_BBDT) rather than the one with the name of the first author (CCL_GRANA). CCL_SPAGHETTI = 5, //!< Same as CCL_BOLELLI. It is preferable to use the flag with the name of the algorithm (CCL_SPAGHETTI) rather than the one with the name of the first author (CCL_BOLELLI). @@ -3912,9 +3912,10 @@ image with 4 or 8 way connectivity - returns N, the total number of labels [0, N represents the background label. ltype specifies the output label image type, an important consideration based on the total number of labels or alternatively the total number of pixels in the source image. ccltype specifies the connected components labeling algorithm to use, currently -Grana (BBDT) and Wu's (SAUF) @cite Wu2009 algorithms are supported, see the #ConnectedComponentsAlgorithmsTypes -for details. Note that SAUF algorithm forces a row major ordering of labels while BBDT does not. -This function uses parallel version of both Grana and Wu's algorithms if at least one allowed +Bolelli (Spaghetti) @cite Bolelli2019, Grana (BBDT) @cite Grana2010 and Wu's (SAUF) @cite Wu2009 algorithms +are supported, see the #ConnectedComponentsAlgorithmsTypes for details. Note that SAUF algorithm forces +a row major ordering of labels while Spaghetti and BBDT do not. +This function uses parallel version of the algorithms if at least one allowed parallel framework is enabled and if the rows of the image are at least twice the number returned by #getNumberOfCPUs. @param image the 8-bit single-channel image to be labeled @@ -3944,9 +3945,10 @@ image with 4 or 8 way connectivity - returns N, the total number of labels [0, N represents the background label. ltype specifies the output label image type, an important consideration based on the total number of labels or alternatively the total number of pixels in the source image. ccltype specifies the connected components labeling algorithm to use, currently -Grana's (BBDT) and Wu's (SAUF) @cite Wu2009 algorithms are supported, see the #ConnectedComponentsAlgorithmsTypes -for details. Note that SAUF algorithm forces a row major ordering of labels while BBDT does not. -This function uses parallel version of both Grana and Wu's algorithms (statistics included) if at least one allowed +Bolelli (Spaghetti) @cite Bolelli2019, Grana (BBDT) @cite Grana2010 and Wu's (SAUF) @cite Wu2009 algorithms +are supported, see the #ConnectedComponentsAlgorithmsTypes for details. Note that SAUF algorithm forces +a row major ordering of labels while Spaghetti and BBDT do not. +This function uses parallel version of the algorithms (statistics included) if at least one allowed parallel framework is enabled and if the rows of the image are at least twice the number returned by #getNumberOfCPUs. @param image the 8-bit single-channel image to be labeled diff --git a/modules/imgproc/src/connectedcomponents.cpp b/modules/imgproc/src/connectedcomponents.cpp index e66766c872..1ad74ed38a 100644 --- a/modules/imgproc/src/connectedcomponents.cpp +++ b/modules/imgproc/src/connectedcomponents.cpp @@ -287,9 +287,898 @@ namespace cv{ return LT((y /*+ 1*/) / 2) * LT((w + 1) / 2) + 1; } - //Implementation of Spaghetti algorithm, as described in "Spaghetti Labeling: Directed Acyclic Graphs for Block-Based - //Connected Components Labeling" (only for 8-connectivity) - //Federico Bolelli et. al. + //Parallel implementation of Spaghetti algorithm, described in "Spaghetti Labeling: Directed Acyclic Graphs + //for Block-Based Connected Components Labeling", IEEE Transactions on Image Processing, Federico Bolelli et. al. + //Parallelization method described in "Two More Strategies to Speed Up Connected Components Labeling Algorithms", + //Image Analysis and Processing - ICIAP 2017, Federico Bolelli et. al. + template + struct LabelingBolelliParallel { + + class FirstScan : public cv::ParallelLoopBody { + private: + const cv::Mat& img_; + cv::Mat& imgLabels_; + LabelT* P_; + int* chunksSizeAndLabels_; + + public: + FirstScan(const cv::Mat& img, cv::Mat& imgLabels, LabelT* P, int* chunksSizeAndLabels) + : img_(img), imgLabels_(imgLabels), P_(P), chunksSizeAndLabels_(chunksSizeAndLabels) {} + + FirstScan& operator=(const FirstScan&) { return *this; } + + void operator()(const cv::Range& range2) const CV_OVERRIDE + { + const Range range(range2.start * 2, std::min(range2.end * 2, img_.rows)); + + const int startR = range.start; + chunksSizeAndLabels_[startR] = range.end; + + LabelT label = stripeFirstLabel8Connectivity(startR, imgLabels_.cols); + + const LabelT firstLabel = label; + //const int h = img_.rows; + const int w = img_.cols; + const int stripe_h = range.end - range.start; + //const int limitLine = startR + 1; + + const int e_rows = stripe_h & -2; + const bool o_rows = stripe_h % 2 == 1; + //const int e_cols = w & -2; + //const bool o_cols = w % 2 == 1; + + { +#define CONDITION_B img_row_prev_prev[c-1]>0 +#define CONDITION_C img_row_prev_prev[c]>0 +#define CONDITION_D img_row_prev_prev[c+1]>0 +#define CONDITION_E img_row_prev_prev[c+2]>0 + +#define CONDITION_G img_row_prev[c-2]>0 +#define CONDITION_H img_row_prev[c-1]>0 +#define CONDITION_I img_row_prev[c]>0 +#define CONDITION_J img_row_prev[c+1]>0 +#define CONDITION_K img_row_prev[c+2]>0 + +#define CONDITION_M img_row[c-2]>0 +#define CONDITION_N img_row[c-1]>0 +#define CONDITION_O img_row[c]>0 +#define CONDITION_P img_row[c+1]>0 + +#define CONDITION_R img_row_fol[c-1]>0 +#define CONDITION_S img_row_fol[c]>0 +#define CONDITION_T img_row_fol[c+1]>0 + + // Action 1: No action +#define ACTION_1 img_labels_row[c] = 0; +// Action 2: New label (the block has foreground pixels and is not connected to anything else) +#define ACTION_2 img_labels_row[c] = label; \ + P_[label] = label; \ + label = label + 1; +//Action 3: Assign label of block P +#define ACTION_3 img_labels_row[c] = img_labels_row_prev_prev[c - 2]; +// Action 4: Assign label of block Q +#define ACTION_4 img_labels_row[c] = img_labels_row_prev_prev[c]; +// Action 5: Assign label of block R +#define ACTION_5 img_labels_row[c] = img_labels_row_prev_prev[c + 2]; +// Action 6: Assign label of block S +#define ACTION_6 img_labels_row[c] = img_labels_row[c - 2]; +// Action 7: Merge labels of block P and Q +#define ACTION_7 img_labels_row[c] = set_union(P_, img_labels_row_prev_prev[c - 2], img_labels_row_prev_prev[c]); +//Action 8: Merge labels of block P and R +#define ACTION_8 img_labels_row[c] = set_union(P_, img_labels_row_prev_prev[c - 2], img_labels_row_prev_prev[c + 2]); +// Action 9 Merge labels of block P and S +#define ACTION_9 img_labels_row[c] = set_union(P_, img_labels_row_prev_prev[c - 2], img_labels_row[c - 2]); +// Action 10 Merge labels of block Q and R +#define ACTION_10 img_labels_row[c] = set_union(P_, img_labels_row_prev_prev[c], img_labels_row_prev_prev[c + 2]); +// Action 11: Merge labels of block Q and S +#define ACTION_11 img_labels_row[c] = set_union(P_, img_labels_row_prev_prev[c], img_labels_row[c - 2]); +// Action 12: Merge labels of block R and S +#define ACTION_12 img_labels_row[c] = set_union(P_, img_labels_row_prev_prev[c + 2], img_labels_row[c - 2]); +// Action 13: Merge labels of block P, Q and R +#define ACTION_13 img_labels_row[c] = set_union(P_, set_union(P_, img_labels_row_prev_prev[c - 2], img_labels_row_prev_prev[c]), img_labels_row_prev_prev[c + 2]); +// Action 14: Merge labels of block P, Q and S +#define ACTION_14 img_labels_row[c] = set_union(P_, set_union(P_, img_labels_row_prev_prev[c - 2], img_labels_row_prev_prev[c]), img_labels_row[c - 2]); +//Action 15: Merge labels of block P, R and S +#define ACTION_15 img_labels_row[c] = set_union(P_, set_union(P_, img_labels_row_prev_prev[c - 2], img_labels_row_prev_prev[c + 2]), img_labels_row[c - 2]); +//Action 16: labels of block Q, R and S +#define ACTION_16 img_labels_row[c] = set_union(P_, set_union(P_, img_labels_row_prev_prev[c], img_labels_row_prev_prev[c + 2]), img_labels_row[c - 2]); + } + // The following Directed Rooted Acyclic Graphs (DAGs) allow to choose which action to + // perform, checking as few conditions as possible. Special DAGs are used for the first/last + // line of the image and for single line images. Actions: the blocks label are provisionally + // stored in the top left pixel of the block in the labels image. + if (stripe_h == 1) { + // Single line + const PixelT* const img_row = img_.ptr(startR); + LabelT* const img_labels_row = imgLabels_.ptr(startR); + int c = -2; +#include "ccl_bolelli_forest_singleline.inc.hpp" + } + else { + // More than one line + + // First couple of lines + { + const PixelT* const img_row = img_.ptr(startR); + const PixelT* const img_row_fol = (PixelT*)(((char*)img_row) + img_.step.p[0]); + LabelT* const img_labels_row = imgLabels_.ptr(startR); + int c = -2; +#include "ccl_bolelli_forest_firstline.inc.hpp" + } + + // Every other line but the last one if image has an odd number of rows + for (int r = startR + 2; r < startR + e_rows; r += 2) { + // Get rows pointer + const PixelT* const img_row = img_.ptr(r); + const PixelT* const img_row_prev = (PixelT*)(((char*)img_row) - img_.step.p[0]); + const PixelT* const img_row_prev_prev = (PixelT*)(((char*)img_row_prev) - img_.step.p[0]); + const PixelT* const img_row_fol = (PixelT*)(((char*)img_row) + img_.step.p[0]); + LabelT* const img_labels_row = imgLabels_.ptr(r); + LabelT* const img_labels_row_prev_prev = (LabelT*)(((char*)img_labels_row) - imgLabels_.step.p[0] - imgLabels_.step.p[0]); + + int c = -2; + goto tree_0; + +#include "ccl_bolelli_forest.inc.hpp" + } + + // Last line (in case the rows are odd) + if (o_rows) { + const int r = startR + stripe_h - 1; + const PixelT* const img_row = img_.ptr(r); + const PixelT* const img_row_prev = (PixelT*)(((char*)img_row) - img_.step.p[0]); + const PixelT* const img_row_prev_prev = (PixelT*)(((char*)img_row_prev) - img_.step.p[0]); + LabelT* const img_labels_row = imgLabels_.ptr(r); + LabelT* const img_labels_row_prev_prev = (LabelT*)(((char*)img_labels_row) - imgLabels_.step.p[0] - imgLabels_.step.p[0]); + int c = -2; +#include "ccl_bolelli_forest_lastline.inc.hpp" + } + } + + //write in the follower memory location + chunksSizeAndLabels_[startR + 1] = label - firstLabel; + + // undef conditions and actions + { +#undef ACTION_1 +#undef ACTION_2 +#undef ACTION_3 +#undef ACTION_4 +#undef ACTION_5 +#undef ACTION_6 +#undef ACTION_7 +#undef ACTION_8 +#undef ACTION_9 +#undef ACTION_10 +#undef ACTION_11 +#undef ACTION_12 +#undef ACTION_13 +#undef ACTION_14 +#undef ACTION_15 +#undef ACTION_16 + +#undef CONDITION_B +#undef CONDITION_C +#undef CONDITION_D +#undef CONDITION_E + +#undef CONDITION_G +#undef CONDITION_H +#undef CONDITION_I +#undef CONDITION_J +#undef CONDITION_K + +#undef CONDITION_M +#undef CONDITION_N +#undef CONDITION_O +#undef CONDITION_P + +#undef CONDITION_R +#undef CONDITION_S +#undef CONDITION_T + } + } + }; + + class SecondScan : public cv::ParallelLoopBody { + private: + const cv::Mat& img_; + cv::Mat& imgLabels_; + LabelT* P_; + StatsOp& sop_; + StatsOp* sopArray_; + LabelT& nLabels_; + + public: + SecondScan(const cv::Mat& img, cv::Mat& imgLabels, LabelT* P, StatsOp& sop, StatsOp* sopArray, LabelT& nLabels) + : img_(img), imgLabels_(imgLabels), P_(P), sop_(sop), sopArray_(sopArray), nLabels_(nLabels) {} + + void operator()(const cv::Range& range2) const CV_OVERRIDE + { + const Range range(range2.start * 2, std::min(range2.end * 2, img_.rows)); + int r = range.start; + + const int rowBegin = r; + const int rowEnd = range.end; + + if (rowBegin > 0) { + sopArray_[rowBegin].initElement(nLabels_); + sopArray_[rowBegin].setNextLoc(rowEnd); //_nextLoc = rowEnd; + + if (imgLabels_.rows & 1) { + if (imgLabels_.cols & 1) { + //Case 1: both rows and cols odd + for (; r < rowEnd; r += 2) { + // Get rows pointer + const PixelT* const img_row = img_.ptr(r); + const PixelT* const img_row_fol = (PixelT*)(((char*)img_row) + img_.step.p[0]); + + LabelT* const imgLabels_row = imgLabels_.ptr(r); + LabelT* const imgLabels_row_fol = (LabelT*)(((char*)imgLabels_row) + imgLabels_.step.p[0]); + // Get rows pointer + for (int c = 0; c < imgLabels_.cols; c += 2) { + LabelT iLabel = imgLabels_row[c]; + if (iLabel > 0) { + iLabel = P_[iLabel]; + if (img_row[c] > 0) { + imgLabels_row[c] = iLabel; + sopArray_[rowBegin](r, c, iLabel); + } + else { + imgLabels_row[c] = 0; + sopArray_[rowBegin](r, c, 0); + } + if (c + 1 < imgLabels_.cols) { + if (img_row[c + 1] > 0) { + imgLabels_row[c + 1] = iLabel; + sopArray_[rowBegin](r, c + 1, iLabel); + } + else { + imgLabels_row[c + 1] = 0; + sopArray_[rowBegin](r, c + 1, 0); + } + if (r + 1 < imgLabels_.rows) { + if (img_row_fol[c] > 0) { + imgLabels_row_fol[c] = iLabel; + sopArray_[rowBegin](r + 1, c, iLabel); + } + else { + imgLabels_row_fol[c] = 0; + sopArray_[rowBegin](r + 1, c, 0); + } + if (img_row_fol[c + 1] > 0) { + imgLabels_row_fol[c + 1] = iLabel; + sopArray_[rowBegin](r + 1, c + 1, iLabel); + } + else { + imgLabels_row_fol[c + 1] = 0; + sopArray_[rowBegin](r + 1, c + 1, 0); + } + } + } + else if (r + 1 < imgLabels_.rows) { + if (img_row_fol[c] > 0) { + imgLabels_row_fol[c] = iLabel; + sopArray_[rowBegin](r + 1, c, iLabel); + } + else { + imgLabels_row_fol[c] = 0; + sopArray_[rowBegin](r + 1, c, 0); + } + } + } + else { + imgLabels_row[c] = 0; + sopArray_[rowBegin](r, c, 0); + if (c + 1 < imgLabels_.cols) { + imgLabels_row[c + 1] = 0; + sopArray_[rowBegin](r, c + 1, 0); + if (r + 1 < imgLabels_.rows) { + imgLabels_row_fol[c] = 0; + imgLabels_row_fol[c + 1] = 0; + sopArray_[rowBegin](r + 1, c, 0); + sopArray_[rowBegin](r + 1, c + 1, 0); + } + } + else if (r + 1 < imgLabels_.rows) { + imgLabels_row_fol[c] = 0; + sopArray_[rowBegin](r + 1, c, 0); + } + } + } + } + }//END Case 1 + else { + //Case 2: only rows odd + for (; r < rowEnd; r += 2) { + // Get rows pointer + const PixelT* const img_row = img_.ptr(r); + const PixelT* const img_row_fol = (PixelT*)(((char*)img_row) + img_.step.p[0]); + LabelT* const imgLabels_row = imgLabels_.ptr(r); + LabelT* const imgLabels_row_fol = (LabelT*)(((char*)imgLabels_row) + imgLabels_.step.p[0]); + // Get rows pointer + for (int c = 0; c < imgLabels_.cols; c += 2) { + LabelT iLabel = imgLabels_row[c]; + if (iLabel > 0) { + iLabel = P_[iLabel]; + if (img_row[c] > 0) { + imgLabels_row[c] = iLabel; + sopArray_[rowBegin](r, c, iLabel); + } + else { + imgLabels_row[c] = 0; + sopArray_[rowBegin](r, c, 0); + } + if (img_row[c + 1] > 0) { + imgLabels_row[c + 1] = iLabel; + sopArray_[rowBegin](r, c + 1, iLabel); + } + else { + imgLabels_row[c + 1] = 0; + sopArray_[rowBegin](r, c + 1, 0); + } + if (r + 1 < imgLabels_.rows) { + if (img_row_fol[c] > 0) { + imgLabels_row_fol[c] = iLabel; + sopArray_[rowBegin](r + 1, c, iLabel); + } + else { + imgLabels_row_fol[c] = 0; + sopArray_[rowBegin](r + 1, c, 0); + } + if (img_row_fol[c + 1] > 0) { + imgLabels_row_fol[c + 1] = iLabel; + sopArray_[rowBegin](r + 1, c + 1, iLabel); + } + else { + imgLabels_row_fol[c + 1] = 0; + sopArray_[rowBegin](r + 1, c + 1, 0); + } + } + } + else { + imgLabels_row[c] = 0; + imgLabels_row[c + 1] = 0; + sopArray_[rowBegin](r, c, 0); + sopArray_[rowBegin](r, c + 1, 0); + if (r + 1 < imgLabels_.rows) { + imgLabels_row_fol[c] = 0; + imgLabels_row_fol[c + 1] = 0; + sopArray_[rowBegin](r + 1, c, 0); + sopArray_[rowBegin](r + 1, c + 1, 0); + } + } + } + } + }// END Case 2 + } + else { + if (imgLabels_.cols & 1) { + //Case 3: only cols odd + for (; r < rowEnd; r += 2) { + // Get rows pointer + const PixelT* const img_row = img_.ptr(r); + const PixelT* const img_row_fol = (PixelT*)(((char*)img_row) + img_.step.p[0]); + LabelT* const imgLabels_row = imgLabels_.ptr(r); + LabelT* const imgLabels_row_fol = (LabelT*)(((char*)imgLabels_row) + imgLabels_.step.p[0]); + // Get rows pointer + for (int c = 0; c < imgLabels_.cols; c += 2) { + LabelT iLabel = imgLabels_row[c]; + if (iLabel > 0) { + iLabel = P_[iLabel]; + if (img_row[c] > 0) { + imgLabels_row[c] = iLabel; + sopArray_[rowBegin](r, c, iLabel); + } + else { + imgLabels_row[c] = 0; + sopArray_[rowBegin](r, c, 0); + } + if (img_row_fol[c] > 0) { + imgLabels_row_fol[c] = iLabel; + sopArray_[rowBegin](r + 1, c, iLabel); + } + else { + imgLabels_row_fol[c] = 0; + sopArray_[rowBegin](r + 1, c, 0); + } + if (c + 1 < imgLabels_.cols) { + if (img_row[c + 1] > 0) { + imgLabels_row[c + 1] = iLabel; + sopArray_[rowBegin](r, c + 1, iLabel); + } + else { + imgLabels_row[c + 1] = 0; + sopArray_[rowBegin](r, c + 1, 0); + } + if (img_row_fol[c + 1] > 0) { + imgLabels_row_fol[c + 1] = iLabel; + sopArray_[rowBegin](r + 1, c + 1, iLabel); + } + else { + imgLabels_row_fol[c + 1] = 0; + sopArray_[rowBegin](r + 1, c + 1, 0); + } + } + } + else { + imgLabels_row[c] = 0; + imgLabels_row_fol[c] = 0; + sopArray_[rowBegin](r, c, 0); + sopArray_[rowBegin](r + 1, c, 0); + if (c + 1 < imgLabels_.cols) { + imgLabels_row[c + 1] = 0; + imgLabels_row_fol[c + 1] = 0; + sopArray_[rowBegin](r, c + 1, 0); + sopArray_[rowBegin](r + 1, c + 1, 0); + } + } + } + } + }// END case 3 + else { + //Case 4: nothing odd + for (; r < rowEnd; r += 2) { + // Get rows pointer + const PixelT* const img_row = img_.ptr(r); + const PixelT* const img_row_fol = (PixelT*)(((char*)img_row) + img_.step.p[0]); + LabelT* const imgLabels_row = imgLabels_.ptr(r); + LabelT* const imgLabels_row_fol = (LabelT*)(((char*)imgLabels_row) + imgLabels_.step.p[0]); + // Get rows pointer + for (int c = 0; c < imgLabels_.cols; c += 2) { + LabelT iLabel = imgLabels_row[c]; + if (iLabel > 0) { + iLabel = P_[iLabel]; + if (img_row[c] > 0) { + imgLabels_row[c] = iLabel; + sopArray_[rowBegin](r, c, iLabel); + } + else { + imgLabels_row[c] = 0; + sopArray_[rowBegin](r, c, 0); + } + if (img_row[c + 1] > 0) { + imgLabels_row[c + 1] = iLabel; + sopArray_[rowBegin](r, c + 1, iLabel); + } + else { + imgLabels_row[c + 1] = 0; + sopArray_[rowBegin](r, c + 1, 0); + } + if (img_row_fol[c] > 0) { + imgLabels_row_fol[c] = iLabel; + sopArray_[rowBegin](r + 1, c, iLabel); + } + else { + imgLabels_row_fol[c] = 0; + sopArray_[rowBegin](r + 1, c, 0); + } + if (img_row_fol[c + 1] > 0) { + imgLabels_row_fol[c + 1] = iLabel; + sopArray_[rowBegin](r + 1, c + 1, iLabel); + } + else { + imgLabels_row_fol[c + 1] = 0; + sopArray_[rowBegin](r + 1, c + 1, 0); + } + } + else { + imgLabels_row[c] = 0; + imgLabels_row[c + 1] = 0; + imgLabels_row_fol[c] = 0; + imgLabels_row_fol[c + 1] = 0; + sopArray_[rowBegin](r, c, 0); + sopArray_[rowBegin](r, c + 1, 0); + sopArray_[rowBegin](r + 1, c, 0); + sopArray_[rowBegin](r + 1, c + 1, 0); + } + } + }//END case 4 + } + } + } + else { + //the first thread uses sop in order to make less merges + sop_.setNextLoc(rowEnd); + if (imgLabels_.rows & 1) { + if (imgLabels_.cols & 1) { + //Case 1: both rows and cols odd + for (; r < rowEnd; r += 2) { + // Get rows pointer + const PixelT* const img_row = img_.ptr(r); + const PixelT* const img_row_fol = (PixelT*)(((char*)img_row) + img_.step.p[0]); + + LabelT* const imgLabels_row = imgLabels_.ptr(r); + LabelT* const imgLabels_row_fol = (LabelT*)(((char*)imgLabels_row) + imgLabels_.step.p[0]); + // Get rows pointer + for (int c = 0; c < imgLabels_.cols; c += 2) { + LabelT iLabel = imgLabels_row[c]; + if (iLabel > 0) { + iLabel = P_[iLabel]; + if (img_row[c] > 0) { + imgLabels_row[c] = iLabel; + sop_(r, c, iLabel); + } + else { + imgLabels_row[c] = 0; + sop_(r, c, 0); + } + if (c + 1 < imgLabels_.cols) { + if (img_row[c + 1] > 0) { + imgLabels_row[c + 1] = iLabel; + sop_(r, c + 1, iLabel); + } + else { + imgLabels_row[c + 1] = 0; + sop_(r, c + 1, 0); + } + if (r + 1 < imgLabels_.rows) { + if (img_row_fol[c] > 0) { + imgLabels_row_fol[c] = iLabel; + sop_(r + 1, c, iLabel); + } + else { + imgLabels_row_fol[c] = 0; + sop_(r + 1, c, 0); + } + if (img_row_fol[c + 1] > 0) { + imgLabels_row_fol[c + 1] = iLabel; + sop_(r + 1, c + 1, iLabel); + } + else { + imgLabels_row_fol[c + 1] = 0; + sop_(r + 1, c + 1, 0); + } + } + } + else if (r + 1 < imgLabels_.rows) { + if (img_row_fol[c] > 0) { + imgLabels_row_fol[c] = iLabel; + sop_(r + 1, c, iLabel); + } + else { + imgLabels_row_fol[c] = 0; + sop_(r + 1, c, 0); + } + } + } + else { + imgLabels_row[c] = 0; + sop_(r, c, 0); + if (c + 1 < imgLabels_.cols) { + imgLabels_row[c + 1] = 0; + sop_(r, c + 1, 0); + if (r + 1 < imgLabels_.rows) { + imgLabels_row_fol[c] = 0; + imgLabels_row_fol[c + 1] = 0; + sop_(r + 1, c, 0); + sop_(r + 1, c + 1, 0); + } + } + else if (r + 1 < imgLabels_.rows) { + imgLabels_row_fol[c] = 0; + sop_(r + 1, c, 0); + } + } + } + } + }//END Case 1 + else { + //Case 2: only rows odd + for (; r < rowEnd; r += 2) { + // Get rows pointer + const PixelT* const img_row = img_.ptr(r); + const PixelT* const img_row_fol = (PixelT*)(((char*)img_row) + img_.step.p[0]); + LabelT* const imgLabels_row = imgLabels_.ptr(r); + LabelT* const imgLabels_row_fol = (LabelT*)(((char*)imgLabels_row) + imgLabels_.step.p[0]); + // Get rows pointer + for (int c = 0; c < imgLabels_.cols; c += 2) { + LabelT iLabel = imgLabels_row[c]; + if (iLabel > 0) { + iLabel = P_[iLabel]; + if (img_row[c] > 0) { + imgLabels_row[c] = iLabel; + sop_(r, c, iLabel); + } + else { + imgLabels_row[c] = 0; + sop_(r, c, 0); + } + if (img_row[c + 1] > 0) { + imgLabels_row[c + 1] = iLabel; + sop_(r, c + 1, iLabel); + } + else { + imgLabels_row[c + 1] = 0; + sop_(r, c + 1, 0); + } + if (r + 1 < imgLabels_.rows) { + if (img_row_fol[c] > 0) { + imgLabels_row_fol[c] = iLabel; + sop_(r + 1, c, iLabel); + } + else { + imgLabels_row_fol[c] = 0; + sop_(r + 1, c, 0); + } + if (img_row_fol[c + 1] > 0) { + imgLabels_row_fol[c + 1] = iLabel; + sop_(r + 1, c + 1, iLabel); + } + else { + imgLabels_row_fol[c + 1] = 0; + sop_(r + 1, c + 1, 0); + } + } + } + else { + imgLabels_row[c] = 0; + imgLabels_row[c + 1] = 0; + sop_(r, c, 0); + sop_(r, c + 1, 0); + if (r + 1 < imgLabels_.rows) { + imgLabels_row_fol[c] = 0; + imgLabels_row_fol[c + 1] = 0; + sop_(r + 1, c, 0); + sop_(r + 1, c + 1, 0); + } + } + } + } + }// END Case 2 + } + else { + if (imgLabels_.cols & 1) { + //Case 3: only cols odd + for (; r < rowEnd; r += 2) { + // Get rows pointer + const PixelT* const img_row = img_.ptr(r); + const PixelT* const img_row_fol = (PixelT*)(((char*)img_row) + img_.step.p[0]); + LabelT* const imgLabels_row = imgLabels_.ptr(r); + LabelT* const imgLabels_row_fol = (LabelT*)(((char*)imgLabels_row) + imgLabels_.step.p[0]); + // Get rows pointer + for (int c = 0; c < imgLabels_.cols; c += 2) { + LabelT iLabel = imgLabels_row[c]; + if (iLabel > 0) { + iLabel = P_[iLabel]; + if (img_row[c] > 0) { + imgLabels_row[c] = iLabel; + sop_(r, c, iLabel); + } + else { + imgLabels_row[c] = 0; + sop_(r, c, 0); + } + if (img_row_fol[c] > 0) { + imgLabels_row_fol[c] = iLabel; + sop_(r + 1, c, iLabel); + } + else { + imgLabels_row_fol[c] = 0; + sop_(r + 1, c, 0); + } + if (c + 1 < imgLabels_.cols) { + if (img_row[c + 1] > 0) { + imgLabels_row[c + 1] = iLabel; + sop_(r, c + 1, iLabel); + } + else { + imgLabels_row[c + 1] = 0; + sop_(r, c + 1, 0); + } + if (img_row_fol[c + 1] > 0) { + imgLabels_row_fol[c + 1] = iLabel; + sop_(r + 1, c + 1, iLabel); + } + else { + imgLabels_row_fol[c + 1] = 0; + sop_(r + 1, c + 1, 0); + } + } + } + else { + imgLabels_row[c] = 0; + imgLabels_row_fol[c] = 0; + sop_(r, c, 0); + sop_(r + 1, c, 0); + if (c + 1 < imgLabels_.cols) { + imgLabels_row[c + 1] = 0; + imgLabels_row_fol[c + 1] = 0; + sop_(r, c + 1, 0); + sop_(r + 1, c + 1, 0); + } + } + } + } + }// END case 3 + else { + //Case 4: nothing odd + for (; r < rowEnd; r += 2) { + // Get rows pointer + const PixelT* const img_row = img_.ptr(r); + const PixelT* const img_row_fol = (PixelT*)(((char*)img_row) + img_.step.p[0]); + LabelT* const imgLabels_row = imgLabels_.ptr(r); + LabelT* const imgLabels_row_fol = (LabelT*)(((char*)imgLabels_row) + imgLabels_.step.p[0]); + // Get rows pointer + for (int c = 0; c < imgLabels_.cols; c += 2) { + LabelT iLabel = imgLabels_row[c]; + if (iLabel > 0) { + iLabel = P_[iLabel]; + if (img_row[c] > 0) { + imgLabels_row[c] = iLabel; + sop_(r, c, iLabel); + } + else { + imgLabels_row[c] = 0; + sop_(r, c, 0); + } + if (img_row[c + 1] > 0) { + imgLabels_row[c + 1] = iLabel; + sop_(r, c + 1, iLabel); + } + else { + imgLabels_row[c + 1] = 0; + sop_(r, c + 1, 0); + } + if (img_row_fol[c] > 0) { + imgLabels_row_fol[c] = iLabel; + sop_(r + 1, c, iLabel); + } + else { + imgLabels_row_fol[c] = 0; + sop_(r + 1, c, 0); + } + if (img_row_fol[c + 1] > 0) { + imgLabels_row_fol[c + 1] = iLabel; + sop_(r + 1, c + 1, iLabel); + } + else { + imgLabels_row_fol[c + 1] = 0; + sop_(r + 1, c + 1, 0); + } + } + else { + imgLabels_row[c] = 0; + imgLabels_row[c + 1] = 0; + imgLabels_row_fol[c] = 0; + imgLabels_row_fol[c + 1] = 0; + sop_(r, c, 0); + sop_(r, c + 1, 0); + sop_(r + 1, c, 0); + sop_(r + 1, c + 1, 0); + } + } + }//END case 4 + } + } + } + } + }; + + inline static + void mergeLabels(const cv::Mat& img, cv::Mat& imgLabels, LabelT* P, int* chunksSizeAndLabels) { + + // Merge Mask + // +---+---+---+ + // |P -|Q -|R -| + // |- -|- -|- -| + // +---+---+---+ + // |X -| + // |- -| + // +---+ + const int w = imgLabels.cols, h = imgLabels.rows; + + for (int r = chunksSizeAndLabels[0]; r < h; r = chunksSizeAndLabels[r]) { + + LabelT* const imgLabels_row = imgLabels.ptr(r); + LabelT* const imgLabels_row_prev_prev = (LabelT*)(((char*)imgLabels_row) - imgLabels.step.p[0] - imgLabels.step.p[0]); + const PixelT* const img_row = img.ptr(r); + const PixelT* const img_row_prev = (PixelT*)(((char*)img_row) - img.step.p[0]); + + for (int c = 0; c < w; c += 2) { + +#define condition_x imgLabels_row[c] > 0 +#define condition_pppr c > 1 && imgLabels_row_prev_prev[c - 2] > 0 +#define condition_qppr imgLabels_row_prev_prev[c] > 0 +#define condition_qppr1 c < w - 1 +#define condition_qppr2 c < w +#define condition_rppr c < w - 2 && imgLabels_row_prev_prev[c + 2] > 0 + + if (condition_x) { + if (condition_pppr) { + //check in img + if (img_row[c] > 0 && img_row_prev[c - 1] > 0) + //assign the same label + imgLabels_row[c] = set_union(P, imgLabels_row_prev_prev[c - 2], imgLabels_row[c]); + } + if (condition_qppr) { + if (condition_qppr1) { + if ((img_row[c] > 0 && img_row_prev[c] > 0) || (img_row[c + 1] > 0 && img_row_prev[c] > 0) || + (img_row[c] > 0 && img_row_prev[c + 1] > 0) || (img_row[c + 1] > 0 && img_row_prev[c + 1] > 0)) { + imgLabels_row[c] = set_union(P, imgLabels_row_prev_prev[c], imgLabels_row[c]); + } + } + else /*if (condition_qppr2)*/ { + if (img_row[c] > 0 && img_row_prev[c] > 0) + imgLabels_row[c] = set_union(P, imgLabels_row_prev_prev[c], imgLabels_row[c]); + } + } + if (condition_rppr) { + if (img_row[c + 1] > 0 && img_row_prev[c + 2] > 0) + imgLabels_row[c] = set_union(P, imgLabels_row_prev_prev[c + 2], imgLabels_row[c]); + } + } + +#undef condition_x +#undef condition_pppr +#undef condition_qppr +#undef condition_qppr1 +#undef condition_qppr2 +#undef condition_rppr + + } + } + } + + LabelT operator()(const cv::Mat& img, cv::Mat& imgLabels, int connectivity, StatsOp& sop) { + CV_Assert(img.rows == imgLabels.rows); + CV_Assert(img.cols == imgLabels.cols); + CV_Assert(connectivity == 8); + + const int h = img.rows; + const int w = img.cols; + + //A quick and dirty upper bound for the maximum number of labels. + //Following formula comes from the fact that a 2x2 block in 8-connectivity case + //can never have more than 1 new label and 1 label for background. + //Worst case image example pattern: + //1 0 1 0 1... + //0 0 0 0 0... + //1 0 1 0 1... + //............ + const size_t Plength = size_t(((h + 1) / 2) * size_t((w + 1) / 2)) + 1; + + //Array used to store info and labeled pixel by each thread. + //Different threads affect different memory location of chunksSizeAndLabels + const int chunksSizeAndLabelsSize = roundUp(h, 2); + std::vector chunksSizeAndLabels(chunksSizeAndLabelsSize); + + //Tree of labels + std::vector P(Plength, 0); + //First label is for background + //P[0] = 0; + + cv::Range range2(0, divUp(h, 2)); + const double nParallelStripes = std::max(1, std::min(h / 2, getNumThreads() * 4)); + + //First scan + cv::parallel_for_(range2, FirstScan(img, imgLabels, P.data(), chunksSizeAndLabels.data()), nParallelStripes); + + //merge labels of different chunks + mergeLabels(img, imgLabels, P.data(), chunksSizeAndLabels.data()); + + LabelT nLabels = 1; + for (int i = 0; i < h; i = chunksSizeAndLabels[i]) { + CV_DbgAssert(i + 1 < chunksSizeAndLabelsSize); + flattenL(P.data(), stripeFirstLabel8Connectivity(i, w), chunksSizeAndLabels[i + 1], nLabels); + } + + //Array for statistics data + std::vector sopArray(h); + sop.init(nLabels); + + //Second scan + cv::parallel_for_(range2, SecondScan(img, imgLabels, P.data(), sop, sopArray.data(), nLabels), nParallelStripes); + + StatsOp::mergeStats(imgLabels, sopArray.data(), sop, nLabels); + sop.finish(); + + return nLabels; + } + };//End struct LabelingBolelliParallel + + //Implementation of Spaghetti algorithm, as described in "Spaghetti Labeling: Directed Acyclic Graphs + //for Block-Based Connected Components Labeling", IEEE Transactions on Image Processing, Federico Bolelli et. al. template struct LabelingBolelli { @@ -644,9 +1533,440 @@ namespace cv{ }//End function LabelingBolelli operator() };//End struct LabelingBolelli + //Parallel implementation of Spaghetti algorithm for 4-way connectivity, generated with the tool described in "One DAG to + //Rule Them All", IEEE Transactions on Pattern Analysis and Machine Intelligence, Federico Bolelli et. al. + //Parallelization method described in "Two More Strategies to Speed Up Connected Components Labeling Algorithms", + //Image Analysis and Processing - ICIAP 2017, Federico Bolelli et. al. + template + struct LabelingBolelli4CParallel { + + class FirstScan : public cv::ParallelLoopBody { + const cv::Mat& img_; + cv::Mat& imgLabels_; + LabelT* P_; + int* chunksSizeAndLabels_; + + public: + FirstScan(const cv::Mat& img, cv::Mat& imgLabels, LabelT* P, int* chunksSizeAndLabels) + : img_(img), imgLabels_(imgLabels), P_(P), chunksSizeAndLabels_(chunksSizeAndLabels) {} + + FirstScan& operator=(const FirstScan&) { return *this; } + + void operator()(const cv::Range& range2) const CV_OVERRIDE + { + const Range range(range2.start * 2, std::min(range2.end * 2, img_.rows)); + int r = range.start; + + chunksSizeAndLabels_[r] = range.end; + + LabelT label = stripeFirstLabel4Connectivity(r, imgLabels_.cols); + + const LabelT firstLabel = label; + const int w = img_.cols; + const int startR = r; + + { +#define CONDITION_Q img_row_prev[c] > 0 +#define CONDITION_S img_row[c - 1] > 0 +#define CONDITION_X img_row[c] > 0 + +#define ACTION_1 // nothing to do +#define ACTION_2 img_labels_row[c] = label; \ + P_[label] = label; \ + label = label + 1; +#define ACTION_3 img_labels_row[c] = img_labels_row_prev[c]; // x <- q +#define ACTION_4 img_labels_row[c] = img_labels_row[c - 1]; // x <- s +#define ACTION_5 img_labels_row[c] = set_union(P_, img_labels_row_prev[c], img_labels_row[c - 1]); // x <- q + s + } + + // First row + { + const PixelT* const img_row = img_.ptr(r); + LabelT* const img_labels_row = imgLabels_.ptr(r); + int c = -1; + + goto fl_tree_0; + fl_tree_0: if ((c += 1) >= w) goto fl_break; + if (CONDITION_X) { + ACTION_2 + goto fl_tree_1; + } + else { + ACTION_1 + goto fl_tree_0; + } + fl_tree_1: if ((c += 1) >= w) goto fl_break; + if (CONDITION_X) { + ACTION_4 + goto fl_tree_1; + } + else { + ACTION_1 + goto fl_tree_0; + } + fl_break:; + } + + // Other rows + ++r; + for (; r < range.end; ++r) { + // Get row pointers + const PixelT* const img_row = img_.ptr(r); + const PixelT* const img_row_prev = (PixelT*)(((char*)img_row) - img_.step.p[0]); + LabelT* const img_labels_row = imgLabels_.ptr(r); + LabelT* const img_labels_row_prev = (LabelT*)(((char*)img_labels_row) - imgLabels_.step.p[0]); + int c = -1; + + goto cl_tree_0; + cl_tree_0: if ((c += 1) >= w) goto cl_break; + if (CONDITION_X) { + if (CONDITION_Q) { + ACTION_3 + goto cl_tree_1; + } + else { + ACTION_2 + goto cl_tree_1; + } + } + else { + ACTION_1 + goto cl_tree_0; + } + cl_tree_1: if ((c += 1) >= w) goto cl_break; + if (CONDITION_X) { + if (CONDITION_Q) { + ACTION_5 + goto cl_tree_1; + } + else { + ACTION_4 + goto cl_tree_1; + } + } + else { + ACTION_1 + goto cl_tree_0; + } + cl_break:; + } + + // undef conditions and actions + { +#undef ACTION_1 +#undef ACTION_2 +#undef ACTION_3 +#undef ACTION_4 +#undef ACTION_5 + +#undef CONDITION_Q +#undef CONDITION_S +#undef CONDITION_X + } + + //write in the following memory location + chunksSizeAndLabels_[startR + 1] = label - firstLabel; + } + }; + + class SecondScan : public cv::ParallelLoopBody { + cv::Mat& imgLabels_; + const LabelT* P_; + StatsOp& sop_; + StatsOp* sopArray_; + LabelT& nLabels_; + public: + SecondScan(cv::Mat& imgLabels, const LabelT* P, StatsOp& sop, StatsOp* sopArray, LabelT& nLabels) + : imgLabels_(imgLabels), P_(P), sop_(sop), sopArray_(sopArray), nLabels_(nLabels) {} + + SecondScan& operator=(const SecondScan&) { return *this; } + + void operator()(const cv::Range& range2) const CV_OVERRIDE + { + const Range range(range2.start * 2, std::min(range2.end * 2, imgLabels_.rows)); + int r = range.start; + const int rowBegin = r; + const int rowEnd = range.end; + + if (rowBegin > 0) { + sopArray_[rowBegin].initElement(nLabels_); + sopArray_[rowBegin].setNextLoc(rowEnd); //_nextLoc = rowEnd; + + for (; r < rowEnd; ++r) { + LabelT* img_row_start = imgLabels_.ptr(r); + LabelT* const img_row_end = img_row_start + imgLabels_.cols; + for (int c = 0; img_row_start != img_row_end; ++img_row_start, ++c) { + *img_row_start = P_[*img_row_start]; + sopArray_[rowBegin](r, c, *img_row_start); + } + } + } + else { + //the first thread uses sop in order to make less merges + sop_.setNextLoc(rowEnd); + for (; r < rowEnd; ++r) { + LabelT* img_row_start = imgLabels_.ptr(r); + LabelT* const img_row_end = img_row_start + imgLabels_.cols; + for (int c = 0; img_row_start != img_row_end; ++img_row_start, ++c) { + *img_row_start = P_[*img_row_start]; + sop_(r, c, *img_row_start); + } + } + } + } + }; + + inline static + void mergeLabels(cv::Mat& imgLabels, LabelT* P, const int* chunksSizeAndLabels) { + + // Merge Mask + // +-+-+-+ + // |-|q|-| + // +-+-+-+ + // |x| + // +-+ + const int w = imgLabels.cols, h = imgLabels.rows; + + for (int r = chunksSizeAndLabels[0]; r < h; r = chunksSizeAndLabels[r]) { + + LabelT* const imgLabels_row = imgLabels.ptr(r); + LabelT* const imgLabels_row_prev = (LabelT*)(((char*)imgLabels_row) - imgLabels.step.p[0]); + + for (int c = 0; c < w; ++c) { + +#define condition_q imgLabels_row_prev[c] > 0 +#define condition_x imgLabels_row[c] > 0 + + if (condition_x) { + if (condition_q) { + //merge of two label + imgLabels_row[c] = set_union(P, imgLabels_row_prev[c], imgLabels_row[c]); + } + } + } + } +#undef condition_q +#undef condition_x + } + + LabelT operator()(const cv::Mat& img, cv::Mat& imgLabels, int connectivity, StatsOp& sop) { + CV_Assert(img.rows == imgLabels.rows); + CV_Assert(img.cols == imgLabels.cols); + CV_Assert(connectivity == 4); + + const int h = img.rows; + const int w = img.cols; + + //A quick and dirty upper bound for the maximum number of labels. + //Following formula comes from the fact that a 2x2 block in 4-way connectivity + //labeling can never have more than 2 new labels and 1 label for background. + //Worst case image example pattern: + //1 0 1 0 1... + //0 1 0 1 0... + //1 0 1 0 1... + //............ + const size_t Plength = (size_t(h) * size_t(w) + 1) / 2 + 1; + + //Array used to store info and labeled pixel by each thread. + //Different threads affect different memory location of chunksSizeAndLabels + std::vector chunksSizeAndLabels(roundUp(h, 2)); + + //Tree of labels + std::vector P_(Plength, 0); + LabelT* P = P_.data(); + //First label is for background + //P[0] = 0; + + cv::Range range2(0, divUp(h, 2)); + const double nParallelStripes = std::max(1, std::min(h / 2, getNumThreads() * 4)); + + LabelT nLabels = 1; + + //First scan + cv::parallel_for_(range2, FirstScan(img, imgLabels, P, chunksSizeAndLabels.data()), nParallelStripes); + + //merge labels of different chunks + mergeLabels(imgLabels, P, chunksSizeAndLabels.data()); + + for (int i = 0; i < h; i = chunksSizeAndLabels[i]) { + flattenL(P, stripeFirstLabel4Connectivity(i, w), chunksSizeAndLabels[i + 1], nLabels); + } + + //Array for statistics dataof threads + std::vector sopArray(h); + + sop.init(nLabels); + //Second scan + cv::parallel_for_(range2, SecondScan(imgLabels, P, sop, sopArray.data(), nLabels), nParallelStripes); + StatsOp::mergeStats(imgLabels, sopArray.data(), sop, nLabels); + sop.finish(); + + return nLabels; + } + };//End struct LabelingBolelli4CParallel + + //Implementation of Spaghetti algorithm for 4-way connectivity, generated with the tool described in "One DAG to + //Rule Them All", IEEE Transactions on Pattern Analysis and Machine Intelligence, Federico Bolelli et. al. + template + struct LabelingBolelli4C + { + LabelT operator()(const cv::Mat& img, cv::Mat& imgLabels, int connectivity, StatsOp& sop) + { + CV_Assert(img.rows == imgLabels.rows); + CV_Assert(img.cols == imgLabels.cols); + CV_Assert(connectivity == 4); + + const int h = img.rows; + const int w = img.cols; + + // A quick and dirty upper bound for the maximum number of labels. + // Following formula comes from the fact that a 2x2 block in 4-connectivity case + // can never have more than 2 new labels and 1 label for background. + // Worst case image example pattern: + // 1 0 1 0 1... + // 0 1 0 1 0... + // 1 0 1 0 1... + // ............ + const size_t Plength = size_t((size_t(h) * size_t(w) + 1) / 2) + 1; + + std::vector P_(Plength, 0); + LabelT* P = P_.data(); + //P[0] = 0; + LabelT lunique = 1; + + // First scan + + // We work with the 4-conn Rosenfeld mask + // +-+ + // |q| + // +-+-+ + // |s|x| + // +-+-+ + + // A bunch of defines is used to check if the pixels are foreground + // and to define actions to be performed + { + +#define CONDITION_Q img_row_prev[c] > 0 +#define CONDITION_S img_row[c - 1] > 0 +#define CONDITION_X img_row[c] > 0 + +#define ACTION_1 // nothing to do +#define ACTION_2 img_labels_row[c] = lunique; \ + P[lunique] = lunique; \ + lunique = lunique + 1; // new label +#define ACTION_3 img_labels_row[c] = img_labels_row_prev[c]; // x <- q +#define ACTION_4 img_labels_row[c] = img_labels_row[c - 1]; // x <- s +#define ACTION_5 img_labels_row[c] = set_union(P, img_labels_row_prev[c], img_labels_row[c - 1]); // x <- q + s + } + + // First row + { + const PixelT* const img_row = img.ptr(0); + LabelT* const img_labels_row = imgLabels.ptr(0); + int c = -1; + + goto fl_tree_0; + fl_tree_0: if ((c += 1) >= w) goto fl_break; + if (CONDITION_X) { + ACTION_2 + goto fl_tree_1; + } + else { + ACTION_1 + goto fl_tree_0; + } + fl_tree_1: if ((c += 1) >= w) goto fl_break; + if (CONDITION_X) { + ACTION_4 + goto fl_tree_1; + } + else { + ACTION_1 + goto fl_tree_0; + } + fl_break:; + } + + + // Other rows + for (int r = 1; r < h; ++r) { + // Get row pointers + const PixelT* const img_row = img.ptr(r); + const PixelT* const img_row_prev = (PixelT*)(((char*)img_row) - img.step.p[0]); + LabelT* const img_labels_row = imgLabels.ptr(r); + LabelT* const img_labels_row_prev = (LabelT*)(((char*)img_labels_row) - imgLabels.step.p[0]); + int c = -1; + + goto cl_tree_0; + cl_tree_0: if ((c += 1) >= w) goto cl_break; + if (CONDITION_X) { + if (CONDITION_Q) { + ACTION_3 + goto cl_tree_1; + } + else { + ACTION_2 + goto cl_tree_1; + } + } + else { + ACTION_1 + goto cl_tree_0; + } + cl_tree_1: if ((c += 1) >= w) goto cl_break; + if (CONDITION_X) { + if (CONDITION_Q) { + ACTION_5 + goto cl_tree_1; + } + else { + ACTION_4 + goto cl_tree_1; + } + } + else { + ACTION_1 + goto cl_tree_0; + } + cl_break:; + } + + // undef conditions and actions + { +#undef ACTION_1 +#undef ACTION_2 +#undef ACTION_3 +#undef ACTION_4 +#undef ACTION_5 + +#undef CONDITION_Q +#undef CONDITION_S +#undef CONDITION_X + } + + // Second scan + analysis + LabelT nLabels = flattenL(P, lunique); + sop.init(nLabels); + + for (int r = 0; r < h; ++r) { + LabelT* img_row_start = imgLabels.ptr(r); + LabelT* const img_row_end = img_row_start + w; + for (int c = 0; img_row_start != img_row_end; ++img_row_start, ++c) { + *img_row_start = P[*img_row_start]; + sop(r, c, *img_row_start); + } + } + + sop.finish(); + + return nLabels; + + }//End function LabelingBolelli4C operator() + };//End struct LabelingBolelli4C + //Parallel implementation of Scan Array-based Union Find (SAUF) algorithm, as described in "Two More Strategies to Speed - //Up Connected Components Labeling Algorithms" - //Federico Bolelli et. al. + //Up Connected Components Labeling Algorithms", Image Analysis and Processing - ICIAP 2017, Federico Bolelli et. al. template struct LabelingWuParallel{ @@ -1029,8 +2349,7 @@ namespace cv{ };//End struct LabelingWuParallel //Based on "Two Strategies to Speed up Connected Components Algorithms", the SAUF (Scan Array-based Union Find) variant - //using decision trees - //Kesheng Wu et. al. + //using decision trees, Kesheng Wu et. al. template struct LabelingWu{ LabelT operator()(const cv::Mat& img, cv::Mat& imgLabels, int connectivity, StatsOp& sop){ @@ -1199,8 +2518,7 @@ namespace cv{ };//End struct LabelingWu //Parallel implementation of BBDT (Block-Based with Decision Tree) algorithm, as described in "Two More Strategies to Speed - //Up Connected Components Labeling Algorithms" - //Federico Bolelli et. al. + //Up Connected Components Labeling Algorithms", Image Analysis and Processing - ICIAP 2017, Federico Bolelli et. al. template struct LabelingGranaParallel{ @@ -2960,8 +4278,7 @@ namespace cv{ };//End struct LabelingGranaParallel //Implementation of BBDT (Block-Based with Decision Tree) algorithm, as described in "Optimized Block-based Connected - //Components Labeling with Decision Trees" (only for 8-connectivity) - //Costantino Grana et. al. + //Components Labeling with Decision Trees", IEEE Transactions on Image Processing, Costantino Grana et. al. template struct LabelingGrana{ LabelT operator()(const cv::Mat& img, cv::Mat& imgLabels, int connectivity, StatsOp& sop){ @@ -4317,7 +5634,7 @@ namespace cv{ //Run parallel labeling only if the rows of the image are at least twice the number of available threads const bool is_parallel = currentParallelFramework != NULL && nThreads > 1 && L.rows / nThreads >= 2; - if (ccltype == CCL_SAUF || ccltype == CCL_WU || connectivity == 4){ + if (ccltype == CCL_SAUF || ccltype == CCL_WU || ((ccltype == CCL_BBDT || ccltype == CCL_GRANA) && connectivity == 4)){ // SAUF algorithm is used using connectedcomponents::LabelingWu; using connectedcomponents::LabelingWuParallel; @@ -4337,7 +5654,7 @@ namespace cv{ return (int)LabelingWuParallel()(I, L, connectivity, sop); } } - else if ((ccltype == CCL_BBDT || ccltype == CCL_GRANA || ccltype == CCL_DEFAULT) && connectivity == 8){ + else if ((ccltype == CCL_BBDT || ccltype == CCL_GRANA) && connectivity == 8){ // BBDT algorithm is used using connectedcomponents::LabelingGrana; using connectedcomponents::LabelingGranaParallel; @@ -4357,21 +5674,45 @@ namespace cv{ return (int)LabelingGranaParallel()(I, L, connectivity, sop); } } - else if ((ccltype == CCL_SPAGHETTI || ccltype == CCL_BOLELLI) && connectivity == 8) { + else if (ccltype == CCL_SPAGHETTI || ccltype == CCL_BOLELLI || ccltype == CCL_DEFAULT) { // Spaghetti algorithm is used - using connectedcomponents::LabelingBolelli; - //using connectedcomponents::LabelingBolelliParallel; // Not implemented - //warn if L's depth is not sufficient? - if (lDepth == CV_8U) { - //Not supported yet + if (connectivity == 8) { + using connectedcomponents::LabelingBolelli; + using connectedcomponents::LabelingBolelliParallel; + //warn if L's depth is not sufficient? + if (lDepth == CV_8U) { + //Not supported yet + } + else if (lDepth == CV_16U) { + return (int)LabelingBolelli()(I, L, connectivity, sop); + } + else if (lDepth == CV_32S) { + //note that signed types don't really make sense here and not being able to use unsigned matters for scientific projects + //OpenCV: how should we proceed? .at typechecks in debug mode + if (!is_parallel) + return (int)LabelingBolelli()(I, L, connectivity, sop); + else + return (int)LabelingBolelliParallel()(I, L, connectivity, sop); + } } - else if (lDepth == CV_16U) { - return (int)LabelingBolelli()(I, L, connectivity, sop); - } - else if (lDepth == CV_32S) { - //note that signed types don't really make sense here and not being able to use unsigned matters for scientific projects - //OpenCV: how should we proceed? .at typechecks in debug mode - return (int)LabelingBolelli()(I, L, connectivity, sop); + else { + using connectedcomponents::LabelingBolelli4C; + using connectedcomponents::LabelingBolelli4CParallel; + //warn if L's depth is not sufficient? + if (lDepth == CV_8U) { + //Not supported yet + } + else if (lDepth == CV_16U) { + return (int)LabelingBolelli4C()(I, L, connectivity, sop); + } + else if (lDepth == CV_32S) { + //note that signed types don't really make sense here and not being able to use unsigned matters for scientific projects + //OpenCV: how should we proceed? .at typechecks in debug mode + if (!is_parallel) + return (int)LabelingBolelli4C()(I, L, connectivity, sop); + else + return (int)LabelingBolelli4CParallel()(I, L, connectivity, sop); + } } } diff --git a/modules/imgproc/test/test_connectedcomponents.cpp b/modules/imgproc/test/test_connectedcomponents.cpp index e57c24a937..ed11ea6fda 100644 --- a/modules/imgproc/test/test_connectedcomponents.cpp +++ b/modules/imgproc/test/test_connectedcomponents.cpp @@ -42,7 +42,8 @@ #include "test_precomp.hpp" -namespace opencv_test { namespace { +namespace opencv_test { +namespace { class CV_ConnectedComponentsTest : public cvtest::BaseTest { @@ -61,10 +62,10 @@ void normalizeLabels(Mat1i& imgLabels, int iNumLabels) { vector vecNewLabels(iNumLabels + 1, 0); int iMaxNewLabel = 0; - for (int r = 0; r0) { + if (iCurLabel > 0) { if (vecNewLabels[iCurLabel] == 0) { vecNewLabels[iCurLabel] = ++iMaxNewLabel; } @@ -74,7 +75,7 @@ void normalizeLabels(Mat1i& imgLabels, int iNumLabels) { } } -void CV_ConnectedComponentsTest::run( int /* start_from */) +void CV_ConnectedComponentsTest::run(int /* start_from */) { int ccltype[] = { cv::CCL_DEFAULT, cv::CCL_WU, cv::CCL_GRANA, cv::CCL_BOLELLI, cv::CCL_SAUF, cv::CCL_BBDT, cv::CCL_SPAGHETTI }; @@ -91,7 +92,7 @@ void CV_ConnectedComponentsTest::run( int /* start_from */) Mat bw = orig > 128; - for (uint cclt = 0; cclt < sizeof(ccltype)/sizeof(int); ++cclt) + for (uint cclt = 0; cclt < sizeof(ccltype) / sizeof(int); ++cclt) { Mat1i labelImage; @@ -100,11 +101,11 @@ void CV_ConnectedComponentsTest::run( int /* start_from */) normalizeLabels(labelImage, nLabels); // Validate test results - for (int r = 0; r < labelImage.rows; ++r){ - for (int c = 0; c < labelImage.cols; ++c){ + for (int r = 0; r < labelImage.rows; ++r) { + for (int c = 0; c < labelImage.cols; ++c) { int l = labelImage.at(r, c); bool pass = l >= 0 && l <= nLabels; - if (!pass){ + if (!pass) { ts->set_failed_test_info(cvtest::TS::FAIL_INVALID_OUTPUT); return; } @@ -166,12 +167,12 @@ static cv::Mat createCrashMat(int numThreads) { for (int s = stripeRange.start; s < stripeRange.end; s++) { cv::Range sr(s, s + 1); cv::Range r; - r.start = (int) (wholeRange.start + - ((uint64) sr.start * (wholeRange.end - wholeRange.start) + nstripes / 2) / nstripes); + r.start = (int)(wholeRange.start + + ((uint64)sr.start * (wholeRange.end - wholeRange.start) + nstripes / 2) / nstripes); r.end = sr.end >= nstripes ? - wholeRange.end : - (int) (wholeRange.start + - ((uint64) sr.end * (wholeRange.end - wholeRange.start) + nstripes / 2) / nstripes); + wholeRange.end : + (int)(wholeRange.start + + ((uint64)sr.end * (wholeRange.end - wholeRange.start) + nstripes / 2) / nstripes); if (r.start > 0 && r.start % 2 == 1 && r.end % 2 == 0 && r.end >= r.start + 2) { bugRange = r; @@ -203,7 +204,7 @@ static cv::Mat createCrashMat(int numThreads) { TEST(Imgproc_ConnectedComponents, parallel_wu_labels) { cv::Mat mat = createCrashMat(cv::getNumThreads()); - if(mat.empty()) { + if (mat.empty()) { return; } @@ -213,10 +214,10 @@ TEST(Imgproc_ConnectedComponents, parallel_wu_labels) cv::Mat stats; cv::Mat centroids; int nb = 0; - EXPECT_NO_THROW( nb = cv::connectedComponentsWithStats(mat, labels, stats, centroids, 8, CV_32S, cv::CCL_WU) ); + EXPECT_NO_THROW(nb = cv::connectedComponentsWithStats(mat, labels, stats, centroids, 8, CV_32S, cv::CCL_WU)); int area = 0; - for(int i=1; i(i, cv::CC_STAT_AREA); } @@ -229,7 +230,7 @@ TEST(Imgproc_ConnectedComponents, missing_background_pixels) cv::Mat labels; cv::Mat stats; cv::Mat centroids; - EXPECT_NO_THROW(cv::connectedComponentsWithStats(m, labels, stats, centroids, 8, CV_32S, cv::CCL_WU) ); + EXPECT_NO_THROW(cv::connectedComponentsWithStats(m, labels, stats, centroids, 8, CV_32S, cv::CCL_WU)); EXPECT_EQ(stats.at(0, cv::CC_STAT_WIDTH), 0); EXPECT_EQ(stats.at(0, cv::CC_STAT_HEIGHT), 0); EXPECT_EQ(stats.at(0, cv::CC_STAT_LEFT), -1); @@ -241,21 +242,21 @@ TEST(Imgproc_ConnectedComponents, spaghetti_bbdt_sauf_stats) { cv::Mat1b img(16, 16); img << 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, - 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, - 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, - 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, - 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, - 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, - 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, - 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, - 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1; + 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, + 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, + 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, + 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, + 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, + 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, + 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1; cv::Mat1i labels; cv::Mat1i stats; @@ -357,4 +358,436 @@ TEST(Imgproc_ConnectedComponents, spaghetti_bbdt_sauf_stats) } } -}} // namespace +TEST(Imgproc_ConnectedComponents, chessboard_even) +{ + cv::Size size(16, 16); + cv::Mat1b input(size); + cv::Mat1i output_8c(size); + cv::Mat1i output_4c(size); + + // Chessboard image with even number of rows and cols + // Note that this is the maximum number of labels for 4-way connectivity + { + input << + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1; + + output_8c << + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1; + + output_4c << + 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0, 8, 0, + 0, 9, 0, 10, 0, 11, 0, 12, 0, 13, 0, 14, 0, 15, 0, 16, + 17, 0, 18, 0, 19, 0, 20, 0, 21, 0, 22, 0, 23, 0, 24, 0, + 0, 25, 0, 26, 0, 27, 0, 28, 0, 29, 0, 30, 0, 31, 0, 32, + 33, 0, 34, 0, 35, 0, 36, 0, 37, 0, 38, 0, 39, 0, 40, 0, + 0, 41, 0, 42, 0, 43, 0, 44, 0, 45, 0, 46, 0, 47, 0, 48, + 49, 0, 50, 0, 51, 0, 52, 0, 53, 0, 54, 0, 55, 0, 56, 0, + 0, 57, 0, 58, 0, 59, 0, 60, 0, 61, 0, 62, 0, 63, 0, 64, + 65, 0, 66, 0, 67, 0, 68, 0, 69, 0, 70, 0, 71, 0, 72, 0, + 0, 73, 0, 74, 0, 75, 0, 76, 0, 77, 0, 78, 0, 79, 0, 80, + 81, 0, 82, 0, 83, 0, 84, 0, 85, 0, 86, 0, 87, 0, 88, 0, + 0, 89, 0, 90, 0, 91, 0, 92, 0, 93, 0, 94, 0, 95, 0, 96, + 97, 0, 98, 0, 99, 0, 100, 0, 101, 0, 102, 0, 103, 0, 104, 0, + 0, 105, 0, 106, 0, 107, 0, 108, 0, 109, 0, 110, 0, 111, 0, 112, + 113, 0, 114, 0, 115, 0, 116, 0, 117, 0, 118, 0, 119, 0, 120, 0, + 0, 121, 0, 122, 0, 123, 0, 124, 0, 125, 0, 126, 0, 127, 0, 128; + } + + int ccltype[] = { cv::CCL_DEFAULT, cv::CCL_WU, cv::CCL_GRANA, cv::CCL_BOLELLI, cv::CCL_SAUF, cv::CCL_BBDT, cv::CCL_SPAGHETTI }; + + cv::Mat1i labels; + cv::Mat diff; + int nLabels = 0; + for (size_t cclt = 0; cclt < sizeof(ccltype) / sizeof(int); ++cclt) { + + EXPECT_NO_THROW(nLabels = cv::connectedComponents(input, labels, 8, CV_32S, ccltype[cclt])); + normalizeLabels(labels, nLabels); + + diff = labels != output_8c; + EXPECT_EQ(cv::countNonZero(diff), 0); + + + EXPECT_NO_THROW(nLabels = cv::connectedComponents(input, labels, 4, CV_32S, ccltype[cclt])); + normalizeLabels(labels, nLabels); + + diff = labels != output_4c; + EXPECT_EQ(cv::countNonZero(diff), 0); + } + +} + +TEST(Imgproc_ConnectedComponents, chessboard_odd) +{ + cv::Size size(15, 15); + cv::Mat1b input(size); + cv::Mat1i output_8c(size); + cv::Mat1i output_4c(size); + + // Chessboard image with odd number of rows and cols + // Note that this is the maximum number of labels for 4-way connectivity + { + input << + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1; + + output_8c << + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1; + + output_4c << + 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0, 8, + 0, 9, 0, 10, 0, 11, 0, 12, 0, 13, 0, 14, 0, 15, 0, + 16, 0, 17, 0, 18, 0, 19, 0, 20, 0, 21, 0, 22, 0, 23, + 0, 24, 0, 25, 0, 26, 0, 27, 0, 28, 0, 29, 0, 30, 0, + 31, 0, 32, 0, 33, 0, 34, 0, 35, 0, 36, 0, 37, 0, 38, + 0, 39, 0, 40, 0, 41, 0, 42, 0, 43, 0, 44, 0, 45, 0, + 46, 0, 47, 0, 48, 0, 49, 0, 50, 0, 51, 0, 52, 0, 53, + 0, 54, 0, 55, 0, 56, 0, 57, 0, 58, 0, 59, 0, 60, 0, + 61, 0, 62, 0, 63, 0, 64, 0, 65, 0, 66, 0, 67, 0, 68, + 0, 69, 0, 70, 0, 71, 0, 72, 0, 73, 0, 74, 0, 75, 0, + 76, 0, 77, 0, 78, 0, 79, 0, 80, 0, 81, 0, 82, 0, 83, + 0, 84, 0, 85, 0, 86, 0, 87, 0, 88, 0, 89, 0, 90, 0, + 91, 0, 92, 0, 93, 0, 94, 0, 95, 0, 96, 0, 97, 0, 98, + 0, 99, 0, 100, 0, 101, 0, 102, 0, 103, 0, 104, 0, 105, 0, + 106, 0, 107, 0, 108, 0, 109, 0, 110, 0, 111, 0, 112, 0, 113; + } + + int ccltype[] = { cv::CCL_DEFAULT, cv::CCL_WU, cv::CCL_GRANA, cv::CCL_BOLELLI, cv::CCL_SAUF, cv::CCL_BBDT, cv::CCL_SPAGHETTI }; + + cv::Mat1i labels; + cv::Mat diff; + int nLabels = 0; + for (size_t cclt = 0; cclt < sizeof(ccltype) / sizeof(int); ++cclt) { + + EXPECT_NO_THROW(nLabels = cv::connectedComponents(input, labels, 8, CV_32S, ccltype[cclt])); + normalizeLabels(labels, nLabels); + + diff = labels != output_8c; + EXPECT_EQ(cv::countNonZero(diff), 0); + + + EXPECT_NO_THROW(nLabels = cv::connectedComponents(input, labels, 4, CV_32S, ccltype[cclt])); + normalizeLabels(labels, nLabels); + + diff = labels != output_4c; + EXPECT_EQ(cv::countNonZero(diff), 0); + } + +} + +TEST(Imgproc_ConnectedComponents, maxlabels_8conn_even) +{ + cv::Size size(16, 16); + cv::Mat1b input(size); + cv::Mat1i output_8c(size); + cv::Mat1i output_4c(size); + + { + input << + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0; + + output_8c << + 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0, 8, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 10, 0, 11, 0, 12, 0, 13, 0, 14, 0, 15, 0, 16, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 17, 0, 18, 0, 19, 0, 20, 0, 21, 0, 22, 0, 23, 0, 24, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 25, 0, 26, 0, 27, 0, 28, 0, 29, 0, 30, 0, 31, 0, 32, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 34, 0, 35, 0, 36, 0, 37, 0, 38, 0, 39, 0, 40, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 42, 0, 43, 0, 44, 0, 45, 0, 46, 0, 47, 0, 48, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 49, 0, 50, 0, 51, 0, 52, 0, 53, 0, 54, 0, 55, 0, 56, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 57, 0, 58, 0, 59, 0, 60, 0, 61, 0, 62, 0, 63, 0, 64, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0; + + output_4c << + 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0, 8, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 10, 0, 11, 0, 12, 0, 13, 0, 14, 0, 15, 0, 16, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 17, 0, 18, 0, 19, 0, 20, 0, 21, 0, 22, 0, 23, 0, 24, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 25, 0, 26, 0, 27, 0, 28, 0, 29, 0, 30, 0, 31, 0, 32, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 34, 0, 35, 0, 36, 0, 37, 0, 38, 0, 39, 0, 40, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 42, 0, 43, 0, 44, 0, 45, 0, 46, 0, 47, 0, 48, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 49, 0, 50, 0, 51, 0, 52, 0, 53, 0, 54, 0, 55, 0, 56, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 57, 0, 58, 0, 59, 0, 60, 0, 61, 0, 62, 0, 63, 0, 64, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0; + } + + int ccltype[] = { cv::CCL_DEFAULT, cv::CCL_WU, cv::CCL_GRANA, cv::CCL_BOLELLI, cv::CCL_SAUF, cv::CCL_BBDT, cv::CCL_SPAGHETTI }; + + cv::Mat1i labels; + cv::Mat diff; + int nLabels = 0; + for (size_t cclt = 0; cclt < sizeof(ccltype) / sizeof(int); ++cclt) { + + EXPECT_NO_THROW(nLabels = cv::connectedComponents(input, labels, 8, CV_32S, ccltype[cclt])); + normalizeLabels(labels, nLabels); + + diff = labels != output_8c; + EXPECT_EQ(cv::countNonZero(diff), 0); + + + EXPECT_NO_THROW(nLabels = cv::connectedComponents(input, labels, 4, CV_32S, ccltype[cclt])); + normalizeLabels(labels, nLabels); + + diff = labels != output_4c; + EXPECT_EQ(cv::countNonZero(diff), 0); + } + +} + +TEST(Imgproc_ConnectedComponents, maxlabels_8conn_odd) +{ + cv::Size size(15, 15); + cv::Mat1b input(size); + cv::Mat1i output_8c(size); + cv::Mat1i output_4c(size); + + { + input << + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1; + + output_8c << + 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0, 8, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 10, 0, 11, 0, 12, 0, 13, 0, 14, 0, 15, 0, 16, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 17, 0, 18, 0, 19, 0, 20, 0, 21, 0, 22, 0, 23, 0, 24, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 25, 0, 26, 0, 27, 0, 28, 0, 29, 0, 30, 0, 31, 0, 32, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 34, 0, 35, 0, 36, 0, 37, 0, 38, 0, 39, 0, 40, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 42, 0, 43, 0, 44, 0, 45, 0, 46, 0, 47, 0, 48, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 49, 0, 50, 0, 51, 0, 52, 0, 53, 0, 54, 0, 55, 0, 56, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 57, 0, 58, 0, 59, 0, 60, 0, 61, 0, 62, 0, 63, 0, 64; + + output_4c << + 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0, 8, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 10, 0, 11, 0, 12, 0, 13, 0, 14, 0, 15, 0, 16, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 17, 0, 18, 0, 19, 0, 20, 0, 21, 0, 22, 0, 23, 0, 24, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 25, 0, 26, 0, 27, 0, 28, 0, 29, 0, 30, 0, 31, 0, 32, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 34, 0, 35, 0, 36, 0, 37, 0, 38, 0, 39, 0, 40, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 42, 0, 43, 0, 44, 0, 45, 0, 46, 0, 47, 0, 48, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 49, 0, 50, 0, 51, 0, 52, 0, 53, 0, 54, 0, 55, 0, 56, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 57, 0, 58, 0, 59, 0, 60, 0, 61, 0, 62, 0, 63, 0, 64; + } + + int ccltype[] = { cv::CCL_DEFAULT, cv::CCL_WU, cv::CCL_GRANA, cv::CCL_BOLELLI, cv::CCL_SAUF, cv::CCL_BBDT, cv::CCL_SPAGHETTI }; + + cv::Mat1i labels; + cv::Mat diff; + int nLabels = 0; + for (size_t cclt = 0; cclt < sizeof(ccltype) / sizeof(int); ++cclt) { + + EXPECT_NO_THROW(nLabels = cv::connectedComponents(input, labels, 8, CV_32S, ccltype[cclt])); + normalizeLabels(labels, nLabels); + + diff = labels != output_8c; + EXPECT_EQ(cv::countNonZero(diff), 0); + + + EXPECT_NO_THROW(nLabels = cv::connectedComponents(input, labels, 4, CV_32S, ccltype[cclt])); + normalizeLabels(labels, nLabels); + + diff = labels != output_4c; + EXPECT_EQ(cv::countNonZero(diff), 0); + } + +} + +TEST(Imgproc_ConnectedComponents, single_row) +{ + cv::Size size(1, 15); + cv::Mat1b input(size); + cv::Mat1i output_8c(size); + cv::Mat1i output_4c(size); + + { + input << + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1; + + + output_8c << + 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0, 8; + + + output_4c << + 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0, 8; + + } + + int ccltype[] = { cv::CCL_DEFAULT, cv::CCL_WU, cv::CCL_GRANA, cv::CCL_BOLELLI, cv::CCL_SAUF, cv::CCL_BBDT, cv::CCL_SPAGHETTI }; + + cv::Mat1i labels; + cv::Mat diff; + int nLabels = 0; + for (size_t cclt = 0; cclt < sizeof(ccltype) / sizeof(int); ++cclt) { + + EXPECT_NO_THROW(nLabels = cv::connectedComponents(input, labels, 8, CV_32S, ccltype[cclt])); + normalizeLabels(labels, nLabels); + + diff = labels != output_8c; + EXPECT_EQ(cv::countNonZero(diff), 0); + + + EXPECT_NO_THROW(nLabels = cv::connectedComponents(input, labels, 4, CV_32S, ccltype[cclt])); + normalizeLabels(labels, nLabels); + + diff = labels != output_4c; + EXPECT_EQ(cv::countNonZero(diff), 0); + } + +} + +TEST(Imgproc_ConnectedComponents, single_column) +{ + cv::Size size(15, 1); + cv::Mat1b input(size); + cv::Mat1i output_8c(size); + cv::Mat1i output_4c(size); + + { + input << + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1; + + + output_8c << + 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0, 8; + + + output_4c << + 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0, 8; + + } + + int ccltype[] = { cv::CCL_DEFAULT, cv::CCL_WU, cv::CCL_GRANA, cv::CCL_BOLELLI, cv::CCL_SAUF, cv::CCL_BBDT, cv::CCL_SPAGHETTI }; + + cv::Mat1i labels; + cv::Mat diff; + int nLabels = 0; + for (size_t cclt = 0; cclt < sizeof(ccltype) / sizeof(int); ++cclt) { + + EXPECT_NO_THROW(nLabels = cv::connectedComponents(input, labels, 8, CV_32S, ccltype[cclt])); + normalizeLabels(labels, nLabels); + + diff = labels != output_8c; + EXPECT_EQ(cv::countNonZero(diff), 0); + + + EXPECT_NO_THROW(nLabels = cv::connectedComponents(input, labels, 4, CV_32S, ccltype[cclt])); + normalizeLabels(labels, nLabels); + + diff = labels != output_4c; + EXPECT_EQ(cv::countNonZero(diff), 0); + } + +} + + +} +} // namespace From 71a22e45b0448aa9dc496012ac43326151a6d40b Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Fri, 3 Dec 2021 23:17:07 +0300 Subject: [PATCH 188/226] add celu, hardsigmoid, selu, thresholdedrelu layers --- .../dnn/include/opencv2/dnn/all_layers.hpp | 34 +++ modules/dnn/src/cuda/activations.cu | 28 ++ modules/dnn/src/cuda/functors.hpp | 78 +++++ .../dnn/src/cuda4dnn/kernels/activations.hpp | 12 + .../src/cuda4dnn/primitives/activation.hpp | 62 ++++ modules/dnn/src/init.cpp | 4 + modules/dnn/src/layers/elementwise_layers.cpp | 288 +++++++++++++----- .../dnn/src/onnx/onnx_graph_simplifier.cpp | 65 ++++ modules/dnn/src/opencl/activations.cl | 34 +++ ..._conformance_layer_parser_denylist.inl.hpp | 10 - 10 files changed, 526 insertions(+), 89 deletions(-) diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index 26d7a9b069..44b16f7800 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -738,6 +738,40 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + class CV_EXPORTS CeluLayer : public ActivationLayer + { + public: + float alpha; + + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS HardSigmoidLayer : public ActivationLayer + { + public: + float alpha; + float beta; + + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS SeluLayer : public ActivationLayer + { + public: + float alpha; + float gamma; + + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS ThresholdedReluLayer : public ActivationLayer + { + public: + float alpha; + + static Ptr create(const LayerParams ¶ms); + }; + class CV_EXPORTS ActivationLayerInt8 : public ActivationLayer { public: diff --git a/modules/dnn/src/cuda/activations.cu b/modules/dnn/src/cuda/activations.cu index 3d99a03ae3..f5dafcea7f 100644 --- a/modules/dnn/src/cuda/activations.cu +++ b/modules/dnn/src/cuda/activations.cu @@ -233,6 +233,26 @@ void tan(const Stream& stream, Span output, View input) { generic_op>(stream, output, input); } +template +void celu(const Stream& stream, Span output, View input, T alpha) { + generic_op>(stream, output, input, {alpha}); +} + +template +void hardsigmoid(const Stream& stream, Span output, View input, T alpha, T beta) { + generic_op>(stream, output, input, {alpha, beta}); +} + +template +void selu(const Stream& stream, Span output, View input, T alpha, T gamma) { + generic_op>(stream, output, input, {alpha, gamma}); +} + +template +void thresholdedrelu(const Stream& stream, Span output, View input, T alpha) { + generic_op>(stream, output, input, {alpha}); +} + template void abs(const Stream& stream, Span output, View input) { generic_op>(stream, output, input); @@ -286,6 +306,10 @@ template void sinh<__half>(const Stream&, Span<__half>, View<__half>); template void softplus<__half>(const Stream&, Span<__half>, View<__half>); template void softsign<__half>(const Stream&, Span<__half>, View<__half>); template void tan<__half>(const Stream&, Span<__half>, View<__half>); +template void celu<__half>(const Stream&, Span<__half>, View<__half>, __half); +template void hardsigmoid<__half>(const Stream&, Span<__half>, View<__half>, __half, __half); +template void selu<__half>(const Stream&, Span<__half>, View<__half>, __half, __half); +template void thresholdedrelu<__half>(const Stream&, Span<__half>, View<__half>, __half); template void power<__half>(const Stream&, Span<__half>, View<__half>, __half, __half, __half); template void exp<__half>(const Stream&, Span<__half>, View<__half>, __half, __half); #endif @@ -321,6 +345,10 @@ template void sinh(const Stream&, Span, View); template void softplus(const Stream&, Span, View); template void softsign(const Stream&, Span, View); template void tan(const Stream&, Span, View); +template void celu(const Stream&, Span, View, float); +template void hardsigmoid(const Stream&, Span, View, float, float); +template void selu(const Stream&, Span, View, float, float); +template void thresholdedrelu(const Stream&, Span, View, float); template void power(const Stream&, Span, View, float, float, float); template void exp(const Stream&, Span, View, float, float); diff --git a/modules/dnn/src/cuda/functors.hpp b/modules/dnn/src/cuda/functors.hpp index c3d1669344..640c7c8ad6 100644 --- a/modules/dnn/src/cuda/functors.hpp +++ b/modules/dnn/src/cuda/functors.hpp @@ -528,6 +528,84 @@ struct TanFunctor { } }; +template +struct CeluFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() : alpha(1) { } + CUDA4DNN_HOST_DEVICE Params(T alpha_) : alpha(alpha_) { } + T alpha; + }; + + CUDA4DNN_DEVICE CeluFunctor() : CeluFunctor(Params{}) { } + CUDA4DNN_DEVICE CeluFunctor(const Params& params) : alpha{params.alpha} { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::min; + using csl::device::max; + using csl::device::expm1; + return max(T(0), value) + min(T(0), alpha * expm1(value / alpha)); + } + + T alpha; +}; + +template +struct HardSigmoidFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() : alpha(0.2), beta(0.5) { } + CUDA4DNN_HOST_DEVICE Params(T alpha_, T beta_) : alpha(alpha_), beta(beta_) { } + T alpha, beta; + }; + + CUDA4DNN_DEVICE HardSigmoidFunctor() : HardSigmoidFunctor(Params{}) { } + CUDA4DNN_DEVICE HardSigmoidFunctor(const Params& params): alpha{params.alpha}, beta{params.beta} { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::clamp; + return clamp(alpha * value + beta, T(0), T(1)); + } + + T alpha, beta; +}; + +template +struct SeluFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() : alpha(1.6732632423543772848170429916717), + gamma(1.0507009873554804934193349852946) { } + CUDA4DNN_HOST_DEVICE Params(T alpha_, T gamma_) : alpha(alpha_), gamma(gamma_) { } + T alpha, gamma; + }; + + CUDA4DNN_DEVICE SeluFunctor() : SeluFunctor(Params{}) { } + CUDA4DNN_DEVICE SeluFunctor(const Params& params): alpha{params.alpha}, gamma{params.gamma} { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::expm1; + return gamma * (value > T(0) ? value : alpha * expm1(value)); + } + + T alpha, gamma; +}; + +template +struct ThresholdedReluFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() : alpha(1) { } + CUDA4DNN_HOST_DEVICE Params(T alpha_) : alpha(alpha_) { } + T alpha; + }; + + CUDA4DNN_DEVICE ThresholdedReluFunctor() : ThresholdedReluFunctor(Params{}) { } + CUDA4DNN_DEVICE ThresholdedReluFunctor(const Params& params) : alpha{params.alpha} { } + + CUDA4DNN_DEVICE T operator()(T value) { + return (value > alpha) ? value : T(0); + } + + T alpha; +}; + template struct PowerFunctor { struct Params { diff --git a/modules/dnn/src/cuda4dnn/kernels/activations.hpp b/modules/dnn/src/cuda4dnn/kernels/activations.hpp index 854bc8ac0c..ef1f6da3e6 100644 --- a/modules/dnn/src/cuda4dnn/kernels/activations.hpp +++ b/modules/dnn/src/cuda4dnn/kernels/activations.hpp @@ -105,6 +105,18 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { template void tan(const csl::Stream& stream, csl::Span output, csl::View input); + template + void celu(const csl::Stream& stream, csl::Span output, csl::View input, T alpha); + + template + void hardsigmoid(const csl::Stream& stream, csl::Span output, csl::View input, T alpha, T beta); + + template + void selu(const csl::Stream& stream, csl::Span output, csl::View input, T alpha, T gamma); + + template + void thresholdedrelu(const csl::Stream& stream, csl::Span output, csl::View input, T alpha); + template void power(const csl::Stream& stream, csl::Span output, csl::View input, T exp, T scale, T shift); diff --git a/modules/dnn/src/cuda4dnn/primitives/activation.hpp b/modules/dnn/src/cuda4dnn/primitives/activation.hpp index 4691996d4e..39ebf513a7 100644 --- a/modules/dnn/src/cuda4dnn/primitives/activation.hpp +++ b/modules/dnn/src/cuda4dnn/primitives/activation.hpp @@ -490,6 +490,68 @@ namespace cv { namespace dnn { namespace cuda4dnn { csl::Stream stream; }; + template + class CeluOp final : public BaseOp { + public: + CeluOp(csl::Stream stream_, T alpha_) : stream(std::move(stream_)), alpha{ alpha_ } { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::celu(stream, output, input, alpha); + } + + private: + csl::Stream stream; + const T alpha; + }; + + template + class HardSigmoidOp final : public BaseOp { + public: + HardSigmoidOp(csl::Stream stream_, T alpha_, T beta_) + : stream(std::move(stream_)), alpha{ alpha_ }, beta{ beta_ } { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::hardsigmoid(stream, output, input, alpha, beta); + } + + private: + csl::Stream stream; + const T alpha, beta; + }; + + template + class SeluOp final : public BaseOp { + public: + SeluOp(csl::Stream stream_, T alpha_, T gamma_) + : stream(std::move(stream_)), alpha{ alpha_ }, gamma{ gamma_ } { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::selu(stream, output, input, alpha, gamma); + } + + private: + csl::Stream stream; + const T alpha, gamma; + }; + + template + class ThresholdedReluOp final : public BaseOp { + public: + ThresholdedReluOp(csl::Stream stream_, T alpha_) : stream(std::move(stream_)), alpha{ alpha_ } { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::thresholdedrelu(stream, output, input, alpha); + } + + private: + csl::Stream stream; + const T alpha; + }; + template class PowerOp final : public BaseOp { public: diff --git a/modules/dnn/src/init.cpp b/modules/dnn/src/init.cpp index 89a91e17ae..55ed1e5d17 100644 --- a/modules/dnn/src/init.cpp +++ b/modules/dnn/src/init.cpp @@ -132,6 +132,10 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(Softplus, SoftplusLayer); CV_DNN_REGISTER_LAYER_CLASS(Softsign, SoftsignLayer); CV_DNN_REGISTER_LAYER_CLASS(Tan, TanLayer); + CV_DNN_REGISTER_LAYER_CLASS(Celu, CeluLayer); + CV_DNN_REGISTER_LAYER_CLASS(HardSigmoid, HardSigmoidLayer); + CV_DNN_REGISTER_LAYER_CLASS(Selu, SeluLayer); + CV_DNN_REGISTER_LAYER_CLASS(ThresholdedRelu,ThresholdedReluLayer); CV_DNN_REGISTER_LAYER_CLASS(BatchNorm, BatchNormLayer); CV_DNN_REGISTER_LAYER_CLASS(MaxUnpool, MaxUnpoolLayer); CV_DNN_REGISTER_LAYER_CLASS(Dropout, BlankLayer); diff --git a/modules/dnn/src/layers/elementwise_layers.cpp b/modules/dnn/src/layers/elementwise_layers.cpp index 772dfca602..bfabef9d68 100644 --- a/modules/dnn/src/layers/elementwise_layers.cpp +++ b/modules/dnn/src/layers/elementwise_layers.cpp @@ -71,6 +71,7 @@ namespace dnn using std::abs; using std::exp; +using std::expm1; using std::tanh; using std::pow; using std::ceil; @@ -728,6 +729,20 @@ struct BaseDefaultFunctor : public BaseFunctor return true; } +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + CV_Error(Error::StsNotImplemented, ""); + } +#endif + +#ifdef HAVE_HALIDE + void attachHalide(const Halide::Expr& input, Halide::Func& top) + { + CV_Error(Error::StsNotImplemented, ""); + } +#endif // HAVE_HALIDE + #ifdef HAVE_DNN_IE_NN_BUILDER_2019 InferenceEngine::Builder::Layer initInfEngineBuilderAPI() { @@ -746,8 +761,6 @@ struct BaseDefaultFunctor : public BaseFunctor ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) { CV_Error(Error::StsNotImplemented, ""); - ml::Operand operand; - return operand; } #endif @@ -759,20 +772,6 @@ struct BaseDefaultFunctor : public BaseFunctor } #endif // HAVE_VULKAN -#ifdef HAVE_CUDA - Ptr initCUDA(int target, csl::Stream stream) - { - CV_Error(Error::StsNotImplemented, ""); - } -#endif - -#ifdef HAVE_HALIDE - void attachHalide(const Halide::Expr& input, Halide::Func& top) - { - CV_Error(Error::StsNotImplemented, ""); - } -#endif // HAVE_HALIDE - private: static const char* const ocl_kernel_name; }; @@ -823,15 +822,6 @@ struct TanHFunctor : public BaseDefaultFunctor } #endif // HAVE_DNN_NGRAPH -#ifdef HAVE_WEBNN - ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) - { - CV_Error(Error::StsNotImplemented, ""); - ml::Operand operand; - return operand; - } -#endif - int64 getFLOPSPerElement() const { return 1; } }; @@ -935,15 +925,6 @@ struct MishFunctor : public BaseDefaultFunctor } #endif // HAVE_DNN_NGRAPH -#ifdef HAVE_WEBNN - ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) - { - CV_Error(Error::StsNotImplemented, ""); - ml::Operand operand; - return operand; - } -#endif - int64 getFLOPSPerElement() const { return 3; } }; @@ -996,15 +977,6 @@ struct SigmoidFunctor : public BaseDefaultFunctor } #endif // HAVE_DNN_NGRAPH -#ifdef HAVE_WEBNN - ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) - { - CV_Error(Error::StsNotImplemented, ""); - ml::Operand operand; - return operand; - } -#endif - int64 getFLOPSPerElement() const { return 3; } }; @@ -1123,15 +1095,6 @@ struct AbsValFunctor : public BaseDefaultFunctor } #endif // HAVE_DNN_NGRAPH -#ifdef HAVE_WEBNN - ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) - { - CV_Error(Error::StsNotImplemented, ""); - ml::Operand operand; - return operand; - } -#endif - int64 getFLOPSPerElement() const { return 1; } }; @@ -1261,15 +1224,6 @@ struct LogFunctor : public BaseDefaultFunctor return log(x); } -#ifdef HAVE_WEBNN - ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) - { - CV_Error(Error::StsNotImplemented, ""); - ml::Operand operand; - return operand; - } -#endif - #ifdef HAVE_CUDA Ptr initCUDA(int target, csl::Stream stream) { @@ -1367,15 +1321,6 @@ struct SqrtFunctor : public BaseDefaultFunctor } #endif // HAVE_DNN_NGRAPH -#ifdef HAVE_WEBNN - ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) - { - CV_Error(Error::StsNotImplemented, ""); - ml::Operand operand; - return operand; - } -#endif - int64 getFLOPSPerElement() const { return 1; } }; @@ -1822,6 +1767,156 @@ struct TanFunctor : public BaseDefaultFunctor template<> const char* const BaseDefaultFunctor::ocl_kernel_name = "TanForward"; +struct CeluFunctor : public BaseDefaultFunctor +{ + typedef CeluLayer Layer; + + float alpha; + + explicit CeluFunctor(float alpha_ = 1.f) : alpha(alpha_) {} + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA; + } + + inline float calculate(float x) const + { + return max(0.f, x) + min(0.f, alpha * expm1(x / alpha)); + } + + inline void setKernelParams(ocl::Kernel& kernel) const + { + kernel.set(3, alpha); + } + +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream, alpha); + } +#endif + + int64 getFLOPSPerElement() const { return 1; } +}; + +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "CeluForward"; + +struct HardSigmoidFunctor : public BaseDefaultFunctor +{ + typedef HardSigmoidLayer Layer; + + float alpha; + float beta; + + explicit HardSigmoidFunctor(float alpha_ = 0.2f, float beta_ = 0.5f) : alpha(alpha_), beta(beta_) {} + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA; + } + + inline float calculate(float x) const + { + return max(0.f, min(1.f, alpha * x + beta)); + } + + inline void setKernelParams(ocl::Kernel& kernel) const + { + kernel.set(3, alpha); + kernel.set(4, beta); + } + +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream, alpha, beta); + } +#endif + + int64 getFLOPSPerElement() const { return 1; } +}; + +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "HardSigmoidForward"; + +struct SeluFunctor : public BaseDefaultFunctor +{ + typedef SeluLayer Layer; + + float alpha; + float gamma; + + explicit SeluFunctor(float alpha_ = 1.67326319217681884765625f, + float gamma_ = 1.05070102214813232421875f) : alpha(alpha_), gamma(gamma_) {} + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA; + } + + inline float calculate(float x) const + { + return gamma * (x > 0.f ? x : alpha * expm1(x)); + } + + inline void setKernelParams(ocl::Kernel& kernel) const + { + kernel.set(3, alpha); + kernel.set(4, gamma); + } + +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream, alpha, gamma); + } +#endif + + int64 getFLOPSPerElement() const { return 1; } +}; + +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "SeluForward"; + +struct ThresholdedReluFunctor : public BaseDefaultFunctor +{ + typedef ThresholdedReluLayer Layer; + + float alpha; + + explicit ThresholdedReluFunctor(float alpha_ = 1.f) : alpha(alpha_) {} + + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA; + } + + inline float calculate(float x) const + { + return x > alpha ? x : 0.f; + } + + inline void setKernelParams(ocl::Kernel& kernel) const + { + kernel.set(3, alpha); + } + +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream, alpha); + } +#endif + + int64 getFLOPSPerElement() const { return 1; } +}; + +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "ThresholdedReluForward"; + struct PowerFunctor : public BaseFunctor { typedef PowerLayer Layer; @@ -2074,15 +2169,6 @@ struct ExpFunctor : public BaseDefaultFunctor } #endif // HAVE_DNN_NGRAPH -#ifdef HAVE_WEBNN - ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) - { - CV_Error(Error::StsNotImplemented, ""); - ml::Operand operand; - return operand; - } -#endif - int64 getFLOPSPerElement() const { return 3; } }; @@ -2489,6 +2575,50 @@ Ptr TanLayer::create(const LayerParams& params) return l; } +Ptr CeluLayer::create(const LayerParams& params) +{ + float alpha = params.get("alpha", 1.f); + Ptr l(new ElementWiseLayer(CeluFunctor(alpha))); + l->setParamsFrom(params); + l->alpha = alpha; + + return l; +} + +Ptr HardSigmoidLayer::create(const LayerParams& params) +{ + float alpha = params.get("alpha", 0.2f); + float beta = params.get("beta", 0.5f); + Ptr l(new ElementWiseLayer(HardSigmoidFunctor(alpha, beta))); + l->setParamsFrom(params); + l->alpha = alpha; + l->beta = beta; + + return l; +} + +Ptr SeluLayer::create(const LayerParams& params) +{ + float alpha = params.get("alpha", 1.67326319217681884765625f); + float gamma = params.get("gamma", 1.05070102214813232421875f); + Ptr l(new ElementWiseLayer(SeluFunctor(alpha, gamma))); + l->setParamsFrom(params); + l->alpha = alpha; + l->gamma = gamma; + + return l; +} + +Ptr ThresholdedReluLayer::create(const LayerParams& params) +{ + float alpha = params.get("alpha", 1.f); + Ptr l(new ElementWiseLayer(ThresholdedReluFunctor(alpha))); + l->setParamsFrom(params); + l->alpha = alpha; + + return l; +} + Ptr PowerLayer::create(const LayerParams& params) { float power = params.get("power", 1.0f); diff --git a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp index 81a5df1a28..80fe0b173e 100644 --- a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp +++ b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp @@ -242,6 +242,70 @@ public: } }; +class CeluSubgraph : public Subgraph +{ +public: + CeluSubgraph() : alpha(1.f) + { + int input = addNodeToMatch(""); + int div = addNodeToMatch("Div", input, addNodeToMatch("")); + int elu = addNodeToMatch("Elu", div); + addNodeToMatch("Mul", addNodeToMatch(""), elu); + setFusedNode("Celu", input); + } + + static float extractAlpha(const Ptr& net, int node_id, int input_id) + { + const Ptr node = net->getNode(node_id); + int const_id = getInputNodeId(net, node, input_id); + Ptr alpha_ptr = net->getNode(const_id); + opencv_onnx::NodeProto* alpha_node = alpha_ptr.dynamicCast()->node; + opencv_onnx::TensorProto alpha_proto = alpha_node->attribute(0).t(); + Mat alpha_mat = getMatFromTensor(alpha_proto); + return *alpha_mat.ptr(); + } + + virtual bool match(const Ptr& net, int nodeId, + std::vector& matchedNodesIds, + std::vector& targetNodesIds) CV_OVERRIDE + { + if (Subgraph::match(net, nodeId, matchedNodesIds, targetNodesIds)) + { + float alpha_div = extractAlpha(net, matchedNodesIds[0], 1); + float alpha_mul = extractAlpha(net, matchedNodesIds[2], 0); + float alpha_elu = 1.f; + + Ptr elu_ptr = net->getNode(matchedNodesIds[1]); + opencv_onnx::NodeProto* elu_node = elu_ptr.dynamicCast()->node; + + for (int i = 0; i < elu_node->attribute_size(); i++) + { + opencv_onnx::AttributeProto attr = elu_node->attribute(i); + if (attr.name() != "alpha") + continue; + alpha_elu = attr.f(); + } + + alpha = alpha_div; + return alpha_elu == 1.f && alpha_div == alpha_mul; + } + return false; + } + + virtual void finalize(const Ptr&, + const Ptr& fusedNode, + std::vector >&) CV_OVERRIDE + { + opencv_onnx::NodeProto* node = fusedNode.dynamicCast()->node; + opencv_onnx::AttributeProto* alpha_attr = node->add_attribute(); + alpha_attr->set_name("alpha"); + alpha_attr->set_f(alpha); + } + +protected: + float alpha; +}; + class NormalizeSubgraphBase : public Subgraph { public: @@ -662,6 +726,7 @@ void simplifySubgraphs(opencv_onnx::GraphProto& net) subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); + subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); diff --git a/modules/dnn/src/opencl/activations.cl b/modules/dnn/src/opencl/activations.cl index 02ed9345c3..040ee20d8a 100644 --- a/modules/dnn/src/opencl/activations.cl +++ b/modules/dnn/src/opencl/activations.cl @@ -272,3 +272,37 @@ __kernel void TanForward(const int n, __global T* in, __global T* out) { if(index < n) out[index] = tan(in[index]); } + +__kernel void CeluForward(const int n, __global T* in, __global T* out, + const KERNEL_ARG_DTYPE alpha) +{ + int index = get_global_id(0); + if(index < n) + out[index] = max(0.f, in[index]) + min(0.f, alpha * expm1(in[index] / alpha)); +} + +__kernel void HardSigmoidForward(const int n, __global T* in, __global T* out, + const KERNEL_ARG_DTYPE alpha, + const KERNEL_ARG_DTYPE beta) +{ + int index = get_global_id(0); + if(index < n) + out[index] = max(0.f, min(1.f, alpha * in[index] + beta)); +} + +__kernel void SeluForward(const int n, __global T* in, __global T* out, + const KERNEL_ARG_DTYPE alpha, + const KERNEL_ARG_DTYPE gamma) +{ + int index = get_global_id(0); + if(index < n) + out[index] = gamma * (in[index] > 0.f ? in[index] : alpha * expm1(in[index])); +} + +__kernel void ThresholdedReluForward(const int n, __global T* in, __global T* out, + const KERNEL_ARG_DTYPE alpha) +{ + int index = get_global_id(0); + if(index < n) + out[index] = (in[index] > alpha ? in[index] : 0.f); +} diff --git a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp index a69ace0d14..e5d0ead9da 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp @@ -57,7 +57,6 @@ "test_castlike_FLOAT_to_FLOAT16_expanded", "test_castlike_FLOAT_to_STRING", "test_castlike_STRING_to_FLOAT", -"test_celu", "test_clip", "test_clip_default_inbounds", "test_clip_default_int8_inbounds", @@ -173,9 +172,6 @@ "test_hardmax_example", "test_hardmax_negative_axis", "test_hardmax_one_hot", -"test_hardsigmoid", -"test_hardsigmoid_default", -"test_hardsigmoid_example", "test_identity_opt", "test_identity_sequence", "test_if", @@ -524,9 +520,6 @@ "test_sce_sum_expanded", "test_sce_sum_log_prob", "test_sce_sum_log_prob_expanded", -"test_selu", -"test_selu_default", -"test_selu_example", "test_sequence_insert_at_back", "test_sequence_insert_at_front", "test_shape", @@ -579,9 +572,6 @@ "test_tfidfvectorizer_tf_onlybigrams_levelempty", "test_tfidfvectorizer_tf_onlybigrams_skip5", "test_tfidfvectorizer_tf_uniandbigrams_skip5", -"test_thresholdedrelu", -"test_thresholdedrelu_default", -"test_thresholdedrelu_example", "test_tile", "test_tile_precomputed", "test_top_k", From bc3da25a4abfe3c20d175d4da8d1e2594612285e Mon Sep 17 00:00:00 2001 From: Maksim Shabunin Date: Mon, 20 Dec 2021 13:25:08 +0300 Subject: [PATCH 189/226] videoio: added gst-audio dependency --- modules/videoio/cmake/detect_gstreamer.cmake | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/videoio/cmake/detect_gstreamer.cmake b/modules/videoio/cmake/detect_gstreamer.cmake index 47ea7a0b30..fc6c347383 100644 --- a/modules/videoio/cmake/detect_gstreamer.cmake +++ b/modules/videoio/cmake/detect_gstreamer.cmake @@ -93,11 +93,12 @@ if(NOT HAVE_GSTREAMER AND PKG_CONFIG_FOUND) ocv_check_modules(GSTREAMER_riff gstreamer-riff-1.0) ocv_check_modules(GSTREAMER_pbutils gstreamer-pbutils-1.0) ocv_check_modules(GSTREAMER_video gstreamer-video-1.0) - if(GSTREAMER_base_FOUND AND GSTREAMER_app_FOUND AND GSTREAMER_riff_FOUND AND GSTREAMER_pbutils_FOUND AND GSTREAMER_video_FOUND) + ocv_check_modules(GSTREAMER_audio gstreamer-audio-1.0) + if(GSTREAMER_base_FOUND AND GSTREAMER_app_FOUND AND GSTREAMER_riff_FOUND AND GSTREAMER_pbutils_FOUND AND GSTREAMER_video_FOUND AND GSTREAMER_audio_FOUND) set(HAVE_GSTREAMER TRUE) set(GSTREAMER_VERSION ${GSTREAMER_base_VERSION}) # informational - set(GSTREAMER_LIBRARIES ${GSTREAMER_base_LIBRARIES} ${GSTREAMER_app_LIBRARIES} ${GSTREAMER_riff_LIBRARIES} ${GSTREAMER_pbutils_LIBRARIES} ${GSTREAMER_video_LIBRARIES}) - set(GSTREAMER_INCLUDE_DIRS ${GSTREAMER_base_INCLUDE_DIRS} ${GSTREAMER_app_INCLUDE_DIRS} ${GSTREAMER_riff_INCLUDE_DIRS} ${GSTREAMER_pbutils_INCLUDE_DIRS} ${GSTREAMER_video_INCLUDE_DIRS}) + set(GSTREAMER_LIBRARIES ${GSTREAMER_base_LIBRARIES} ${GSTREAMER_app_LIBRARIES} ${GSTREAMER_riff_LIBRARIES} ${GSTREAMER_pbutils_LIBRARIES} ${GSTREAMER_video_LIBRARIES} ${GSTREAMER_audio_LIBRARIES}) + set(GSTREAMER_INCLUDE_DIRS ${GSTREAMER_base_INCLUDE_DIRS} ${GSTREAMER_app_INCLUDE_DIRS} ${GSTREAMER_riff_INCLUDE_DIRS} ${GSTREAMER_pbutils_INCLUDE_DIRS} ${GSTREAMER_video_INCLUDE_DIRS} ${GSTREAMER_audio_INCLUDE_DIRS}) endif() endif() From a22dd28e0272ec0f1cfee8811d3f5f0392827c65 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 20 Dec 2021 15:03:21 +0000 Subject: [PATCH 190/226] videoio: fix ffmpeg standalone build --- modules/videoio/src/cap_ffmpeg_impl.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/videoio/src/cap_ffmpeg_impl.hpp b/modules/videoio/src/cap_ffmpeg_impl.hpp index d5be759f43..a724d7f724 100644 --- a/modules/videoio/src/cap_ffmpeg_impl.hpp +++ b/modules/videoio/src/cap_ffmpeg_impl.hpp @@ -67,6 +67,10 @@ #ifndef CV_UNUSED // Required for standalone compilation mode (OpenCV defines this in base.hpp) #define CV_UNUSED(name) (void)name #endif +#ifndef CV_Assert // Required for standalone compilation mode (OpenCV defines this in base.hpp) +#include +#define CV_Assert(expr) assert(expr) +#endif #ifdef __cplusplus extern "C" { From 8fa86d4d34939dff48a6c12dfe8440a80e160367 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 20 Dec 2021 15:32:05 +0000 Subject: [PATCH 191/226] ffmpeg/3.4: update FFmpeg wrapper 2021.12 - FFmpeg 3.4.9 --- 3rdparty/ffmpeg/ffmpeg.cmake | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/3rdparty/ffmpeg/ffmpeg.cmake b/3rdparty/ffmpeg/ffmpeg.cmake index 336bf0d227..ad10940335 100644 --- a/3rdparty/ffmpeg/ffmpeg.cmake +++ b/3rdparty/ffmpeg/ffmpeg.cmake @@ -1,8 +1,8 @@ -# Binaries branch name: ffmpeg/3.4_20211005 -# Binaries were created for OpenCV: 95c1d2a8872b222f32bd88db9f1efcbd9f70a9cf -ocv_update(FFMPEG_BINARIES_COMMIT "0bf6c0753d435d2c82c03c48db0c6e18ac79976c") -ocv_update(FFMPEG_FILE_HASH_BIN32 "55c25bbc13e4a12d4339b70d3b76987f") -ocv_update(FFMPEG_FILE_HASH_BIN64 "67caee9231c6843483b4de9815d6526e") +# Binaries branch name: ffmpeg/3.4_20211220 +# Binaries were created for OpenCV: a22dd28e0272ec0f1cfee8811d3f5f0392827c65 +ocv_update(FFMPEG_BINARIES_COMMIT "5a7644ec3940c6eed41c6ebb5a0602a5615fdb3f") +ocv_update(FFMPEG_FILE_HASH_BIN32 "8ad9de6f1f2ca77786748d1f3a4e83ea") +ocv_update(FFMPEG_FILE_HASH_BIN64 "2c670f068252e7cd28d3883993dc1d6e") ocv_update(FFMPEG_FILE_HASH_CMAKE "3b90f67f4b429e77d3da36698cef700c") function(download_win_ffmpeg script_var) From c4c43c3d267c7594f3befb84729440828f30f451 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 20 Dec 2021 16:30:21 +0000 Subject: [PATCH 192/226] ffmpeg/4.x: update FFmpeg wrapper 2021.12 - FFmpeg 4.4.1 --- 3rdparty/ffmpeg/ffmpeg.cmake | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/3rdparty/ffmpeg/ffmpeg.cmake b/3rdparty/ffmpeg/ffmpeg.cmake index 2eea658178..f6c1486598 100644 --- a/3rdparty/ffmpeg/ffmpeg.cmake +++ b/3rdparty/ffmpeg/ffmpeg.cmake @@ -1,8 +1,8 @@ -# Binaries branch name: ffmpeg/master_20211005 -# Binaries were created for OpenCV: 672399c751c431bbe52818b33fd3ca17b51e0e16 -ocv_update(FFMPEG_BINARIES_COMMIT "40b4666d1aa374205fd61373496e15d92ecd5313") -ocv_update(FFMPEG_FILE_HASH_BIN32 "c2f9a897d464a2dce2286f8067ad9d90") -ocv_update(FFMPEG_FILE_HASH_BIN64 "878a4e8fe5a4d68f18c9cdde543b9ead") +# Binaries branch name: ffmpeg/4.x_20211220 +# Binaries were created for OpenCV: 0e274fc4bed8a64ba4f1c201a21304286e217afc +ocv_update(FFMPEG_BINARIES_COMMIT "4d348507d156ec797a88a887cfa7f9129a35afac") +ocv_update(FFMPEG_FILE_HASH_BIN32 "eece4ec8304188117ffc7d5dfd0fc0ae") +ocv_update(FFMPEG_FILE_HASH_BIN64 "20deefbfe023c8b8d11a52e5a6527c6a") ocv_update(FFMPEG_FILE_HASH_CMAKE "8862c87496e2e8c375965e1277dee1c7") function(download_win_ffmpeg script_var) From 0a178a687a747fa9ad89fed3a66a368d53327028 Mon Sep 17 00:00:00 2001 From: rogday Date: Mon, 20 Dec 2021 19:53:37 +0300 Subject: [PATCH 193/226] fix const/x in Div --- modules/dnn/src/onnx/onnx_importer.cpp | 11 ++++++++++- modules/dnn/test/test_onnx_importer.cpp | 5 +++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 5d88844e61..7bedf9543f 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -1472,7 +1472,16 @@ void ONNXImporter::parseMul(LayerParams& layerParams, const opencv_onnx::NodePro blob = blob.reshape(1, 1); if (blob.total() == 1) { float blob_value = blob.ptr()[0]; - float coeff = isDiv ? 1.0 / blob_value : blob_value; + float coeff = blob_value; + if (isDiv) + { + coeff = 1.f / blob_value; + if (constId == 0) + { + // Power layer calculates (x*scale + shift)^power, so const/x -> (x * (1/const) + 0)^(-1) + layerParams.set("power", -1.f); + } + } layerParams.set("scale", coeff); layerParams.type = "Power"; } diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index a432f2251e..d5a5a86091 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -1056,6 +1056,11 @@ TEST_P(Test_ONNX_layers, SubFromConst) testONNXModels("sub_from_const_broadcast"); } +TEST_P(Test_ONNX_layers, DivConst) +{ + testONNXModels("div_const"); +} + INSTANTIATE_TEST_CASE_P(/*nothing*/, Test_ONNX_layers, dnnBackendsAndTargets()); class Test_ONNX_nets : public Test_ONNX_layers From 4dbba5ac98608c387bf7ec10595b62b02e321cbe Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 21 Dec 2021 14:32:19 +0000 Subject: [PATCH 194/226] videoio: add non zero check --- modules/videoio/src/cap_gstreamer.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/videoio/src/cap_gstreamer.cpp b/modules/videoio/src/cap_gstreamer.cpp index adbc94fc95..16b09cc590 100644 --- a/modules/videoio/src/cap_gstreamer.cpp +++ b/modules/videoio/src/cap_gstreamer.cpp @@ -527,6 +527,7 @@ bool GStreamerCapture::grabAudioFrame() CV_Error(Error::StsError, "GStreamer: gst_audio_info_from_caps() is failed. Can't handle unknown layout"); } int bpf = GST_AUDIO_INFO_BPF(&info); + CV_CheckGT(bpf, 0, ""); GstStructure* structure = gst_caps_get_structure(frame_caps, 0); // no lifetime transfer if (!structure) From 3e01a387bad0b9fe7f639eacd724403dec7971b2 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 21 Dec 2021 15:44:59 +0000 Subject: [PATCH 195/226] highgui: fix Win32 with OPENGL=ON --- modules/highgui/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/highgui/CMakeLists.txt b/modules/highgui/CMakeLists.txt index 8339a1d7d2..416fb5a315 100644 --- a/modules/highgui/CMakeLists.txt +++ b/modules/highgui/CMakeLists.txt @@ -163,6 +163,10 @@ if(APPLE) add_apple_compiler_options(${the_module}) endif() +if(HAVE_WIN32UI AND HAVE_OPENGL AND OPENGL_LIBRARIES) + ocv_target_link_libraries(${the_module} PRIVATE "${OPENGL_LIBRARIES}") +endif() + if(MSVC AND NOT BUILD_SHARED_LIBS AND BUILD_WITH_STATIC_CRT) set_target_properties(${the_module} PROPERTIES LINK_FLAGS "/NODEFAULTLIB:atlthunk.lib /NODEFAULTLIB:atlsd.lib /NODEFAULTLIB:libcmt.lib /DEBUG") endif() From dcc32e84b914cada7f07a38eedaa5731b8dfce07 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 21 Dec 2021 15:51:22 +0000 Subject: [PATCH 196/226] highgui: fix Win32 with OPENGL=ON --- modules/highgui/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/highgui/CMakeLists.txt b/modules/highgui/CMakeLists.txt index 27cfbb3672..0d63d42fea 100644 --- a/modules/highgui/CMakeLists.txt +++ b/modules/highgui/CMakeLists.txt @@ -271,6 +271,10 @@ if(APPLE) add_apple_compiler_options(${the_module}) endif() +if(OPENCV_HIGHGUI_BUILTIN_BACKEND STREQUAL "WIN32UI" AND HAVE_OPENGL AND OPENGL_LIBRARIES) + ocv_target_link_libraries(${the_module} PRIVATE "${OPENGL_LIBRARIES}") +endif() + if(MSVC AND NOT BUILD_SHARED_LIBS AND BUILD_WITH_STATIC_CRT) set_target_properties(${the_module} PROPERTIES LINK_FLAGS "/NODEFAULTLIB:atlthunk.lib /NODEFAULTLIB:atlsd.lib /NODEFAULTLIB:libcmt.lib /DEBUG") endif() From c80b2706781c83bbb6d49fcebac1ce33333d2c73 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 21 Dec 2021 16:34:48 +0000 Subject: [PATCH 197/226] cmake: force lowercase plugins internal names --- modules/core/CMakeLists.txt | 1 + modules/highgui/cmake/init.cmake | 1 + modules/videoio/CMakeLists.txt | 1 + 3 files changed, 3 insertions(+) diff --git a/modules/core/CMakeLists.txt b/modules/core/CMakeLists.txt index 51a0abbfe8..b78bb98162 100644 --- a/modules/core/CMakeLists.txt +++ b/modules/core/CMakeLists.txt @@ -27,6 +27,7 @@ set(PARALLEL_ENABLE_PLUGINS "${PARALLEL_ENABLE_PLUGINS_DEFAULT}" CACHE BOOL "All # TODO building plugins with OpenCV is not supported yet #set(PARALLEL_PLUGIN_LIST "" CACHE STRING "List of parallel backends to be compiled as plugins (tbb, openmp or special value 'all')") #string(REPLACE "," ";" PARALLEL_PLUGIN_LIST "${PARALLEL_PLUGIN_LIST}") # support comma-separated list (,) too +#string(TOLOWER "${PARALLEL_PLUGIN_LIST}" PARALLEL_PLUGIN_LIST) ocv_add_module(core diff --git a/modules/highgui/cmake/init.cmake b/modules/highgui/cmake/init.cmake index 2002ff0e9d..0b4178c868 100644 --- a/modules/highgui/cmake/init.cmake +++ b/modules/highgui/cmake/init.cmake @@ -8,6 +8,7 @@ if(PROJECT_NAME STREQUAL "OpenCV") mark_as_advanced(HIGHGUI_PLUGIN_LIST HIGHGUI_ENABLE_PLUGINS) string(REPLACE "," ";" HIGHGUI_PLUGIN_LIST "${HIGHGUI_PLUGIN_LIST}") # support comma-separated list (,) too + string(TOLOWER "${HIGHGUI_PLUGIN_LIST}" HIGHGUI_PLUGIN_LIST) if(NOT HIGHGUI_ENABLE_PLUGINS) if(HIGHGUI_PLUGIN_LIST) message(WARNING "HighGUI: plugins are disabled through HIGHGUI_ENABLE_PLUGINS, so HIGHGUI_PLUGIN_LIST='${HIGHGUI_PLUGIN_LIST}' is ignored") diff --git a/modules/videoio/CMakeLists.txt b/modules/videoio/CMakeLists.txt index 98127d1c98..377029f45f 100644 --- a/modules/videoio/CMakeLists.txt +++ b/modules/videoio/CMakeLists.txt @@ -8,6 +8,7 @@ set(VIDEOIO_ENABLE_PLUGINS "${VIDEOIO_ENABLE_PLUGINS_DEFAULT}" CACHE BOOL "Allow mark_as_advanced(VIDEOIO_PLUGIN_LIST VIDEOIO_ENABLE_PLUGINS) string(REPLACE "," ";" VIDEOIO_PLUGIN_LIST "${VIDEOIO_PLUGIN_LIST}") # support comma-separated list (,) too +string(TOLOWER "${VIDEOIO_PLUGIN_LIST}" VIDEOIO_PLUGIN_LIST) if(NOT VIDEOIO_ENABLE_PLUGINS) if(VIDEOIO_PLUGIN_LIST) message(WARNING "VideoIO: plugins are disabled through VIDEOIO_ENABLE_PLUGINS, so VIDEOIO_PLUGIN_LIST='${VIDEOIO_PLUGIN_LIST}' is ignored") From 54c180092d2ca02e0460eac7176cab23890fc11e Mon Sep 17 00:00:00 2001 From: dwardor <50771662+dwardor@users.noreply.github.com> Date: Wed, 22 Dec 2021 13:00:00 +0100 Subject: [PATCH 198/226] Merge pull request #21114 from dwardor:patch-1 * Fix compile against lapack-3.10.0 Fix compilation against lapack >= 3.9.1 and 3.10.0 while not breaking older versions OpenCVFindLAPACK.cmake & CMakeLists.txt: determine OPENCV_USE_LAPACK_PREFIX from LAPACK_VERSION hal_internal.cpp : Only apply LAPACK_FUNC to functions whose number of inputs depends on LAPACK_FORTRAN_STR_LEN in lapack >= 3.9.1 lapack_check.cpp : remove LAPACK_FUNC which is not OK as function are not used with input parameters (so lapack.h preprocessing of "LAPACK_xxxx(...)" is not applicable with lapack >= 3.9.1 If not removed lapack_check fails so LAPACK is deactivated in build (not want we want) use OCV_ prefix and don't use Global, instead generate OCV_LAPACK_FUNC depending on CMake Conditions Remove CONFIG from find_package(LAPACK) and use LAPACK_GLOBAL and LAPACK_NAME to figure out if using netlib's reference LAPACK implementation and how to #define OCV_LAPACK_FUNC(f) * Fix typos and grammar in comments --- cmake/OpenCVFindLAPACK.cmake | 17 +++++++++++++++ modules/core/src/hal_internal.cpp | 36 +++++++++++++++---------------- 2 files changed, 35 insertions(+), 18 deletions(-) diff --git a/cmake/OpenCVFindLAPACK.cmake b/cmake/OpenCVFindLAPACK.cmake index 342bebc723..3f17b7b289 100644 --- a/cmake/OpenCVFindLAPACK.cmake +++ b/cmake/OpenCVFindLAPACK.cmake @@ -51,6 +51,23 @@ macro(ocv_lapack_check) if(NOT "${OPENCV_CBLAS_H_PATH_${_lapack_impl}}" STREQUAL "${OPENCV_LAPACKE_H_PATH_${_lapack_impl}}") list(APPEND _lapack_content "#include \"${OPENCV_LAPACKE_H_PATH_${_lapack_impl}}\"") endif() + list(APPEND _lapack_content " +#if defined(LAPACK_GLOBAL) || defined(LAPACK_NAME) +/* + * Using netlib's reference LAPACK implementation version >= 3.4.0 (first with C interface). + * Use LAPACK_xxxx to transparently (via predefined lapack macros) deal with pre and post 3.9.1 versions. + * LAPACK 3.9.1 introduces LAPACK_FORTRAN_STRLEN_END and modifies (through preprocessing) the declarations of the following functions used in opencv + * sposv_, dposv_, spotrf_, dpotrf_, sgesdd_, dgesdd_, sgels_, dgels_ + * which end up with an extra parameter. + * So we also need to preprocess the function calls in opencv coding by prefixing them with LAPACK_. + * The good news is the preprocessing works fine whatever netlib's LAPACK version. + */ +#define OCV_LAPACK_FUNC(f) LAPACK_##f +#else +/* Using other LAPACK implementations so fall back to opencv's assumption until now */ +#define OCV_LAPACK_FUNC(f) f##_ +#endif +") if(${_lapack_add_extern_c}) list(APPEND _lapack_content "}") endif() diff --git a/modules/core/src/hal_internal.cpp b/modules/core/src/hal_internal.cpp index 483281d1f7..cbe02780d2 100644 --- a/modules/core/src/hal_internal.cpp +++ b/modules/core/src/hal_internal.cpp @@ -163,9 +163,9 @@ lapack_Cholesky(fptype* a, size_t a_step, int m, fptype* b, size_t b_step, int n if(n == 1 && b_step == sizeof(fptype)) { if(typeid(fptype) == typeid(float)) - sposv_(L, &m, &n, (float*)a, &lda, (float*)b, &m, &lapackStatus); + OCV_LAPACK_FUNC(sposv)(L, &m, &n, (float*)a, &lda, (float*)b, &m, &lapackStatus); else if(typeid(fptype) == typeid(double)) - dposv_(L, &m, &n, (double*)a, &lda, (double*)b, &m, &lapackStatus); + OCV_LAPACK_FUNC(dposv)(L, &m, &n, (double*)a, &lda, (double*)b, &m, &lapackStatus); } else { @@ -174,9 +174,9 @@ lapack_Cholesky(fptype* a, size_t a_step, int m, fptype* b, size_t b_step, int n transpose(b, ldb, tmpB, m, m, n); if(typeid(fptype) == typeid(float)) - sposv_(L, &m, &n, (float*)a, &lda, (float*)tmpB, &m, &lapackStatus); + OCV_LAPACK_FUNC(sposv)(L, &m, &n, (float*)a, &lda, (float*)tmpB, &m, &lapackStatus); else if(typeid(fptype) == typeid(double)) - dposv_(L, &m, &n, (double*)a, &lda, (double*)tmpB, &m, &lapackStatus); + OCV_LAPACK_FUNC(dposv)(L, &m, &n, (double*)a, &lda, (double*)tmpB, &m, &lapackStatus); transpose(tmpB, m, b, ldb, n, m); delete[] tmpB; @@ -185,9 +185,9 @@ lapack_Cholesky(fptype* a, size_t a_step, int m, fptype* b, size_t b_step, int n else { if(typeid(fptype) == typeid(float)) - spotrf_(L, &m, (float*)a, &lda, &lapackStatus); + OCV_LAPACK_FUNC(spotrf)(L, &m, (float*)a, &lda, &lapackStatus); else if(typeid(fptype) == typeid(double)) - dpotrf_(L, &m, (double*)a, &lda, &lapackStatus); + OCV_LAPACK_FUNC(dpotrf)(L, &m, (double*)a, &lda, &lapackStatus); } if(lapackStatus == 0) *info = true; @@ -227,17 +227,17 @@ lapack_SVD(fptype* a, size_t a_step, fptype *w, fptype* u, size_t u_step, fptype } if(typeid(fptype) == typeid(float)) - sgesdd_(mode, &m, &n, (float*)a, &lda, (float*)w, (float*)u, &ldu, (float*)vt, &ldv, (float*)&work1, &lwork, iworkBuf, info); + OCV_LAPACK_FUNC(sgesdd)(mode, &m, &n, (float*)a, &lda, (float*)w, (float*)u, &ldu, (float*)vt, &ldv, (float*)&work1, &lwork, iworkBuf, info); else if(typeid(fptype) == typeid(double)) - dgesdd_(mode, &m, &n, (double*)a, &lda, (double*)w, (double*)u, &ldu, (double*)vt, &ldv, (double*)&work1, &lwork, iworkBuf, info); + OCV_LAPACK_FUNC(dgesdd)(mode, &m, &n, (double*)a, &lda, (double*)w, (double*)u, &ldu, (double*)vt, &ldv, (double*)&work1, &lwork, iworkBuf, info); lwork = (int)round(work1); //optimal buffer size fptype* buffer = new fptype[lwork + 1]; if(typeid(fptype) == typeid(float)) - sgesdd_(mode, &m, &n, (float*)a, &lda, (float*)w, (float*)u, &ldu, (float*)vt, &ldv, (float*)buffer, &lwork, iworkBuf, info); + OCV_LAPACK_FUNC(sgesdd)(mode, &m, &n, (float*)a, &lda, (float*)w, (float*)u, &ldu, (float*)vt, &ldv, (float*)buffer, &lwork, iworkBuf, info); else if(typeid(fptype) == typeid(double)) - dgesdd_(mode, &m, &n, (double*)a, &lda, (double*)w, (double*)u, &ldu, (double*)vt, &ldv, (double*)buffer, &lwork, iworkBuf, info); + OCV_LAPACK_FUNC(dgesdd)(mode, &m, &n, (double*)a, &lda, (double*)w, (double*)u, &ldu, (double*)vt, &ldv, (double*)buffer, &lwork, iworkBuf, info); if(!(flags & CV_HAL_SVD_NO_UV)) transpose_square_inplace(vt, ldv, n); @@ -288,18 +288,18 @@ lapack_QR(fptype* a, size_t a_step, int m, int n, int k, fptype* b, size_t b_ste if (k == 1 && b_step == sizeof(fptype)) { if (typeid(fptype) == typeid(float)) - sgels_(mode, &m, &n, &k, (float*)tmpA, &ldtmpA, (float*)b, &m, (float*)&work1, &lwork, info); + OCV_LAPACK_FUNC(sgels)(mode, &m, &n, &k, (float*)tmpA, &ldtmpA, (float*)b, &m, (float*)&work1, &lwork, info); else if (typeid(fptype) == typeid(double)) - dgels_(mode, &m, &n, &k, (double*)tmpA, &ldtmpA, (double*)b, &m, (double*)&work1, &lwork, info); + OCV_LAPACK_FUNC(dgels)(mode, &m, &n, &k, (double*)tmpA, &ldtmpA, (double*)b, &m, (double*)&work1, &lwork, info); lwork = cvRound(work1); //optimal buffer size std::vector workBufMemHolder(lwork + 1); fptype* buffer = &workBufMemHolder.front(); if (typeid(fptype) == typeid(float)) - sgels_(mode, &m, &n, &k, (float*)tmpA, &ldtmpA, (float*)b, &m, (float*)buffer, &lwork, info); + OCV_LAPACK_FUNC(sgels)(mode, &m, &n, &k, (float*)tmpA, &ldtmpA, (float*)b, &m, (float*)buffer, &lwork, info); else if (typeid(fptype) == typeid(double)) - dgels_(mode, &m, &n, &k, (double*)tmpA, &ldtmpA, (double*)b, &m, (double*)buffer, &lwork, info); + OCV_LAPACK_FUNC(dgels)(mode, &m, &n, &k, (double*)tmpA, &ldtmpA, (double*)b, &m, (double*)buffer, &lwork, info); } else { @@ -309,18 +309,18 @@ lapack_QR(fptype* a, size_t a_step, int m, int n, int k, fptype* b, size_t b_ste transpose(b, ldb, tmpB, m, m, k); if (typeid(fptype) == typeid(float)) - sgels_(mode, &m, &n, &k, (float*)tmpA, &ldtmpA, (float*)tmpB, &m, (float*)&work1, &lwork, info); + OCV_LAPACK_FUNC(sgels)(mode, &m, &n, &k, (float*)tmpA, &ldtmpA, (float*)tmpB, &m, (float*)&work1, &lwork, info); else if (typeid(fptype) == typeid(double)) - dgels_(mode, &m, &n, &k, (double*)tmpA, &ldtmpA, (double*)tmpB, &m, (double*)&work1, &lwork, info); + OCV_LAPACK_FUNC(dgels)(mode, &m, &n, &k, (double*)tmpA, &ldtmpA, (double*)tmpB, &m, (double*)&work1, &lwork, info); lwork = cvRound(work1); //optimal buffer size std::vector workBufMemHolder(lwork + 1); fptype* buffer = &workBufMemHolder.front(); if (typeid(fptype) == typeid(float)) - sgels_(mode, &m, &n, &k, (float*)tmpA, &ldtmpA, (float*)tmpB, &m, (float*)buffer, &lwork, info); + OCV_LAPACK_FUNC(sgels)(mode, &m, &n, &k, (float*)tmpA, &ldtmpA, (float*)tmpB, &m, (float*)buffer, &lwork, info); else if (typeid(fptype) == typeid(double)) - dgels_(mode, &m, &n, &k, (double*)tmpA, &ldtmpA, (double*)tmpB, &m, (double*)buffer, &lwork, info); + OCV_LAPACK_FUNC(dgels)(mode, &m, &n, &k, (double*)tmpA, &ldtmpA, (double*)tmpB, &m, (double*)buffer, &lwork, info); transpose(tmpB, m, b, ldb, k, m); } From b1a57c4cb29d2c6541d367085ff0631203bc386b Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 22 Dec 2021 12:38:21 +0000 Subject: [PATCH 199/226] fix 3.4 links --- .../js_assets/js_image_classification.html | 2 +- .../js_assets/js_image_classification_model_info.json | 10 +++++----- .../js_assets/js_image_classification_with_camera.html | 2 +- doc/js_tutorials/js_assets/js_object_detection.html | 2 +- .../js_assets/js_object_detection_model_info.json | 6 +++--- .../js_assets/js_object_detection_with_camera.html | 2 +- doc/js_tutorials/js_setup/js_nodejs/js_nodejs.markdown | 2 +- doc/js_tutorials/js_setup/js_usage/js_usage.markdown | 4 +++- .../interactive_calibration.markdown | 2 +- doc/tutorials/dnn/dnn_googlenet/dnn_googlenet.markdown | 2 +- .../dnn_halide_scheduling.markdown | 2 +- .../erosion_dilatation/erosion_dilatation.markdown | 2 +- .../random_generator_and_text.markdown | 2 +- modules/dnn/misc/quantize_face_detector.py | 2 +- modules/dnn/test/imagenet_cls_test_inception.py | 2 +- modules/python/test/tests_common.py | 2 +- platforms/readme.txt | 2 +- samples/dnn/README.md | 6 +++--- samples/dnn/openpose.cpp | 4 ++-- 19 files changed, 30 insertions(+), 28 deletions(-) diff --git a/doc/js_tutorials/js_assets/js_image_classification.html b/doc/js_tutorials/js_assets/js_image_classification.html index 656f2720b6..343f486cd6 100644 --- a/doc/js_tutorials/js_assets/js_image_classification.html +++ b/doc/js_tutorials/js_assets/js_image_classification.html @@ -116,7 +116,7 @@ swapRB = false; needSoftmax = false; // url for label file, can from local or Internet -labelsUrl = "https://raw.githubusercontent.com/opencv/opencv/master/samples/data/dnn/classification_classes_ILSVRC2012.txt"; +labelsUrl = "https://raw.githubusercontent.com/opencv/opencv/3.4/samples/data/dnn/classification_classes_ILSVRC2012.txt";