From 024dfd54aff7459166e900962479d4643be746a8 Mon Sep 17 00:00:00 2001 From: Yuantao Feng Date: Wed, 15 Nov 2023 22:57:52 +0800 Subject: [PATCH] dnn cann backend: add hardswish, layernorm and instasnce norm for cann and bug fix (#24462) * add hardswish for cann * gemm cann bug fix * fix indentation * cann: add layer norm * cann: add instance norm * add supportBackend * cann: layer norm does not support axis=-1 due to 1d mat issue * disable instance norm for now * fix doc * remove tensor desc initialization for 1D tensor --- modules/dnn/src/layers/elementwise_layers.cpp | 25 +++++++- modules/dnn/src/layers/gemm_layer.cpp | 1 + .../dnn/src/layers/instance_norm_layer.cpp | 49 +++++++++++++++ modules/dnn/src/layers/layer_norm.cpp | 59 ++++++++++++++++++- 4 files changed, 132 insertions(+), 2 deletions(-) diff --git a/modules/dnn/src/layers/elementwise_layers.cpp b/modules/dnn/src/layers/elementwise_layers.cpp index 2a2245b909..746db69603 100644 --- a/modules/dnn/src/layers/elementwise_layers.cpp +++ b/modules/dnn/src/layers/elementwise_layers.cpp @@ -1890,7 +1890,9 @@ struct HardSwishFunctor : public BaseDefaultFunctor bool supportBackend(int backendId, int) { - return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA; + return backendId == DNN_BACKEND_OPENCV || + backendId == DNN_BACKEND_CUDA || + backendId == DNN_BACKEND_CANN; } inline float calculate(float x) const @@ -1905,6 +1907,27 @@ struct HardSwishFunctor : public BaseDefaultFunctor } #endif +#ifdef HAVE_CANN + Ptr initCannOp(const std::string& name, + const std::vector > &inputs, + const std::vector >& nodes) + { + auto x = inputs[0].dynamicCast(); + + auto op = std::make_shared(name); + + auto op_x = nodes[0].dynamicCast()->getOp(); + op->set_input_x_by_name(*op_x, x->name.c_str()); + auto x_desc = x->getTensorDesc(); + op->update_input_desc_x(*x_desc); + + auto output_desc = std::make_shared(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT); + op->update_output_desc_y(*output_desc); + + return Ptr(new CannBackendNode(op)); + } +#endif + int64 getFLOPSPerElement() const { return 1; } }; diff --git a/modules/dnn/src/layers/gemm_layer.cpp b/modules/dnn/src/layers/gemm_layer.cpp index a553f97568..8bcec78343 100644 --- a/modules/dnn/src/layers/gemm_layer.cpp +++ b/modules/dnn/src/layers/gemm_layer.cpp @@ -274,6 +274,7 @@ public: op->update_input_desc_bias(*(op_const_C->getTensorDesc())); // set outputs + auto output_desc = std::make_shared(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT); op->update_output_desc_y(*output_desc); return Ptr(new CannBackendNode(op)); } diff --git a/modules/dnn/src/layers/instance_norm_layer.cpp b/modules/dnn/src/layers/instance_norm_layer.cpp index fda0efdb94..b43e9bbb7a 100644 --- a/modules/dnn/src/layers/instance_norm_layer.cpp +++ b/modules/dnn/src/layers/instance_norm_layer.cpp @@ -6,6 +6,9 @@ #include #include "./cpu_kernels/fast_norm.hpp" +// CANN backend +#include "../op_cann.hpp" + // OpenVINO backend #include "../op_inf_engine.hpp" #include "../ie_ngraph.hpp" @@ -41,6 +44,7 @@ public: #endif return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA; + // backendId == DNN_BACKEND_CANN; // not supported due to 1d mat shape issue } bool getMemoryShapes(const std::vector &inputs, @@ -169,6 +173,51 @@ public: } #endif +#ifdef HAVE_CANN + virtual Ptr initCann(const std::vector > &inputs, + const std::vector > &outputs, + const std::vector >& nodes) CV_OVERRIDE { + auto input_tensor_wrapper = inputs[0].dynamicCast(); + auto input_tensor_desc = input_tensor_wrapper->getTensorDesc(); + + auto scale_tensor_wrapper = inputs[1].dynamicCast(); + auto scale_tensor_desc = scale_tensor_wrapper->getTensorDesc(); + + auto bias_tensor_wrapper = inputs[2].dynamicCast(); + auto bias_tensor_desc = bias_tensor_wrapper->getTensorDesc(); + + auto last_node = nodes[0].dynamicCast()->getOp(); + auto scale_node = nodes[1].dynamicCast()->getOp(); + auto bias_node = nodes[2].dynamicCast()->getOp(); + + auto op = std::make_shared(name); + + // set attrs + op->set_attr_epsilon(epsilon); + + // set inputs + // set inputs : x + op->set_input_x_by_name(*last_node, input_tensor_wrapper->name.c_str()); + op->update_input_desc_x(*input_tensor_desc); + // set inputs : gamma + op->set_input_gamma_by_name((*scale_node), scale_tensor_wrapper->name.c_str()); + op->update_input_desc_gamma(*scale_tensor_desc); + // set inputs : beta + op->set_input_beta_by_name(*bias_node, bias_tensor_wrapper->name.c_str()); + op->update_input_desc_beta(*bias_tensor_desc); + + // set outputs + auto output_desc_y = std::make_shared(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT); + op->update_output_desc_y(*output_desc_y); + auto output_desc_mean = std::make_shared(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT); + op->update_output_desc_mean(*output_desc_mean); + auto output_desc_var = std::make_shared(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT); + op->update_output_desc_variance(*output_desc_var); + + return Ptr(new CannBackendNode(op)); + } +#endif // HAVE_CANN + #ifdef HAVE_DNN_NGRAPH virtual Ptr initNgraph(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE { diff --git a/modules/dnn/src/layers/layer_norm.cpp b/modules/dnn/src/layers/layer_norm.cpp index 9c16d19e41..0683bdb8c8 100644 --- a/modules/dnn/src/layers/layer_norm.cpp +++ b/modules/dnn/src/layers/layer_norm.cpp @@ -6,6 +6,9 @@ #include "layers_common.hpp" #include "cpu_kernels/fast_norm.hpp" +// CANN backend +#include "../op_cann.hpp" + namespace cv { namespace dnn { class LayerNormLayerImpl CV_FINAL : public LayerNormLayer @@ -22,7 +25,8 @@ public: virtual bool supportBackend(int backendId) CV_OVERRIDE { - return backendId == DNN_BACKEND_OPENCV; + return backendId == DNN_BACKEND_OPENCV || + (backendId == DNN_BACKEND_CANN && axis != -1); // axis=-1 not supported due to 1d mat shape problem } virtual bool getMemoryShapes(const std::vector &inputs, @@ -90,6 +94,59 @@ public: fastNorm(input, scale, output, epsilon, static_cast(axis)); } } + +#ifdef HAVE_CANN + virtual Ptr initCann(const std::vector > &inputs, + const std::vector > &outputs, + const std::vector >& nodes) CV_OVERRIDE { + CV_CheckEQ(inputs.size(), static_cast(3), "LayerNorm/CANN: requires three input wrappers"); + CV_CheckEQ(nodes.size(), static_cast(3), "LayerNorm/CANN: requires three input nodes"); + + auto input_tensor_wrapper = inputs[0].dynamicCast(); + auto input_tensor_desc = input_tensor_wrapper->getTensorDesc(); + + CV_CheckNE(axis, static_cast(input_tensor_desc->GetShape().GetDimNum() - 1), "LayerNorm: CANN does not support axis set as last axis due to 1D mat compatibility issue"); + + auto scale_tensor_wrapper = inputs[1].dynamicCast(); + auto scale_tensor_desc = scale_tensor_wrapper->getTensorDesc(); + + auto bias_tensor_wrapper = inputs[2].dynamicCast(); + auto bias_tensor_desc = bias_tensor_wrapper->getTensorDesc(); + + auto last_node = nodes[0].dynamicCast()->getOp(); + auto scale_node = nodes[1].dynamicCast()->getOp(); + auto bias_node = nodes[2].dynamicCast()->getOp(); + + auto op = std::make_shared(name); + + // set attrs + op->set_attr_begin_norm_axis(axis); + op->set_attr_begin_params_axis(axis); + op->set_attr_epsilon(epsilon); + + // set inputs + // set inputs : x + op->set_input_x_by_name(*last_node, input_tensor_wrapper->name.c_str()); + op->update_input_desc_x(*input_tensor_desc); + // set inputs : gamma + op->set_input_gamma_by_name(*scale_node, scale_tensor_wrapper->name.c_str()); + op->update_input_desc_gamma(*scale_tensor_desc); + // set inputs : beta + op->set_input_beta_by_name(*bias_node, bias_tensor_wrapper->name.c_str()); + op->update_input_desc_beta(*bias_tensor_desc); + + // set outputs + auto output_desc_y = std::make_shared(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT); + op->update_output_desc_y(*output_desc_y); + auto output_desc_mean = std::make_shared(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT); + op->update_output_desc_mean(*output_desc_mean); + auto output_desc_var = std::make_shared(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT); + op->update_output_desc_variance(*output_desc_var); + + return Ptr(new CannBackendNode(op)); + } +#endif // HAVE_CANN + }; Ptr LayerNormLayer::create(const LayerParams& params)