Merge pull request #24552 from fengyuentau:layernorm_backends

dnn: add openvino, opencl and cuda backends for layer normalization layer #24552 Merge after https://github.com/opencv/opencv/pull/24544. Todo: - [x] openvino - [x] opencl - [x] cuda ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake
2026-07-31 00:03:03 +04:00 · 2023-11-21 06:33:01 -06:00
parent fba3c947ef
commit d05fb709f9
6 changed files with 359 additions and 65 deletions
@@ -9,8 +9,26 @@
 // CANN backend
 #include "../op_cann.hpp"

+// OpenVINO backend
+#include "../op_inf_engine.hpp"
+#include "../ie_ngraph.hpp"
+
+// CUDA backend
+#include "../op_cuda.hpp"
+#ifdef HAVE_CUDA
+#include "../cuda4dnn/primitives/layer_norm.hpp"
+using namespace cv::dnn::cuda4dnn;
+#endif
+
+// OpenCL backend
+#ifdef HAVE_OPENCL
+#include "../ocl4dnn/include/math_functions.hpp"
+#include "opencl_kernels_dnn.hpp"
+#endif
+
 namespace cv { namespace dnn {

+// https://github.com/onnx/onnx/blob/main/docs/Operators.md#LayerNormalization
 class LayerNormLayerImpl CV_FINAL : public LayerNormLayer
 {
 public:
@@ -25,7 +43,12 @@ public:

    virtual bool supportBackend(int backendId) CV_OVERRIDE
    {
+#ifdef HAVE_INF_ENGINE
+        if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)
+            return true;
+#endif
        return backendId == DNN_BACKEND_OPENCV ||
+               backendId == DNN_BACKEND_CUDA   ||
               (backendId == DNN_BACKEND_CANN && axis != -1); // axis=-1 not supported due to 1d mat shape problem
    }

@@ -73,6 +96,9 @@ public:
        CV_TRACE_FUNCTION();
        CV_TRACE_ARG_VALUE(name, "name", name.c_str());

+        CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget),
+                   forward_ocl(inputs_arr, outputs_arr, internals_arr))
+
        if (inputs_arr.depth() == CV_16S)
        {
            forward_fallback(inputs_arr, outputs_arr, internals_arr);
@@ -95,6 +121,91 @@ public:
        }
    }

+#ifdef HAVE_OPENCL
+    bool forward_ocl(InputArrayOfArrays inputs_, OutputArrayOfArrays outputs_, OutputArrayOfArrays internals_) {
+        std::vector<UMat> inputs;
+        std::vector<UMat> outputs;
+
+        inputs_.getUMatVector(inputs);
+        outputs_.getUMatVector(outputs);
+
+        const auto &input = inputs[0], &scale = inputs[1]; // &bias = inputs[2]; // bias is optional
+        auto &output = outputs[0];
+
+        const auto input_shape = shape(input);
+        size_t loops = static_cast<size_t>(total(input_shape, 0, axis)),
+               norm_size = static_cast<size_t>(total(input_shape, axis));
+        float inv_norm_size = 1.f / norm_size;
+
+        const auto &bias = inputs.size() == 3 ? inputs[2] : UMat::zeros(norm_size, 1, CV_32F);
+
+        // no fp16 support
+        if (input.depth() == CV_16S) {
+            return false;
+        }
+
+        String base_opts = format(" -DT=float -DT4=float4 -Dconvert_T=convert_float4");
+
+        // Calculate mean
+        UMat one = UMat::ones(norm_size, 1, CV_32F);
+        UMat mean = UMat(loops, 1, CV_32F);
+        UMat mean_square = UMat(loops, 1, CV_32F);
+        UMat tmp = UMat(loops, norm_size, CV_32F);
+        bool ret = ocl4dnn::ocl4dnnGEMV<float>(ocl4dnn::CblasNoTrans, loops, norm_size, inv_norm_size,
+                                               input, 0, one, 0, 0.f, mean, 0);
+        if (!ret) {
+            return false;
+        }
+        // Calculate mean_square
+        int num_vector = (norm_size % 8 == 0) ? 8 : ((norm_size % 4 == 0) ? 4 : 1);
+        size_t global[] = {loops, static_cast<size_t>(norm_size / num_vector)};
+        String build_opt = format(" -DNUM=%d", num_vector) + base_opts;
+        String mean_square_kernel_name = format("calc_mean%d", num_vector);
+        ocl::Kernel mean_square_kernel(mean_square_kernel_name.c_str(), ocl::dnn::mvn_oclsrc, build_opt + " -DKERNEL_MEAN");
+        if (mean_square_kernel.empty()) {
+            return false;
+        }
+        mean_square_kernel.set(0, ocl::KernelArg::PtrReadOnly(input));
+        mean_square_kernel.set(1, (int)loops);
+        mean_square_kernel.set(2, (int)norm_size);
+        mean_square_kernel.set(3, ocl::KernelArg::PtrReadOnly(mean));
+        mean_square_kernel.set(4, ocl::KernelArg::PtrWriteOnly(tmp));
+        ret = mean_square_kernel.run(2, global, NULL, false);
+        if (!ret) {
+            return false;
+        }
+        ret = ocl4dnn::ocl4dnnGEMV<float>(ocl4dnn::CblasNoTrans, loops, norm_size, inv_norm_size,
+                                          tmp, 0, one, 0, 0.f, mean_square, 0);
+        if (!ret) {
+            return false;
+        }
+        // Calculate instance norm: output = scale * (x - mean) / sqrt(var + eps) + bias
+        String mvn_kernel_name = format("mvn%d", num_vector);
+        build_opt += " -DNORM_VARIANCE -DLAYER_NORM -DKERNEL_MVN";
+        ocl::Kernel mvn_kernel(mvn_kernel_name.c_str(), ocl::dnn::mvn_oclsrc, build_opt);
+        if (mvn_kernel.empty()) {
+            return false;
+        }
+        mvn_kernel.set(0, ocl::KernelArg::PtrReadOnly(input));
+        mvn_kernel.set(1, (int)loops);
+        mvn_kernel.set(2, (int)norm_size);
+        mvn_kernel.set(3, (float)epsilon);
+        mvn_kernel.set(4, ocl::KernelArg::PtrReadOnly(mean));
+        mvn_kernel.set(5, ocl::KernelArg::PtrReadOnly(mean_square));
+        mvn_kernel.set(6, ocl::KernelArg::PtrReadOnly(scale));
+        mvn_kernel.set(7, ocl::KernelArg::PtrReadOnly(bias));
+        mvn_kernel.set(8, (int)1);
+        mvn_kernel.set(9, (float)0.f);
+        mvn_kernel.set(10, ocl::KernelArg::PtrWriteOnly(output));
+        ret = mvn_kernel.run(2, global, NULL, false);
+        if (!ret) {
+            return false;
+        }
+
+        return true;
+    }
+#endif
+
 #ifdef HAVE_CANN
    virtual Ptr<BackendNode> initCann(const std::vector<Ptr<BackendWrapper> > &inputs,
                                      const std::vector<Ptr<BackendWrapper> > &outputs,
@@ -147,6 +258,67 @@ public:
    }
 #endif // HAVE_CANN

+#ifdef HAVE_DNN_NGRAPH
+    virtual Ptr<BackendNode> initNgraph(const std::vector<Ptr<BackendWrapper> >& inputs,
+                                        const std::vector<Ptr<BackendNode> >& nodes) CV_OVERRIDE {
+        auto ieInpNode = nodes[0].dynamicCast<InfEngineNgraphNode>()->node;
+        const auto &input_shape = ieInpNode.get_shape();
+        std::shared_ptr<ngraph::Node> mvn, result;
+
+        // mvn
+#if INF_ENGINE_VER_MAJOR_LE(INF_ENGINE_RELEASE_2021_2)
+        // https://docs.openvino.ai/2021.4/api/ngraph_python_api/_autosummary/ngraph.opset3.mvn.html?highlight=mvn#ngraph.opset3.mvn
+        bool across_channels = false;
+        bool normalize_variance = true;
+        mvn = std::make_shared<ngraph::op::MVN>(ieInpNode, across_channels, normalize_variance, epsilon);
+#else
+        // https://docs.openvino.ai/2023.1/openvino_docs_ops_normalization_MVN_6.html
+        std::vector<int64_t> axes_v(input_shape.size() - axis);
+        std::iota(axes_v.begin(), axes_v.end(), axis);
+        auto axes = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{axes_v.size()}, axes_v.data());
+        bool normalize_variance = true;
+        mvn = std::make_shared<ngraph::op::v6::MVN>(ieInpNode, axes, normalize_variance, epsilon, ngraph::op::MVNEpsMode::INSIDE_SQRT);
+#endif
+
+        // layer norm = scale * mvn + bias
+        auto scale = nodes[1].dynamicCast<InfEngineNgraphNode>()->node;
+        ngraph::Output<ngraph::Node> bias;
+        if (nodes.size() == 3) {
+            bias = nodes[2].dynamicCast<InfEngineNgraphNode>()->node;
+        }
+        if (axis == -1 || axis == input_shape.size() - 1) { // special case for 1D tensor (2D mat)
+            std::vector<int64_t> shared_shape_v(input_shape.size(), 1);
+            shared_shape_v.back() = -1;
+            auto shared_shape = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{shared_shape_v.size()}, shared_shape_v.data());
+            scale  = std::make_shared<ngraph::op::v1::Reshape>(scale, shared_shape, true);
+            if (nodes.size() == 3) {
+                bias  = std::make_shared<ngraph::op::v1::Reshape>(bias, shared_shape, true);
+            }
+        }
+
+        result = std::make_shared<ngraph::op::v1::Multiply>(mvn, scale);
+        if (nodes.size() == 3) {
+            result = std::make_shared<ngraph::op::v1::Add>(result, bias);
+        }
+
+        return Ptr<BackendNode>(new InfEngineNgraphNode(result));
+    }
+#endif // HAVE_DNN_NGRAPH
+
+#ifdef HAVE_CUDA
+    Ptr<BackendNode> initCUDA(void *context_,
+                              const std::vector<Ptr<BackendWrapper>>& inputs,
+                              const std::vector<Ptr<BackendWrapper>>& outputs) override {
+        auto context = reinterpret_cast<csl::CSLContext*>(context_);
+
+        auto input_wrapper = inputs[0].dynamicCast<CUDABackendWrapper>();
+        auto input_shape = input_wrapper->getShape();
+        size_t loops = static_cast<size_t>(total(input_shape, 0, axis));
+
+        return make_cuda_node<cuda4dnn::LayerNormOp>(preferableTarget, std::move(context->stream), axis, epsilon, loops);
+    }
+#endif // HAVE_CUDA
+
 };

 Ptr<LayerNormLayer> LayerNormLayer::create(const LayerParams& params)