From f67ae273b3ad6e7c655f09706b0eed9c107acc5b Mon Sep 17 00:00:00 2001 From: Abhishek Gola Date: Tue, 9 Sep 2025 17:15:13 +0530 Subject: [PATCH] Merge pull request #27700 from abhishek-gola:gridsample_layer_add Added gridsample layer to new DNN engine #27700 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- .../dnn/include/opencv2/dnn/all_layers.hpp | 6 + modules/dnn/src/init.cpp | 1 + modules/dnn/src/layers/gridsample_layer.cpp | 328 ++++++++++++++++++ modules/dnn/src/onnx/onnx_importer2.cpp | 8 + ...conformance_layer_filter__openvino.inl.hpp | 14 +- ...yer_filter_opencv_classic_denylist.inl.hpp | 7 + ..._conformance_layer_parser_denylist.inl.hpp | 7 - 7 files changed, 357 insertions(+), 14 deletions(-) create mode 100644 modules/dnn/src/layers/gridsample_layer.cpp diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index 95b6ea2e69..c9282dafba 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -526,6 +526,12 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + class CV_EXPORTS GridSampleLayer : public Layer + { + public: + static Ptr create(const LayerParams& params); + }; + class CV_EXPORTS FlattenLayer : public Layer { public: diff --git a/modules/dnn/src/init.cpp b/modules/dnn/src/init.cpp index 7861ed4a24..613ab2041c 100644 --- a/modules/dnn/src/init.cpp +++ b/modules/dnn/src/init.cpp @@ -113,6 +113,7 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(IsInf, IsInfLayer); CV_DNN_REGISTER_LAYER_CLASS(Det, DetLayer); CV_DNN_REGISTER_LAYER_CLASS(BitShift, BitShiftLayer); + CV_DNN_REGISTER_LAYER_CLASS(GridSample, GridSampleLayer); CV_DNN_REGISTER_LAYER_CLASS(Convolution, ConvolutionLayer); CV_DNN_REGISTER_LAYER_CLASS(Deconvolution, DeconvolutionLayer); diff --git a/modules/dnn/src/layers/gridsample_layer.cpp b/modules/dnn/src/layers/gridsample_layer.cpp new file mode 100644 index 0000000000..cca5c037da --- /dev/null +++ b/modules/dnn/src/layers/gridsample_layer.cpp @@ -0,0 +1,328 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// Copyright (C) 2025, BigVision LLC, all rights reserved. +// Third party copyrights are property of their respective owners. + +#include "../precomp.hpp" +#include "layers_common.hpp" +#include +#include + +// ONNX operator: GridSample +// Spec: https://onnx.ai/onnx/operators/onnx__GridSample.html +// Supported opsets: 16-22 + +namespace cv { +namespace dnn { + +static inline float reflectCoord(float x, int limit, bool align_corners) { + if (limit <= 1) return 0.f; + + const float minv = align_corners ? 0.f : -0.5f; + const float maxv = align_corners ? float(limit - 1) : float(limit) - 0.5f; + const float m = maxv - minv; + const float two_m = 2.f * m; + + float t = fmodf(x - minv, two_m); + if (t < 0) t += two_m; + if (t > m) t = two_m - t; + return t + minv; +} + +static inline void getCubicCoeffs(float x, float A, float* coeffs ) +{ + coeffs[0] = ((A*(x + 1.0f) - 5.0f*A)*(x + 1.0f) + 8.0f*A)*(x + 1.0f) - 4.0f*A; + coeffs[1] = ((A + 2.0f)*x - (A + 3.0f))*x*x + 1.0f; + coeffs[2] = ((A + 2.0f)*(1.0f - x) - (A + 3.0f))*(1.0f - x)*(1.0f - x) + 1.0f; + coeffs[3] = 1.0f - coeffs[0] - coeffs[1] - coeffs[2]; +} + +static inline void computeNormToPixParams(int W, int H, bool align_corners, + float& xscale, float& yscale, + float& xdelta, float& ydelta) +{ + const float delta = align_corners ? 1.f : 0.f; + xscale = 0.5f * (W - delta); + yscale = 0.5f * (H - delta); + xdelta = 0.5f * (W - delta) + 0.5f * (delta - 1.f); + ydelta = 0.5f * (H - delta) + 0.5f * (delta - 1.f); +} + +static inline void normToPix(float nx, float ny, + float xscale, float yscale, + float xdelta, float ydelta, + float& x, float& y) +{ + x = nx * xscale + xdelta; + y = ny * yscale + ydelta; +} + +enum Mode { M_NEAREST=0, M_BILINEAR=1, M_BICUBIC=2 }; +enum Pad { P_ZEROS=0, P_BORDER=1, P_REFLECTION=2 }; + +template +static inline void gridSampleComputeRows( + const T* Xptr, + const float* Gptr, + T* Yptr, + int N, int C, int H, int W, + int Hout, int Wout, + bool align_corners) +{ + size_t xNStride = (size_t)C * H * W; + size_t xCStride = (size_t)H * W; + size_t xHStride = (size_t)W; + + size_t gNStride = (size_t)Hout * Wout * 2; + size_t gHStride = (size_t)Wout * 2; + size_t gWStride = 2; + + size_t yNStride = (size_t)C * Hout * Wout; + size_t yCStride = (size_t)Hout * Wout; + size_t yHStride = (size_t)Wout; + + auto fetch = [&](const T* baseNC, int yy, int xx)->float { + int px = xx, py = yy; + if (PAD == P_BORDER) { + px = saturate_cast(std::min(float(W - 1), std::max(0.f, float(px)))); + py = saturate_cast(std::min(float(H - 1), std::max(0.f, float(py)))); + } else if (PAD == P_REFLECTION) { + px = saturate_cast(std::floor(reflectCoord((float)px, W, align_corners) + 0.5f)); + py = saturate_cast(std::floor(reflectCoord((float)py, H, align_corners) + 0.5f)); + } + if (px < 0 || py < 0 || px >= W || py >= H) return 0.f; + const T* p = baseNC + (size_t)py * xHStride + (size_t)px; + return (float)(*p); + }; + + int nstripes = Hout * C * N; + parallel_for_(Range(0, nstripes), [&](const Range& range) { + for (int row = range.start; row < range.end; row++) { + int n = row / (Hout * C); + int c = (row - n * Hout * C) / Hout; + int h = row % Hout; + + const T* baseNC = Xptr + n * xNStride + c * xCStride; + size_t yRowBase = n * yNStride + c * yCStride + h * yHStride; + size_t gRowBase = n * gNStride + h * gHStride; + + float xscale, yscale, xdelta, ydelta; + computeNormToPixParams(W, H, align_corners, xscale, yscale, xdelta, ydelta); + for (int w = 0; w < Wout; w++) { + float nx = Gptr[gRowBase + w * gWStride + 0]; + float ny = Gptr[gRowBase + w * gWStride + 1]; + float xf, yf; normToPix(nx, ny, xscale, yscale, xdelta, ydelta, xf, yf); + + float outv = 0.f; + if (MODE == M_NEAREST) { + int px = cvRound(xf); + int py = cvRound(yf); + outv = fetch(baseNC, py, px); + } else if (MODE == M_BILINEAR) { + int x0 = saturate_cast(floorf(xf)); + int y0 = saturate_cast(floorf(yf)); + float dx = xf - x0, dy = yf - y0; + + if (x0 >= 0 && y0 >= 0 && x0 + 1 < W && y0 + 1 < H) + { + const T* p_y0 = baseNC + (size_t)y0 * xHStride; + const T* p_y1 = baseNC + (size_t)(y0 + 1) * xHStride; + float v00 = (float)p_y0[x0]; + float v01 = (float)p_y0[x0 + 1]; + float v10 = (float)p_y1[x0]; + float v11 = (float)p_y1[x0 + 1]; + float vx0 = v00 * (1.f - dx) + v01 * dx; + float vx1 = v10 * (1.f - dx) + v11 * dx; + outv = vx0 * (1.f - dy) + vx1 * dy; + } + else + { + float v00 = fetch(baseNC, y0, x0); + float v01 = fetch(baseNC, y0, x0 + 1); + float v10 = fetch(baseNC, y0 + 1, x0); + float v11 = fetch(baseNC, y0 + 1, x0 + 1); + float vx0 = v00 * (1.f - dx) + v01 * dx; + float vx1 = v10 * (1.f - dx) + v11 * dx; + outv = vx0 * (1.f - dy) + vx1 * dy; + } + } else { + const float alpha = -0.75f; + int x1 = saturate_cast(floorf(xf)); + int y1 = saturate_cast(floorf(yf)); + float tx = xf - x1, ty = yf - y1; + + float wx[4], wy[4]; + getCubicCoeffs(tx, alpha, wx); + getCubicCoeffs(ty, alpha, wy); + + if (x1 >= 1 && y1 >= 1 && x1 + 2 < W && y1 + 2 < H) { + const T* p = baseNC + (size_t)(y1 - 1) * xHStride + (x1 - 1); + float rowv0 = (float)p[0] * wx[0] + (float)p[1] * wx[1] + (float)p[2] * wx[2] + (float)p[3] * wx[3]; + p += xHStride; + float rowv1 = (float)p[0] * wx[0] + (float)p[1] * wx[1] + (float)p[2] * wx[2] + (float)p[3] * wx[3]; + p += xHStride; + float rowv2 = (float)p[0] * wx[0] + (float)p[1] * wx[1] + (float)p[2] * wx[2] + (float)p[3] * wx[3]; + p += xHStride; + float rowv3 = (float)p[0] * wx[0] + (float)p[1] * wx[1] + (float)p[2] * wx[2] + (float)p[3] * wx[3]; + outv = rowv0 * wy[0] + rowv1 * wy[1] + rowv2 * wy[2] + rowv3 * wy[3]; + } else { + float rowv0 = fetch(baseNC, y1 - 1, x1 - 1) * wx[0] + fetch(baseNC, y1 - 1, x1 ) * wx[1] + + fetch(baseNC, y1 - 1, x1 + 1) * wx[2] + fetch(baseNC, y1 - 1, x1 + 2) * wx[3]; + float rowv1 = fetch(baseNC, y1, x1 - 1) * wx[0] + fetch(baseNC, y1, x1 ) * wx[1] + + fetch(baseNC, y1, x1 + 1) * wx[2] + fetch(baseNC, y1, x1 + 2) * wx[3]; + float rowv2 = fetch(baseNC, y1 + 1, x1 - 1) * wx[0] + fetch(baseNC, y1 + 1, x1 ) * wx[1] + + fetch(baseNC, y1 + 1, x1 + 1) * wx[2] + fetch(baseNC, y1 + 1, x1 + 2) * wx[3]; + float rowv3 = fetch(baseNC, y1 + 2, x1 - 1) * wx[0] + fetch(baseNC, y1 + 2, x1 ) * wx[1] + + fetch(baseNC, y1 + 2, x1 + 1) * wx[2] + fetch(baseNC, y1 + 2, x1 + 2) * wx[3]; + outv = rowv0 * wy[0] + rowv1 * wy[1] + rowv2 * wy[2] + rowv3 * wy[3]; + } + } + Yptr[yRowBase + w] = saturate_cast(outv); + } + } + }); +} + +template +static inline void gridSampleDispatch( + const T* Xptr, + const float* Gptr, + T* Yptr, + int N, int C, int H, int W, + int Hout, int Wout, + bool align_corners, + int mode, int padding) +{ + if (mode == M_NEAREST) { + if (padding == P_ZEROS) gridSampleComputeRows(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners); + else if (padding == P_BORDER) gridSampleComputeRows(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners); + else gridSampleComputeRows(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners); + } else if (mode == M_BILINEAR) { + if (padding == P_ZEROS) gridSampleComputeRows(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners); + else if (padding == P_BORDER) gridSampleComputeRows(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners); + else gridSampleComputeRows(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners); + } else { + if (padding == P_ZEROS) gridSampleComputeRows(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners); + else if (padding == P_BORDER) gridSampleComputeRows(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners); + else gridSampleComputeRows(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners); + } +} + +class GridSampleLayerImpl CV_FINAL : public GridSampleLayer +{ +public: + int mode = M_BILINEAR; + int padding = P_ZEROS; + bool align_corners = false; + + GridSampleLayerImpl(const LayerParams& params) { + setParamsFrom(params); + String m = params.get("mode", "bilinear"); + String p = params.get("padding_mode", "zeros"); + align_corners = params.get("align_corners", false); + + if (m == "nearest") mode = M_NEAREST; + else if (m == "bicubic") mode = M_BICUBIC; + else mode = M_BILINEAR; + + if (p == "border") padding = P_BORDER; + else if (p == "reflection") padding = P_REFLECTION; + else padding = P_ZEROS; + } + + bool supportBackend(int backendId) CV_OVERRIDE { + return backendId == DNN_BACKEND_OPENCV; + } + + bool getMemoryShapes(const std::vector& inputs, + const int requiredOutputs, + std::vector& outputs, + std::vector& internals) const CV_OVERRIDE + { + CV_Assert(inputs.size() == 2); + const MatShape& x = inputs[0]; + const MatShape& g = inputs[1]; + CV_Assert(x.size() == 4); + CV_Assert(g.size() == 4 && g[3] == 2); + outputs.assign(1, MatShape({x[0], x[1], g[1], g[2]})); + return false; + } + + void getTypes(const std::vector& inTypes, + const int requiredOutputs, + const int requiredInternals, + std::vector& outTypes, + std::vector& internals) const CV_OVERRIDE + { + CV_Assert(!inTypes.empty()); + outTypes.assign(requiredOutputs, inTypes[0]); + internals.assign(requiredInternals, inTypes[0]); + } + + void forward(InputArrayOfArrays inputs_arr, + OutputArrayOfArrays outputs_arr, + OutputArrayOfArrays) CV_OVERRIDE + { + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + const Mat& X = inputs[0]; + const Mat& G = inputs[1]; + + CV_Assert(X.dims == 4 && G.dims == 4 && G.size[3] == 2); + const int N = X.size[0], C = X.size[1], H = X.size[2], W = X.size[3]; + const int Hout = G.size[1], Wout = G.size[2]; + + Mat Gf = G; + if (G.depth() != CV_32F) G.convertTo(Gf, CV_32F); + + const float* Gptr = (const float*)Gf.data; + + CV_Assert(!outputs.empty()); + int outType = outputs[0].type(); + + Mat Ytmp({N, C, Hout, Wout}, outType); + Ytmp.setTo(Scalar(0)); + + switch (X.depth()) { + case CV_8U: { + const uchar* Xptr = X.ptr(); + uchar* Yptr = Ytmp.ptr(); + gridSampleDispatch(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners, mode, padding); + break; + } + case CV_16F: { + const hfloat* Xptr = X.ptr(); + hfloat* Yptr = Ytmp.ptr(); + gridSampleDispatch(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners, mode, padding); + break; + } +#ifdef CV_16BF + case CV_16BF: { + const bfloat* Xptr = X.ptr(); + bfloat* Yptr = Ytmp.ptr(); + gridSampleDispatch(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners, mode, padding); + break; + } +#endif + case CV_32F: { + const float* Xptr = X.ptr(); + float* Yptr = Ytmp.ptr(); + gridSampleDispatch(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners, mode, padding); + break; + } + default: + CV_Error(Error::StsNotImplemented, "Unsupported input depth for GridSample"); + } + + Ytmp.copyTo(outputs[0]); + } +}; + +Ptr GridSampleLayer::create(const LayerParams& params) { + return Ptr(new GridSampleLayerImpl(params)); +} + +}} diff --git a/modules/dnn/src/onnx/onnx_importer2.cpp b/modules/dnn/src/onnx/onnx_importer2.cpp index 92424a6e96..aeaabfa770 100644 --- a/modules/dnn/src/onnx/onnx_importer2.cpp +++ b/modules/dnn/src/onnx/onnx_importer2.cpp @@ -213,6 +213,7 @@ protected: void parseIsNaN (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseIsInf (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseDet (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseGridSample (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseResize (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseSize (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseReshape (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); @@ -1652,6 +1653,12 @@ void ONNXImporter2::parseDet(LayerParams& layerParams, const opencv_onnx::NodePr addLayer(layerParams, node_proto); } +void ONNXImporter2::parseGridSample(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "GridSample"; + addLayer(layerParams, node_proto); +} + void ONNXImporter2::parseUpsample(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { int n_inputs = node_proto.input_size(); @@ -2485,6 +2492,7 @@ void ONNXImporter2::buildDispatchMap_ONNX_AI(int opset_version) dispatch["IsNaN"] = &ONNXImporter2::parseIsNaN; dispatch["IsInf"] = &ONNXImporter2::parseIsInf; dispatch["Det"] = &ONNXImporter2::parseDet; + dispatch["GridSample"] = &ONNXImporter2::parseGridSample; dispatch["Upsample"] = &ONNXImporter2::parseUpsample; dispatch["BitShift"] = &ONNXImporter2::parseBitShift; dispatch["SoftMax"] = dispatch["Softmax"] = dispatch["LogSoftmax"] = &ONNXImporter2::parseSoftMax; diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp index 9fe52963a4..c220d7321f 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp @@ -723,21 +723,21 @@ CASE(test_greater_equal_bcast_expanded) CASE(test_greater_equal_expanded) // no filter CASE(test_gridsample) - // no filter + SKIP; CASE(test_gridsample_aligncorners_true) - // no filter + SKIP; CASE(test_gridsample_bicubic) // no filter CASE(test_gridsample_bilinear) - // no filter + SKIP; CASE(test_gridsample_border_padding) - // no filter + SKIP; CASE(test_gridsample_nearest) - // no filter + SKIP; CASE(test_gridsample_reflection_padding) - // no filter + SKIP; CASE(test_gridsample_zeros_padding) - // no filter + SKIP; CASE(test_group_normalization_epsilon) // no filter CASE(test_group_normalization_example) diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_classic_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_classic_denylist.inl.hpp index 0f18302fe5..263b3169ee 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_classic_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_classic_denylist.inl.hpp @@ -113,3 +113,10 @@ "test_bitshift_right_uint32", "test_bitshift_right_uint64", "test_bitshift_right_uint8", +"test_gridsample", +"test_gridsample_aligncorners_true", +"test_gridsample_bilinear", +"test_gridsample_border_padding", +"test_gridsample_reflection_padding", +"test_gridsample_zeros_padding", +"test_gridsample_nearest", diff --git a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp index 89112eff78..d79d5e32cf 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp @@ -81,14 +81,7 @@ "test_gelu_default_2_expanded", // parser: no corresponding layer for CastLike "test_gelu_tanh_1_expanded", // parser: no corresponding layer for CastLike "test_gelu_tanh_2_expanded", // parser: no corresponding layer for CastLike -"test_gridsample", // Issues::Layer::Can't create layer "onnx_node_output_0!Y" of type "GridSample" in function 'getLayerInstance' -"test_gridsample_aligncorners_true", // ---- same as above --- "test_gridsample_bicubic", // ---- same as above --- -"test_gridsample_bilinear", // ---- same as above --- -"test_gridsample_border_padding", // ---- same as above --- -"test_gridsample_nearest", // ---- same as above --- -"test_gridsample_reflection_padding", // ---- same as above --- -"test_gridsample_zeros_padding", // ---- same as above --- "test_gru_batchwise", // Issues::Parser::node_proto.input_size() == 6 in function 'parseGRU' "test_gru_defaults", // ---- same as above --- "test_gru_seq_length", // ---- same as above ---