1
0
mirror of https://github.com/opencv/opencv.git synced 2026-07-21 19:33:03 +04:00

Merge pull request #27700 from abhishek-gola:gridsample_layer_add

Added gridsample layer to new DNN engine #27700

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [x] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
Abhishek Gola
2025-09-09 17:15:13 +05:30
committed by GitHub
parent d1a5723864
commit f67ae273b3
7 changed files with 357 additions and 14 deletions
@@ -526,6 +526,12 @@ CV__DNN_INLINE_NS_BEGIN
static Ptr<IsInfLayer> create(const LayerParams& params);
};
class CV_EXPORTS GridSampleLayer : public Layer
{
public:
static Ptr<GridSampleLayer> create(const LayerParams& params);
};
class CV_EXPORTS FlattenLayer : public Layer
{
public:
+1
View File
@@ -113,6 +113,7 @@ void initializeLayerFactory()
CV_DNN_REGISTER_LAYER_CLASS(IsInf, IsInfLayer);
CV_DNN_REGISTER_LAYER_CLASS(Det, DetLayer);
CV_DNN_REGISTER_LAYER_CLASS(BitShift, BitShiftLayer);
CV_DNN_REGISTER_LAYER_CLASS(GridSample, GridSampleLayer);
CV_DNN_REGISTER_LAYER_CLASS(Convolution, ConvolutionLayer);
CV_DNN_REGISTER_LAYER_CLASS(Deconvolution, DeconvolutionLayer);
+328
View File
@@ -0,0 +1,328 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2025, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include "../precomp.hpp"
#include "layers_common.hpp"
#include <opencv2/dnn/shape_utils.hpp>
#include <opencv2/core/utility.hpp>
// ONNX operator: GridSample
// Spec: https://onnx.ai/onnx/operators/onnx__GridSample.html
// Supported opsets: 16-22
namespace cv {
namespace dnn {
static inline float reflectCoord(float x, int limit, bool align_corners) {
if (limit <= 1) return 0.f;
const float minv = align_corners ? 0.f : -0.5f;
const float maxv = align_corners ? float(limit - 1) : float(limit) - 0.5f;
const float m = maxv - minv;
const float two_m = 2.f * m;
float t = fmodf(x - minv, two_m);
if (t < 0) t += two_m;
if (t > m) t = two_m - t;
return t + minv;
}
static inline void getCubicCoeffs(float x, float A, float* coeffs )
{
coeffs[0] = ((A*(x + 1.0f) - 5.0f*A)*(x + 1.0f) + 8.0f*A)*(x + 1.0f) - 4.0f*A;
coeffs[1] = ((A + 2.0f)*x - (A + 3.0f))*x*x + 1.0f;
coeffs[2] = ((A + 2.0f)*(1.0f - x) - (A + 3.0f))*(1.0f - x)*(1.0f - x) + 1.0f;
coeffs[3] = 1.0f - coeffs[0] - coeffs[1] - coeffs[2];
}
static inline void computeNormToPixParams(int W, int H, bool align_corners,
float& xscale, float& yscale,
float& xdelta, float& ydelta)
{
const float delta = align_corners ? 1.f : 0.f;
xscale = 0.5f * (W - delta);
yscale = 0.5f * (H - delta);
xdelta = 0.5f * (W - delta) + 0.5f * (delta - 1.f);
ydelta = 0.5f * (H - delta) + 0.5f * (delta - 1.f);
}
static inline void normToPix(float nx, float ny,
float xscale, float yscale,
float xdelta, float ydelta,
float& x, float& y)
{
x = nx * xscale + xdelta;
y = ny * yscale + ydelta;
}
enum Mode { M_NEAREST=0, M_BILINEAR=1, M_BICUBIC=2 };
enum Pad { P_ZEROS=0, P_BORDER=1, P_REFLECTION=2 };
template<typename T, int MODE, int PAD>
static inline void gridSampleComputeRows(
const T* Xptr,
const float* Gptr,
T* Yptr,
int N, int C, int H, int W,
int Hout, int Wout,
bool align_corners)
{
size_t xNStride = (size_t)C * H * W;
size_t xCStride = (size_t)H * W;
size_t xHStride = (size_t)W;
size_t gNStride = (size_t)Hout * Wout * 2;
size_t gHStride = (size_t)Wout * 2;
size_t gWStride = 2;
size_t yNStride = (size_t)C * Hout * Wout;
size_t yCStride = (size_t)Hout * Wout;
size_t yHStride = (size_t)Wout;
auto fetch = [&](const T* baseNC, int yy, int xx)->float {
int px = xx, py = yy;
if (PAD == P_BORDER) {
px = saturate_cast<int>(std::min(float(W - 1), std::max(0.f, float(px))));
py = saturate_cast<int>(std::min(float(H - 1), std::max(0.f, float(py))));
} else if (PAD == P_REFLECTION) {
px = saturate_cast<int>(std::floor(reflectCoord((float)px, W, align_corners) + 0.5f));
py = saturate_cast<int>(std::floor(reflectCoord((float)py, H, align_corners) + 0.5f));
}
if (px < 0 || py < 0 || px >= W || py >= H) return 0.f;
const T* p = baseNC + (size_t)py * xHStride + (size_t)px;
return (float)(*p);
};
int nstripes = Hout * C * N;
parallel_for_(Range(0, nstripes), [&](const Range& range) {
for (int row = range.start; row < range.end; row++) {
int n = row / (Hout * C);
int c = (row - n * Hout * C) / Hout;
int h = row % Hout;
const T* baseNC = Xptr + n * xNStride + c * xCStride;
size_t yRowBase = n * yNStride + c * yCStride + h * yHStride;
size_t gRowBase = n * gNStride + h * gHStride;
float xscale, yscale, xdelta, ydelta;
computeNormToPixParams(W, H, align_corners, xscale, yscale, xdelta, ydelta);
for (int w = 0; w < Wout; w++) {
float nx = Gptr[gRowBase + w * gWStride + 0];
float ny = Gptr[gRowBase + w * gWStride + 1];
float xf, yf; normToPix(nx, ny, xscale, yscale, xdelta, ydelta, xf, yf);
float outv = 0.f;
if (MODE == M_NEAREST) {
int px = cvRound(xf);
int py = cvRound(yf);
outv = fetch(baseNC, py, px);
} else if (MODE == M_BILINEAR) {
int x0 = saturate_cast<int>(floorf(xf));
int y0 = saturate_cast<int>(floorf(yf));
float dx = xf - x0, dy = yf - y0;
if (x0 >= 0 && y0 >= 0 && x0 + 1 < W && y0 + 1 < H)
{
const T* p_y0 = baseNC + (size_t)y0 * xHStride;
const T* p_y1 = baseNC + (size_t)(y0 + 1) * xHStride;
float v00 = (float)p_y0[x0];
float v01 = (float)p_y0[x0 + 1];
float v10 = (float)p_y1[x0];
float v11 = (float)p_y1[x0 + 1];
float vx0 = v00 * (1.f - dx) + v01 * dx;
float vx1 = v10 * (1.f - dx) + v11 * dx;
outv = vx0 * (1.f - dy) + vx1 * dy;
}
else
{
float v00 = fetch(baseNC, y0, x0);
float v01 = fetch(baseNC, y0, x0 + 1);
float v10 = fetch(baseNC, y0 + 1, x0);
float v11 = fetch(baseNC, y0 + 1, x0 + 1);
float vx0 = v00 * (1.f - dx) + v01 * dx;
float vx1 = v10 * (1.f - dx) + v11 * dx;
outv = vx0 * (1.f - dy) + vx1 * dy;
}
} else {
const float alpha = -0.75f;
int x1 = saturate_cast<int>(floorf(xf));
int y1 = saturate_cast<int>(floorf(yf));
float tx = xf - x1, ty = yf - y1;
float wx[4], wy[4];
getCubicCoeffs(tx, alpha, wx);
getCubicCoeffs(ty, alpha, wy);
if (x1 >= 1 && y1 >= 1 && x1 + 2 < W && y1 + 2 < H) {
const T* p = baseNC + (size_t)(y1 - 1) * xHStride + (x1 - 1);
float rowv0 = (float)p[0] * wx[0] + (float)p[1] * wx[1] + (float)p[2] * wx[2] + (float)p[3] * wx[3];
p += xHStride;
float rowv1 = (float)p[0] * wx[0] + (float)p[1] * wx[1] + (float)p[2] * wx[2] + (float)p[3] * wx[3];
p += xHStride;
float rowv2 = (float)p[0] * wx[0] + (float)p[1] * wx[1] + (float)p[2] * wx[2] + (float)p[3] * wx[3];
p += xHStride;
float rowv3 = (float)p[0] * wx[0] + (float)p[1] * wx[1] + (float)p[2] * wx[2] + (float)p[3] * wx[3];
outv = rowv0 * wy[0] + rowv1 * wy[1] + rowv2 * wy[2] + rowv3 * wy[3];
} else {
float rowv0 = fetch(baseNC, y1 - 1, x1 - 1) * wx[0] + fetch(baseNC, y1 - 1, x1 ) * wx[1] +
fetch(baseNC, y1 - 1, x1 + 1) * wx[2] + fetch(baseNC, y1 - 1, x1 + 2) * wx[3];
float rowv1 = fetch(baseNC, y1, x1 - 1) * wx[0] + fetch(baseNC, y1, x1 ) * wx[1] +
fetch(baseNC, y1, x1 + 1) * wx[2] + fetch(baseNC, y1, x1 + 2) * wx[3];
float rowv2 = fetch(baseNC, y1 + 1, x1 - 1) * wx[0] + fetch(baseNC, y1 + 1, x1 ) * wx[1] +
fetch(baseNC, y1 + 1, x1 + 1) * wx[2] + fetch(baseNC, y1 + 1, x1 + 2) * wx[3];
float rowv3 = fetch(baseNC, y1 + 2, x1 - 1) * wx[0] + fetch(baseNC, y1 + 2, x1 ) * wx[1] +
fetch(baseNC, y1 + 2, x1 + 1) * wx[2] + fetch(baseNC, y1 + 2, x1 + 2) * wx[3];
outv = rowv0 * wy[0] + rowv1 * wy[1] + rowv2 * wy[2] + rowv3 * wy[3];
}
}
Yptr[yRowBase + w] = saturate_cast<T>(outv);
}
}
});
}
template<typename T>
static inline void gridSampleDispatch(
const T* Xptr,
const float* Gptr,
T* Yptr,
int N, int C, int H, int W,
int Hout, int Wout,
bool align_corners,
int mode, int padding)
{
if (mode == M_NEAREST) {
if (padding == P_ZEROS) gridSampleComputeRows<T, M_NEAREST, P_ZEROS>(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners);
else if (padding == P_BORDER) gridSampleComputeRows<T, M_NEAREST, P_BORDER>(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners);
else gridSampleComputeRows<T, M_NEAREST, P_REFLECTION>(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners);
} else if (mode == M_BILINEAR) {
if (padding == P_ZEROS) gridSampleComputeRows<T, M_BILINEAR, P_ZEROS>(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners);
else if (padding == P_BORDER) gridSampleComputeRows<T, M_BILINEAR, P_BORDER>(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners);
else gridSampleComputeRows<T, M_BILINEAR, P_REFLECTION>(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners);
} else {
if (padding == P_ZEROS) gridSampleComputeRows<T, M_BICUBIC, P_ZEROS>(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners);
else if (padding == P_BORDER) gridSampleComputeRows<T, M_BICUBIC, P_BORDER>(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners);
else gridSampleComputeRows<T, M_BICUBIC, P_REFLECTION>(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners);
}
}
class GridSampleLayerImpl CV_FINAL : public GridSampleLayer
{
public:
int mode = M_BILINEAR;
int padding = P_ZEROS;
bool align_corners = false;
GridSampleLayerImpl(const LayerParams& params) {
setParamsFrom(params);
String m = params.get<String>("mode", "bilinear");
String p = params.get<String>("padding_mode", "zeros");
align_corners = params.get<bool>("align_corners", false);
if (m == "nearest") mode = M_NEAREST;
else if (m == "bicubic") mode = M_BICUBIC;
else mode = M_BILINEAR;
if (p == "border") padding = P_BORDER;
else if (p == "reflection") padding = P_REFLECTION;
else padding = P_ZEROS;
}
bool supportBackend(int backendId) CV_OVERRIDE {
return backendId == DNN_BACKEND_OPENCV;
}
bool getMemoryShapes(const std::vector<MatShape>& inputs,
const int requiredOutputs,
std::vector<MatShape>& outputs,
std::vector<MatShape>& internals) const CV_OVERRIDE
{
CV_Assert(inputs.size() == 2);
const MatShape& x = inputs[0];
const MatShape& g = inputs[1];
CV_Assert(x.size() == 4);
CV_Assert(g.size() == 4 && g[3] == 2);
outputs.assign(1, MatShape({x[0], x[1], g[1], g[2]}));
return false;
}
void getTypes(const std::vector<MatType>& inTypes,
const int requiredOutputs,
const int requiredInternals,
std::vector<MatType>& outTypes,
std::vector<MatType>& internals) const CV_OVERRIDE
{
CV_Assert(!inTypes.empty());
outTypes.assign(requiredOutputs, inTypes[0]);
internals.assign(requiredInternals, inTypes[0]);
}
void forward(InputArrayOfArrays inputs_arr,
OutputArrayOfArrays outputs_arr,
OutputArrayOfArrays) CV_OVERRIDE
{
std::vector<Mat> inputs, outputs;
inputs_arr.getMatVector(inputs);
outputs_arr.getMatVector(outputs);
const Mat& X = inputs[0];
const Mat& G = inputs[1];
CV_Assert(X.dims == 4 && G.dims == 4 && G.size[3] == 2);
const int N = X.size[0], C = X.size[1], H = X.size[2], W = X.size[3];
const int Hout = G.size[1], Wout = G.size[2];
Mat Gf = G;
if (G.depth() != CV_32F) G.convertTo(Gf, CV_32F);
const float* Gptr = (const float*)Gf.data;
CV_Assert(!outputs.empty());
int outType = outputs[0].type();
Mat Ytmp({N, C, Hout, Wout}, outType);
Ytmp.setTo(Scalar(0));
switch (X.depth()) {
case CV_8U: {
const uchar* Xptr = X.ptr<uchar>();
uchar* Yptr = Ytmp.ptr<uchar>();
gridSampleDispatch<uchar>(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners, mode, padding);
break;
}
case CV_16F: {
const hfloat* Xptr = X.ptr<hfloat>();
hfloat* Yptr = Ytmp.ptr<hfloat>();
gridSampleDispatch<hfloat>(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners, mode, padding);
break;
}
#ifdef CV_16BF
case CV_16BF: {
const bfloat* Xptr = X.ptr<bfloat>();
bfloat* Yptr = Ytmp.ptr<bfloat>();
gridSampleDispatch<bfloat>(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners, mode, padding);
break;
}
#endif
case CV_32F: {
const float* Xptr = X.ptr<float>();
float* Yptr = Ytmp.ptr<float>();
gridSampleDispatch<float>(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners, mode, padding);
break;
}
default:
CV_Error(Error::StsNotImplemented, "Unsupported input depth for GridSample");
}
Ytmp.copyTo(outputs[0]);
}
};
Ptr<GridSampleLayer> GridSampleLayer::create(const LayerParams& params) {
return Ptr<GridSampleLayer>(new GridSampleLayerImpl(params));
}
}}
+8
View File
@@ -213,6 +213,7 @@ protected:
void parseIsNaN (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseIsInf (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseDet (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseGridSample (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseResize (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseSize (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseReshape (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
@@ -1652,6 +1653,12 @@ void ONNXImporter2::parseDet(LayerParams& layerParams, const opencv_onnx::NodePr
addLayer(layerParams, node_proto);
}
void ONNXImporter2::parseGridSample(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto)
{
layerParams.type = "GridSample";
addLayer(layerParams, node_proto);
}
void ONNXImporter2::parseUpsample(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto)
{
int n_inputs = node_proto.input_size();
@@ -2485,6 +2492,7 @@ void ONNXImporter2::buildDispatchMap_ONNX_AI(int opset_version)
dispatch["IsNaN"] = &ONNXImporter2::parseIsNaN;
dispatch["IsInf"] = &ONNXImporter2::parseIsInf;
dispatch["Det"] = &ONNXImporter2::parseDet;
dispatch["GridSample"] = &ONNXImporter2::parseGridSample;
dispatch["Upsample"] = &ONNXImporter2::parseUpsample;
dispatch["BitShift"] = &ONNXImporter2::parseBitShift;
dispatch["SoftMax"] = dispatch["Softmax"] = dispatch["LogSoftmax"] = &ONNXImporter2::parseSoftMax;
@@ -723,21 +723,21 @@ CASE(test_greater_equal_bcast_expanded)
CASE(test_greater_equal_expanded)
// no filter
CASE(test_gridsample)
// no filter
SKIP;
CASE(test_gridsample_aligncorners_true)
// no filter
SKIP;
CASE(test_gridsample_bicubic)
// no filter
CASE(test_gridsample_bilinear)
// no filter
SKIP;
CASE(test_gridsample_border_padding)
// no filter
SKIP;
CASE(test_gridsample_nearest)
// no filter
SKIP;
CASE(test_gridsample_reflection_padding)
// no filter
SKIP;
CASE(test_gridsample_zeros_padding)
// no filter
SKIP;
CASE(test_group_normalization_epsilon)
// no filter
CASE(test_group_normalization_example)
@@ -113,3 +113,10 @@
"test_bitshift_right_uint32",
"test_bitshift_right_uint64",
"test_bitshift_right_uint8",
"test_gridsample",
"test_gridsample_aligncorners_true",
"test_gridsample_bilinear",
"test_gridsample_border_padding",
"test_gridsample_reflection_padding",
"test_gridsample_zeros_padding",
"test_gridsample_nearest",
@@ -81,14 +81,7 @@
"test_gelu_default_2_expanded", // parser: no corresponding layer for CastLike
"test_gelu_tanh_1_expanded", // parser: no corresponding layer for CastLike
"test_gelu_tanh_2_expanded", // parser: no corresponding layer for CastLike
"test_gridsample", // Issues::Layer::Can't create layer "onnx_node_output_0!Y" of type "GridSample" in function 'getLayerInstance'
"test_gridsample_aligncorners_true", // ---- same as above ---
"test_gridsample_bicubic", // ---- same as above ---
"test_gridsample_bilinear", // ---- same as above ---
"test_gridsample_border_padding", // ---- same as above ---
"test_gridsample_nearest", // ---- same as above ---
"test_gridsample_reflection_padding", // ---- same as above ---
"test_gridsample_zeros_padding", // ---- same as above ---
"test_gru_batchwise", // Issues::Parser::node_proto.input_size() == 6 in function 'parseGRU'
"test_gru_defaults", // ---- same as above ---
"test_gru_seq_length", // ---- same as above ---