Merge pull request #27757 from vpisarev:matshape_inside_mat

Use MatShape instead of MatSize inside cv::Mat/cv::UMat #27757

**Merge together with https://github.com/opencv/opencv_contrib/pull/3996**
---

This PR continues cv::Mat/cv::UMat refactoring. See #26056, where `MatShape` was introduced. Now it's put inside cv::Mat/cv::UMat instead of a weird `MatSize`. MatSize is now an alias for MatShape:

**before:**

```
struct MatShape { ... };
struct MatSize { ... };

struct Mat {
    ...
    int dims;
    int rows;
    int cols;

    ...

    MatShape shape() const { ... /* constructs MatShape out of MatSize and returns it;
                                    layout is always 'unknown', because we don't store it */ }

    MatSize size; // size is not valid without the parent cv::Mat,
                  // because size.p may point to Mat::rows or to Mat::cols,
                  // depending on the dimensionality, and dims() returns Mat::dims.
    MatStep step; // may allocate memory, depending on the dimensionality.
    ...
};
```

**after:**

```
struct MatShape { ... };
typedef MatShape MatSize; // they are now synonyms

struct Mat {
    ...
    int dims;
    int rows;
    int cols;

    ...

    MatShape shape() const { return size; } // just return the embedded shape (including the proper layout information)

    MatSize size; // size is self-contained data structure that can be used without the parent cv::Mat.
                  // size.dims is now a copy of dims; size.p[*] contains copies of Mat::rows and Mat::cols when dims <= 2.
    MatStep step; // does not allocate extra memory buffers.
    ...
};
```

There are several reasons to do that:

1. the main reason is to be able to store data layout (MatShape::layout) inside each cv::Mat/cv::UMat. This is necessary for the proper shape inference in DNN module. In particular, it's necessary for the next step of DNN inference optimization where we introduce block-layout-optimized convolution and other operations. Later on, we can use layout information to support non-interleaved images (e.g. RRR...GGG...BBB...) or even batches of such images in core/imgproc modules.
2. the other reason is to represent 3D/4D/5D etc. tensors as cv::Mat/cv::UMat instances more conveniently, without extra dynamic memory allocation. Before this patch we allocated some memory buffers dynamically to store shape & steps for more than 2D arrays. Now the whole cv::Mat/cv::UMat header can be stored completely on stack/in a container. Creating another copy of Mat/UMat header is now done more efficiently.
3. the third reason is to introduce the new coding pattern: `dst.create(src.size, <dst_type>);`. The pattern is suitable for most of element-wise (including cloning) and filtering operations. This pattern does not only look crisp and self-documenting, it will also automatically copy shape (including layout) from the source tensor into the destination matrix/tensor.
4. in the future we might add `colorspace` member to MatShape that will allow to distinguish RGB from BGR or NV12. `dst.create(src.size, <dst_type>);` will then copy the colorspace information as well.

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [x] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake

modules/core/include/opencv2/core/cuda.inl.hpp

@@ -751,7 +751,7 @@ namespace cv {
 
 inline
 Mat::Mat(const cuda::GpuMat& m)
-    : flags(0), dims(0), rows(0), cols(0), data(0), datastart(0), dataend(0), datalimit(0), allocator(0), u(0), size(&rows)
+    : flags(0), dims(0), rows(0), cols(0), data(0), datastart(0), dataend(0), datalimit(0), allocator(0), u(0)
 {
     m.download(*this);
 }

modules/core/include/opencv2/core/cvstd.inl.hpp

@@ -169,16 +169,22 @@ std::ostream& operator << (std::ostream& out, const Rect_<_Tp>& rect)
     return out << "[" << rect.width << " x " << rect.height << " from (" << rect.x << ", " << rect.y << ")]";
 }
 
-static inline std::ostream& operator << (std::ostream& out, const MatSize& msize)
+static inline std::ostream& operator << (std::ostream& strm, const MatShape& shape)
 {
-    int i, dims = msize.dims();
-    for( i = 0; i < dims; i++ )
-    {
-        out << msize[i];
-        if( i < dims-1 )
-            out << " x ";
+    strm << '[';
+    if (shape.empty()) {
+        strm << "<empty>";
+    } else {
+        size_t n = shape.size();
+        if (n == 0) {
+            strm << "<scalar>";
+        } else {
+            for(size_t i = 0; i < n; ++i)
+                strm << (i > 0 ? " x " : "") << shape[i];
+        }
     }
-    return out;
+    strm << "]";
+    return strm;
 }
 
 static inline std::ostream &operator<< (std::ostream &s, cv::Range &r)

modules/core/include/opencv2/core/mat.hpp

@@ -148,8 +148,9 @@ struct CV_EXPORTS_W_SIMPLE MatShape
     const int& back() const;
     void push_back(int value);
     void emplace_back(int value);
-    int& operator [](size_t idx);
     const int& operator [](size_t idx) const;
+    int& operator [](size_t idx);
+    Size operator()() const; // for compatibility with MatSize
 
     CV_WRAP bool hasSymbols() const; // negative elements in the shape may denote 'symbols' instead of actual values.
 
@@ -162,7 +163,7 @@ struct CV_EXPORTS_W_SIMPLE MatShape
 
     size_t total() const; // returns the total number of elements in the tensor (including padding elements, i.e. the method ignores 'C' in the case of block layout). Returns 1 for scalar tensors. Returns 0 for empty shapes.
 
-    operator std::vector<int>() const;
+    std::vector<int> vec() const;
     std::string str() const;
 
     int dims;
@@ -723,20 +724,7 @@ struct CV_EXPORTS UMatData
 };
 CV_ENUM_FLAGS(UMatData::MemoryFlag)
 
-
-struct CV_EXPORTS MatSize
-{
-    explicit MatSize(int* _p) CV_NOEXCEPT;
-    int dims() const CV_NOEXCEPT;
-    Size operator()() const;
-    const int& operator[](int i) const;
-    int& operator[](int i);
-    operator const int*() const CV_NOEXCEPT;  // TODO OpenCV 4.0: drop this
-    bool operator == (const MatSize& sz) const CV_NOEXCEPT;
-    bool operator != (const MatSize& sz) const CV_NOEXCEPT;
-
-    int* p;
-};
+typedef MatShape MatSize;
 
 struct CV_EXPORTS MatStep
 {
@@ -746,11 +734,9 @@ struct CV_EXPORTS MatStep
     size_t& operator[](int i) CV_NOEXCEPT;
     operator size_t() const;
     MatStep& operator = (size_t s);
+    void clear();
 
-    size_t* p;
-    size_t buf[3];
-protected:
-    MatStep& operator = (const MatStep&);
+    size_t p[MatShape::MAX_DIMS];
 };
 
 /** @example samples/cpp/cout_mat.cpp
@@ -1633,6 +1619,12 @@ public:
     */
     CV_NODISCARD_STD static MatExpr zeros(int ndims, const int* sz, int type);
 
+    /** @overload
+    @param shape Array shape.
+    @param type Created matrix type.
+    */
+    CV_NODISCARD_STD static MatExpr zeros(const MatShape& shape, int type);
+
     /** @brief Returns an array of all 1's of the specified size and type.
 
     The method returns a Matlab-style 1's array initializer, similarly to Mat::zeros. Note that using
@@ -1664,6 +1656,12 @@ public:
     */
     CV_NODISCARD_STD static MatExpr ones(int ndims, const int* sz, int type);
 
+    /** @overload
+    @param shape Array shape.
+    @param type Created matrix type.
+    */
+    CV_NODISCARD_STD static MatExpr ones(const MatShape& shape, int type);
+
     /** @brief Returns an identity matrix of the specified size and type.
 
     The method returns a Matlab-style identity matrix initializer, similarly to Mat::zeros. Similarly to
@@ -2745,6 +2743,8 @@ public:
     //! constructs n-dimensional matrix
     UMat(int ndims, const int* sizes, int type, UMatUsageFlags usageFlags = USAGE_DEFAULT);
     UMat(int ndims, const int* sizes, int type, const Scalar& s, UMatUsageFlags usageFlags = USAGE_DEFAULT);
+    UMat(const MatShape& shape, int type, UMatUsageFlags usageFlags = USAGE_DEFAULT);
+    UMat(const MatShape& shape, int type, const Scalar& s, UMatUsageFlags usageFlags = USAGE_DEFAULT);
 
     //! copy constructor
     UMat(const UMat& m);
@@ -2816,9 +2816,11 @@ public:
     CV_NODISCARD_STD static UMat zeros(int rows, int cols, int type, UMatUsageFlags usageFlags = USAGE_DEFAULT);
     CV_NODISCARD_STD static UMat zeros(Size size, int type, UMatUsageFlags usageFlags = USAGE_DEFAULT);
     CV_NODISCARD_STD static UMat zeros(int ndims, const int* sz, int type, UMatUsageFlags usageFlags = USAGE_DEFAULT);
+    CV_NODISCARD_STD static UMat zeros(const MatShape& shape, int type, UMatUsageFlags usageFlags = USAGE_DEFAULT);
     CV_NODISCARD_STD static UMat ones(int rows, int cols, int type, UMatUsageFlags usageFlags = USAGE_DEFAULT);
     CV_NODISCARD_STD static UMat ones(Size size, int type, UMatUsageFlags usageFlags = USAGE_DEFAULT);
     CV_NODISCARD_STD static UMat ones(int ndims, const int* sz, int type, UMatUsageFlags usageFlags = USAGE_DEFAULT);
+    CV_NODISCARD_STD static UMat ones(const MatShape& shape, int type, UMatUsageFlags usageFlags = USAGE_DEFAULT);
     CV_NODISCARD_STD static UMat eye(int rows, int cols, int type, UMatUsageFlags usageFlags = USAGE_DEFAULT);
     CV_NODISCARD_STD static UMat eye(Size size, int type, UMatUsageFlags usageFlags = USAGE_DEFAULT);
 

modules/core/include/opencv2/core/mat.inl.hpp

@@ -94,16 +94,24 @@ inline const int* MatShape::data() const { return p; }
 
 inline int& MatShape::operator [](size_t idx)
 {
-    CV_Assert(idx < (size_t)(dims >= 0 ? dims : 1));
+    CV_Assert(idx < (size_t)(dims > 0 ? dims : 1));
     return p[idx];
 }
 
 inline const int& MatShape::operator [](size_t idx) const
 {
-    CV_Assert(idx < (size_t)(dims >= 0 ? dims : 1));
+    CV_Assert(idx < (size_t)(dims > 0 ? dims : 1));
     return p[idx];
 }
 
+inline Size MatShape::operator()() const
+{
+    CV_Assert(dims <= 2);
+    int cols = dims > 0 ? p[dims > 1] : int(dims >= 0);
+    int rows = dims > 1 ? p[0] : int(dims >= 0);
+    return Size(cols, rows);
+}
+
 template<typename _It> inline MatShape::MatShape(_It begin, _It end)
 {
     int buf[MAX_DIMS];
@@ -540,11 +548,10 @@ template<typename _Tp> inline
 Mat::Mat(const std::vector<_Tp>& vec, bool copyData)
     : flags(+MAGIC_VAL + traits::Type<_Tp>::value + CV_MAT_CONT_FLAG), dims(1), rows(1),
       cols((int)vec.size()), data(0), datastart(0), dataend(0), datalimit(0),
-      allocator(0), u(0), size(&cols)
+      allocator(0), u(0), size(1)
 {
-    step.buf[1] = sizeof(_Tp);
-    step.buf[0] = cols*step.buf[1];
-    step.p = &step.buf[1];
+    size[0] = cols;
+    step[0] = sizeof(_Tp);
 
     if(vec.empty())
         return;
@@ -585,11 +592,10 @@ template<typename _Tp, std::size_t _Nm> inline
 Mat::Mat(const std::array<_Tp, _Nm>& arr, bool copyData)
     : flags(+MAGIC_VAL + traits::Type<_Tp>::value + CV_MAT_CONT_FLAG), dims(1), rows(1),
       cols((int)arr.size()), data(0), datastart(0), dataend(0), datalimit(0),
-      allocator(0), u(0), size(&cols), step(0)
+      allocator(0), u(0), size(1), step(0)
 {
-    step.buf[1] = sizeof(_Tp);
-    step.buf[0] = cols*step.buf[1];
-    step.p = &step.buf[1];
+    size[0] = cols;
+    step[0] = sizeof(_Tp);
 
     if(arr.empty())
         return;
@@ -605,13 +611,12 @@ Mat::Mat(const std::array<_Tp, _Nm>& arr, bool copyData)
 template<typename _Tp, int n> inline
 Mat::Mat(const Vec<_Tp, n>& vec, bool copyData)
     : flags(+MAGIC_VAL + traits::Type<_Tp>::value + CV_MAT_CONT_FLAG), dims(1), rows(1), cols(n), data(0),
-      datastart(0), dataend(0), datalimit(0), allocator(0), u(0), size(&cols), step(0)
+      datastart(0), dataend(0), datalimit(0), allocator(0), u(0), size(1), step(0)
 {
     if( !copyData )
     {
-        step.p = &step.buf[1];
-        step.buf[1] = sizeof(_Tp);
-        step.buf[0] = cols*step.buf[1];
+        size[0] = cols;
+        step[0] = sizeof(_Tp);
         datastart = data = (uchar*)vec.val;
         datalimit = dataend = datastart + cols * step[0];
     }
@@ -623,13 +628,14 @@ Mat::Mat(const Vec<_Tp, n>& vec, bool copyData)
 template<typename _Tp, int m, int n> inline
 Mat::Mat(const Matx<_Tp,m,n>& M, bool copyData)
     : flags(+MAGIC_VAL + traits::Type<_Tp>::value + CV_MAT_CONT_FLAG), dims(2), rows(m), cols(n), data(0),
-      datastart(0), dataend(0), datalimit(0), allocator(0), u(0), size(&rows), step(0)
+      datastart(0), dataend(0), datalimit(0), allocator(0), u(0), size(2), step(0)
 {
     if( !copyData )
     {
-        step.p = &step.buf[0];
-        step.buf[1] = sizeof(_Tp);
-        step.buf[0] = n*sizeof(_Tp);
+        size[1] = cols;
+        size[0] = rows;
+        step[1] = sizeof(_Tp);
+        step[0] = n*sizeof(_Tp);
         datastart = data = (uchar*)M.val;
         datalimit = dataend = datastart + rows * step[0];
     }
@@ -640,13 +646,12 @@ Mat::Mat(const Matx<_Tp,m,n>& M, bool copyData)
 template<typename _Tp> inline
 Mat::Mat(const Point_<_Tp>& pt, bool copyData)
     : flags(+MAGIC_VAL + traits::Type<_Tp>::value + CV_MAT_CONT_FLAG), dims(1), rows(1), cols(2), data(0),
-      datastart(0), dataend(0), datalimit(0), allocator(0), u(0), size(&cols), step(0)
+      datastart(0), dataend(0), datalimit(0), allocator(0), u(0), size(1), step(0)
 {
     if( !copyData )
     {
-        step.p = &step.buf[1];
-        step.buf[1] = sizeof(_Tp);
-        step.buf[0] = cols*step.buf[1];
+        size[0] = cols;
+        step[0] = sizeof(_Tp);
         datastart = data = (uchar*)&pt.x;
         datalimit = dataend = datastart + cols * step[0];
     }
@@ -661,20 +666,20 @@ Mat::Mat(const Point_<_Tp>& pt, bool copyData)
 
 template<typename _Tp> inline
 Mat::Mat(const Point3_<_Tp>& pt, bool copyData)
-    : flags(+MAGIC_VAL + traits::Type<_Tp>::value + CV_MAT_CONT_FLAG), dims(2), rows(3), cols(1), data(0),
-      datastart(0), dataend(0), datalimit(0), allocator(0), u(0), size(&rows), step(0)
+    : flags(+MAGIC_VAL + traits::Type<_Tp>::value + CV_MAT_CONT_FLAG), dims(1), rows(1), cols(3), data(0),
+      datastart(0), dataend(0), datalimit(0), allocator(0), u(0), size(1), step(0)
 {
     if( !copyData )
     {
-        step.p = &step.buf[1];
-        step.buf[1] = sizeof(_Tp);
-        step.buf[0] = cols*step.buf[1];
+        size[0] = cols;
+        step[0] = sizeof(_Tp);
         datastart = data = (uchar*)&pt.x;
         datalimit = dataend = datastart + cols * step[0];
     }
     else
     {
-        create(3, 1, traits::Type<_Tp>::value);
+        int sz = 3;
+        create(1, &sz, traits::Type<_Tp>::value);
         ((_Tp*)data)[0] = pt.x;
         ((_Tp*)data)[1] = pt.y;
         ((_Tp*)data)[2] = pt.z;
@@ -684,7 +689,7 @@ Mat::Mat(const Point3_<_Tp>& pt, bool copyData)
 template<typename _Tp> inline
 Mat::Mat(const MatCommaInitializer_<_Tp>& commaInitializer)
     : flags(+MAGIC_VAL + traits::Type<_Tp>::value + CV_MAT_CONT_FLAG), dims(0), rows(0), cols(0), data(0),
-      datastart(0), dataend(0), allocator(0), u(0), size(&rows)
+      datastart(0), dataend(0), allocator(0), u(0)
 {
     *this = commaInitializer.operator Mat_<_Tp>();
 }
@@ -791,6 +796,12 @@ int Mat::channels() const
     return CV_MAT_CN(flags);
 }
 
+inline
+MatShape Mat::shape() const
+{
+    return size;
+}
+
 inline
 uchar* Mat::ptr(int y)
 {
@@ -1093,9 +1104,9 @@ const _Tp& Mat::at(int i0) const
     if( isContinuous() || rows == 1 )
         return ((const _Tp*)data)[i0];
     if( cols == 1 )
-        return *(const _Tp*)(data + step.buf[0] * i0);
+        return *(const _Tp*)(data + step[0] * i0);
     int i = i0 / cols, j = i0 - i * cols;
-    return ((const _Tp*)(data + step.buf[0] * i))[j];
+    return ((const _Tp*)(data + step[0] * i))[j];
 }
 
 template<typename _Tp> inline
@@ -1287,6 +1298,10 @@ void Mat::push_back(const _Tp& elem)
     if( !isSubmatrix() && isContinuous() && tmp <= datalimit )
     {
         *(_Tp*)(data + (size.p[0]++) * step.p[0]) = elem;
+        if (dims == 2)
+            rows = size.p[0];
+        else if (dims == 1)
+            cols = size.p[0];
         dataend = tmp;
     }
     else
@@ -1315,6 +1330,7 @@ void Mat::push_back(const std::vector<_Tp>& v)
 
 ///////////////////////////// MatSize ////////////////////////////
 
+/*
 inline
 MatSize::MatSize(int* _p) CV_NOEXCEPT
     : p(_p) {}
@@ -1364,21 +1380,21 @@ bool MatSize::operator != (const MatSize& sz) const CV_NOEXCEPT
 {
     return !(*this == sz);
 }
-
-
+*/
 
 ///////////////////////////// MatStep ////////////////////////////
 
 inline
 MatStep::MatStep() CV_NOEXCEPT
 {
-    p = buf; p[0] = p[1] = 0; p[2] = 153;
+    clear();
 }
 
 inline
 MatStep::MatStep(size_t s) CV_NOEXCEPT
 {
-    p = buf; p[0] = s; p[1] = 0; p[2] = 153;
+    clear();
+    p[0] = s;
 }
 
 inline
@@ -1395,18 +1411,20 @@ size_t& MatStep::operator[](int i) CV_NOEXCEPT
 
 inline MatStep::operator size_t() const
 {
-    CV_DbgAssert( p == buf || p == buf+1 );
-    return *p;
+    return p[0];
 }
 
 inline MatStep& MatStep::operator = (size_t s)
 {
-    CV_DbgAssert( p == buf || p == buf+1 );
-    *p = s;
+    p[0] = s;
     return *this;
 }
 
-
+inline void MatStep::clear()
+{
+    for (int i = 0; i < MatShape::MAX_DIMS; i++)
+        p[i] = 0;
+}
 
 ////////////////////////////// Mat_<_Tp> ////////////////////////////