1
0
mirror of https://github.com/opencv/opencv.git synced 2026-07-30 07:43:03 +04:00

imgproc: optimize tiled parallel filter with TLS buffers and custom border fill

This commit is contained in:
Ismail
2026-04-24 09:22:35 +02:00
parent a5295015b2
commit d16f2bfef2
9 changed files with 472 additions and 32 deletions
+13 -18
View File
@@ -69,13 +69,14 @@ template<typename T, typename ST>
struct RowSum :
public BaseRowFilter
{
RowSum( int _ksize, int _anchor ) :
BaseRowFilter()
RowSum( int _ksize, int _anchor )
{
ksize = _ksize;
anchor = _anchor;
}
bool isStateless() const CV_OVERRIDE { return true; }
virtual void operator()(const uchar* src, uchar* dst, int width, int cn) CV_OVERRIDE
{
CV_INSTRUMENT_REGION();
@@ -180,8 +181,7 @@ template<typename ST, typename T>
struct ColumnSum :
public BaseColumnFilter
{
ColumnSum( int _ksize, int _anchor, double _scale ) :
BaseColumnFilter()
ColumnSum( int _ksize, int _anchor, double _scale )
{
ksize = _ksize;
anchor = _anchor;
@@ -280,8 +280,7 @@ template<>
struct ColumnSum<int, uchar> :
public BaseColumnFilter
{
ColumnSum( int _ksize, int _anchor, double _scale ) :
BaseColumnFilter()
ColumnSum( int _ksize, int _anchor, double _scale )
{
ksize = _ksize;
anchor = _anchor;
@@ -436,8 +435,7 @@ public BaseColumnFilter
{
enum { SHIFT = 23 };
ColumnSum( int _ksize, int _anchor, double _scale ) :
BaseColumnFilter()
ColumnSum( int _ksize, int _anchor, double _scale )
{
ksize = _ksize;
anchor = _anchor;
@@ -613,8 +611,7 @@ template<>
struct ColumnSum<int, short> :
public BaseColumnFilter
{
ColumnSum( int _ksize, int _anchor, double _scale ) :
BaseColumnFilter()
ColumnSum( int _ksize, int _anchor, double _scale )
{
ksize = _ksize;
anchor = _anchor;
@@ -763,8 +760,7 @@ template<>
struct ColumnSum<int, ushort> :
public BaseColumnFilter
{
ColumnSum( int _ksize, int _anchor, double _scale ) :
BaseColumnFilter()
ColumnSum( int _ksize, int _anchor, double _scale )
{
ksize = _ksize;
anchor = _anchor;
@@ -910,8 +906,7 @@ template<>
struct ColumnSum<int, int> :
public BaseColumnFilter
{
ColumnSum( int _ksize, int _anchor, double _scale ) :
BaseColumnFilter()
ColumnSum( int _ksize, int _anchor, double _scale )
{
ksize = _ksize;
anchor = _anchor;
@@ -1044,8 +1039,7 @@ template<>
struct ColumnSum<int, float> :
public BaseColumnFilter
{
ColumnSum( int _ksize, int _anchor, double _scale ) :
BaseColumnFilter()
ColumnSum( int _ksize, int _anchor, double _scale )
{
ksize = _ksize;
anchor = _anchor;
@@ -1700,13 +1694,14 @@ template<typename T, typename ST>
struct SqrRowSum :
public BaseRowFilter
{
SqrRowSum( int _ksize, int _anchor ) :
BaseRowFilter()
SqrRowSum( int _ksize, int _anchor )
{
ksize = _ksize;
anchor = _anchor;
}
bool isStateless() const CV_OVERRIDE { return true; }
virtual void operator()(const uchar* src, uchar* dst, int width, int cn) CV_OVERRIDE
{
CV_INSTRUMENT_REGION();
+15 -6
View File
@@ -63,14 +63,14 @@
namespace cv {
BaseRowFilter::BaseRowFilter() { ksize = anchor = -1; }
BaseRowFilter::BaseRowFilter() : ksize(-1), anchor(-1) {}
BaseRowFilter::~BaseRowFilter() {}
BaseColumnFilter::BaseColumnFilter() { ksize = anchor = -1; }
BaseColumnFilter::BaseColumnFilter() : ksize(-1), anchor(-1) {}
BaseColumnFilter::~BaseColumnFilter() {}
void BaseColumnFilter::reset() {}
BaseFilter::BaseFilter() { ksize = Size(-1,-1); anchor = Point(-1,-1); }
BaseFilter::BaseFilter() : ksize(-1, -1), anchor(-1, -1) {}
BaseFilter::~BaseFilter() {}
void BaseFilter::reset() {}
@@ -207,6 +207,14 @@ int FilterEngine::proceed(const uchar* src, int srcstep, int count,
CV_CPU_DISPATCH_MODES_ALL);
}
bool FilterEngine::isStateless() const
{
bool s2d = !filter2D || filter2D->isStateless();
bool sr = !rowFilter || rowFilter->isStateless();
bool sc = !columnFilter || columnFilter->isStateless();
return s2d && sr && sc;
}
void FilterEngine::apply(const Mat& src, Mat& dst, const Size& wsz, const Point& ofs)
{
CV_INSTRUMENT_REGION();
@@ -455,16 +463,18 @@ template<typename ST, class CastOp, class VecOp> struct Filter2D : public BaseFi
vecOp = _vecOp;
CV_Assert( _kernel.type() == DataType<KT>::type );
preprocess2DKernel( _kernel, coords, coeffs );
ptrs.resize( coords.size() );
}
bool isStateless() const CV_OVERRIDE { return true; }
void operator()(const uchar** src, uchar* dst, int dststep, int count, int width, int cn) CV_OVERRIDE
{
KT _delta = delta;
const Point* pt = &coords[0];
const KT* kf = (const KT*)&coeffs[0];
const ST** kp = (const ST**)&ptrs[0];
int i, k, nz = (int)coords.size();
AutoBuffer<const ST*> _kp(nz);
const ST** kp = _kp.data();
CastOp castOp = castOp0;
width *= cn;
@@ -507,7 +517,6 @@ template<typename ST, class CastOp, class VecOp> struct Filter2D : public BaseFi
std::vector<Point> coords;
std::vector<uchar> coeffs;
std::vector<uchar*> ptrs;
KT delta;
CastOp castOp0;
VecOp vecOp;
+2
View File
@@ -48,6 +48,8 @@
namespace cv
{
#ifdef HAVE_OPENCL
bool ocl_sepFilter2D(
InputArray _src, OutputArray _dst, int ddepth,
+261 -3
View File
@@ -43,6 +43,7 @@
#include "precomp.hpp"
#include "opencv2/core/hal/intrin.hpp"
#include "filter.hpp"
#include "opencv2/core/utils/tls.hpp"
#include <cstddef>
@@ -296,12 +297,264 @@ int FilterEngine__proceed(FilterEngine& this_, const uchar* src, int srcstep, in
return dy;
}
// Lightweight tile border fill, templated on element size (bytes) so the
// compiler can inline and optimize memcpy for common fixed sizes (1, 2, 4 …).
// Avoids the full cv::copyMakeBorder() overhead: no OpenCL/IPP dispatch,
// no Mat header re-allocation, and a single AutoBuffer for the border table.
template<int esz>
static void fillTileBorder(
const uchar* src, size_t srcstep, int src_w, int src_h,
uchar* dst, size_t dststep,
int pad_top, int pad_bottom, int pad_left, int pad_right,
int borderType, const uchar* constVal)
{
const int dst_w = src_w + pad_left + pad_right;
// ── Left/right column offset table (byte offsets into a source row) ────────
AutoBuffer<int> _tab(pad_left + pad_right);
int* tab = _tab.data();
for (int i = 0; i < pad_left; i++)
tab[i] = borderInterpolate(i - pad_left, src_w, borderType) * esz;
for (int i = 0; i < pad_right; i++)
tab[pad_left + i] = borderInterpolate(src_w + i, src_w, borderType) * esz;
// For BORDER_CONSTANT top/bottom rows, pre-build a full-width fill row.
AutoBuffer<uchar> _constRow;
uchar* constRow = nullptr;
if (borderType == BORDER_CONSTANT && (pad_top > 0 || pad_bottom > 0))
{
_constRow.allocate(dst_w * esz);
constRow = _constRow.data();
for (int x = 0; x < dst_w; x++)
memcpy(constRow + x * esz, constVal, esz);
}
// ── Interior rows ─────────────────────────────────────────────────────────
uchar* dstRow = dst + pad_top * dststep;
for (int r = 0; r < src_h; r++, dstRow += dststep, src += srcstep)
{
// Copy source pixels into the interior portion of the row.
memcpy(dstRow + pad_left * esz, src, src_w * esz);
if (borderType == BORDER_CONSTANT)
{
for (int i = 0; i < pad_left; i++)
memcpy(dstRow + i * esz, constVal, esz);
for (int i = 0; i < pad_right; i++)
memcpy(dstRow + (pad_left + src_w + i) * esz, constVal, esz);
}
else
{
for (int i = 0; i < pad_left; i++)
memcpy(dstRow + i * esz, src + tab[i], esz);
for (int i = 0; i < pad_right; i++)
memcpy(dstRow + (pad_left + src_w + i) * esz, src + tab[pad_left + i], esz);
}
}
// ── Top rows ──────────────────────────────────────────────────────────────
for (int r = 0; r < pad_top; r++)
{
if (borderType == BORDER_CONSTANT)
memcpy(dst + r * dststep, constRow, dst_w * esz);
else
{
int j = borderInterpolate(r - pad_top, src_h, borderType);
memcpy(dst + r * dststep, dst + (pad_top + j) * dststep, dst_w * esz);
}
}
// ── Bottom rows ───────────────────────────────────────────────────────────
uchar* dstBot = dst + (pad_top + src_h) * dststep;
for (int r = 0; r < pad_bottom; r++)
{
if (borderType == BORDER_CONSTANT)
memcpy(dstBot + r * dststep, constRow, dst_w * esz);
else
{
int j = borderInterpolate(src_h + r, src_h, borderType);
memcpy(dstBot + r * dststep, dst + (pad_top + j) * dststep, dst_w * esz);
}
}
}
class TiledFilterInvoker : public ParallelLoopBody
{
struct TiledFilterBuffers {
Mat padded_tile;
Mat hbuf;
};
public:
TiledFilterInvoker(FilterEngine& _fe, const Mat& _src, Mat& _dst, int _tileSize = 128)
: fe(_fe), src(_src), dst(_dst), tileSize(_tileSize)
{
tilesX = (dst.cols + tileSize - 1) / tileSize;
}
virtual void operator() (const Range& range) const CV_OVERRIDE
{
int ax = fe.anchor.x, kwidth = fe.ksize.width;
int ay = fe.anchor.y, kheight = fe.ksize.height;
int dx1 = ax, dx2 = kwidth - ax - 1;
int dy1 = ay, dy2 = kheight - ay - 1;
int borderType = fe.rowBorderType;
int cn = CV_MAT_CN(fe.srcType);
bool isSep = fe.isSeparable();
TiledFilterBuffers& tls = tlsData.getRef();
for (int i = range.start; i < range.end; i++)
{
int ty = i / tilesX;
int tx = i % tilesX;
int dst_x = tx * tileSize;
int dst_y = ty * tileSize;
int w = std::min(tileSize, dst.cols - dst_x);
int h = std::min(tileSize, dst.rows - dst_y);
int src_x1 = dst_x - dx1, src_y1 = dst_y - dy1;
int src_x2 = dst_x + w + dx2, src_y2 = dst_y + h + dy2;
int pad_top = std::max(0, -src_y1);
int pad_bottom = std::max(0, src_y2 - src.rows);
int pad_left = std::max(0, -src_x1);
int pad_right = std::max(0, src_x2 - src.cols);
int clamped_x1 = std::max(0, src_x1);
int clamped_y1 = std::max(0, src_y1);
int clamped_x2 = std::min(src.cols, src_x2);
int clamped_y2 = std::min(src.rows, src_y2);
Mat src_region = src(Rect(clamped_x1, clamped_y1,
clamped_x2 - clamped_x1,
clamped_y2 - clamped_y1));
Mat tile_mat;
if (pad_top == 0 && pad_bottom == 0 && pad_left == 0 && pad_right == 0)
{
tile_mat = src_region;
}
else
{
int padded_w = w + dx1 + dx2;
int padded_h = h + dy1 + dy2;
if (tls.padded_tile.cols < padded_w || tls.padded_tile.rows < padded_h || tls.padded_tile.type() != fe.srcType)
tls.padded_tile.create(padded_h, padded_w, fe.srcType);
tile_mat = tls.padded_tile(Rect(0, 0, padded_w, padded_h));
const int esz = (int)CV_ELEM_SIZE(fe.srcType);
const uchar* cval = fe.constBorderValue.empty() ? nullptr : &fe.constBorderValue[0];
#define FILL_BORDER(E) fillTileBorder<E>(src_region.ptr(), (size_t)src_region.step, \
src_region.cols, src_region.rows, tile_mat.ptr(), (size_t)tile_mat.step, \
pad_top, pad_bottom, pad_left, pad_right, borderType, cval)
switch (esz)
{
case 1: FILL_BORDER(1); break;
case 2: FILL_BORDER(2); break;
case 3: FILL_BORDER(3); break;
case 4: FILL_BORDER(4); break;
case 6: FILL_BORDER(6); break;
case 8: FILL_BORDER(8); break;
case 12: FILL_BORDER(12); break;
case 16: FILL_BORDER(16); break;
default:
{
// Generic fallback for exotic element sizes.
Scalar bv = Scalar::all(0);
if (!fe.constBorderValue.empty())
{
const uchar* bptr = &fe.constBorderValue[0];
int depth = CV_MAT_DEPTH(fe.srcType);
for (int k = 0; k < cn; k++)
{
switch(depth)
{
case CV_8U: bv[k] = bptr[k]; break;
case CV_8S: bv[k] = ((const schar*)bptr)[k]; break;
case CV_16U: bv[k] = ((const ushort*)bptr)[k]; break;
case CV_16S: bv[k] = ((const short*)bptr)[k]; break;
case CV_32S: bv[k] = ((const int*)bptr)[k]; break;
case CV_32F: bv[k] = ((const float*)bptr)[k]; break;
case CV_64F: bv[k] = ((const double*)bptr)[k]; break;
default: bv[k] = bptr[k];
}
}
}
copyMakeBorder(src_region, tile_mat, pad_top, pad_bottom, pad_left, pad_right, borderType, bv);
}
}
#undef FILL_BORDER
}
uchar* dst_ptr = dst.ptr(dst_y) + dst_x * dst.elemSize();
if (isSep)
{
int hstep = (int)alignSize(w * CV_ELEM_SIZE(fe.bufType), VEC_ALIGN);
if (tls.hbuf.rows < tile_mat.rows || tls.hbuf.cols < hstep)
tls.hbuf.create(tile_mat.rows, hstep, CV_8U);
uchar* hbuf = tls.hbuf.ptr();
for (int r = 0; r < tile_mat.rows; r++)
(*fe.rowFilter)(tile_mat.ptr(r), hbuf + r * hstep, w, cn);
AutoBuffer<const uchar*> _brows(h + kheight - 1);
const uchar** brows = _brows.data();
for (int m = 0; m < h + kheight - 1; m++)
brows[m] = hbuf + m * hstep;
(*fe.columnFilter)(brows, dst_ptr, (int)dst.step, h, w * cn);
}
else
{
AutoBuffer<const uchar*> _brows(h + kheight - 1);
const uchar** brows = _brows.data();
for (int k = 0; k < h + kheight - 1; k++)
brows[k] = tile_mat.ptr(k);
(*fe.filter2D)(brows, dst_ptr, (int)dst.step, h, w, cn);
}
}
}
private:
FilterEngine& fe;
const Mat& src;
Mat& dst;
int tileSize;
int tilesX;
mutable TLSData<TiledFilterBuffers> tlsData;
};
void FilterEngine__apply(FilterEngine& this_, const Mat& src, Mat& dst, const Size& wsz, const Point& ofs)
{
CV_INSTRUMENT_REGION();
CV_DbgAssert(src.type() == this_.srcType && dst.type() == this_.dstType);
// Tiled Fast Path for stateless parallel filters on large images.
int nthreads = cv::getNumThreads();
if (this_.isStateless() && nthreads > 1 &&
(size_t)src.total() >= std::max((size_t)1024 * 1024, (size_t)nthreads * 64 * 1024) &&
this_.rowBorderType == this_.columnBorderType)
{
// For in-place operations (e.g. morphologyEx MORPH_OPEN), clone src so that
// concurrent tiles read from an immutable snapshot rather than racing on writes.
Mat src_copy = (src.data == dst.data) ? src.clone() : src;
// Heuristic: Balance L2 cache locality (128) vs parallel load balancing (64).
int tileSize = (src.total() < (size_t)nthreads * 128 * 128 * 4) ? 64 : 128;
int totalTiles = ((dst.cols + tileSize - 1) / tileSize) * ((dst.rows + tileSize - 1) / tileSize);
TiledFilterInvoker invoker(this_, src_copy, dst, tileSize);
parallel_for_(Range(0, totalTiles), invoker);
return;
}
FilterEngine__start(this_, wsz, src.size(), ofs);
int y = this_.startY - ofs.y;
FilterEngine__proceed(this_,
@@ -2398,6 +2651,8 @@ template<typename ST, typename DT, class VecOp> struct RowFilter : public BaseRo
vecOp = _vecOp;
}
bool isStateless() const CV_OVERRIDE { return true; }
void operator()(const uchar* src, uchar* dst, int width, int cn) CV_OVERRIDE
{
CV_INSTRUMENT_REGION();
@@ -2599,6 +2854,8 @@ template<class CastOp, class VecOp> struct ColumnFilter : public BaseColumnFilte
(kernel.rows == 1 || kernel.cols == 1));
}
bool isStateless() const CV_OVERRIDE { return true; }
void operator()(const uchar** src, uchar* dst, int dststep, int count, int width) CV_OVERRIDE
{
CV_INSTRUMENT_REGION();
@@ -3116,16 +3373,18 @@ template<typename ST, class CastOp, class VecOp> struct Filter2D : public BaseFi
vecOp = _vecOp;
CV_Assert( _kernel.type() == DataType<KT>::type );
preprocess2DKernel( _kernel, coords, coeffs );
ptrs.resize( coords.size() );
}
bool isStateless() const CV_OVERRIDE { return true; }
void operator()(const uchar** src, uchar* dst, int dststep, int count, int width, int cn) CV_OVERRIDE
{
KT _delta = delta;
const Point* pt = &coords[0];
const KT* kf = (const KT*)&coeffs[0];
const ST** kp = (const ST**)&ptrs[0];
int i, k, nz = (int)coords.size();
AutoBuffer<const ST*> _kp(nz);
const ST** kp = _kp.data();
CastOp castOp = castOp0;
width *= cn;
@@ -3168,7 +3427,6 @@ template<typename ST, class CastOp, class VecOp> struct Filter2D : public BaseFi
std::vector<Point> coords;
std::vector<uchar> coeffs;
std::vector<uchar*> ptrs;
KT delta;
CastOp castOp0;
VecOp vecOp;
+9
View File
@@ -74,6 +74,8 @@ public:
virtual ~BaseRowFilter();
//! the filtering operator. Must be overridden in the derived classes. The horizontal border interpolation is done outside of the class.
virtual void operator()(const uchar* src, uchar* dst, int width, int cn) = 0;
//! returns true if the filter holds no mutable state between calls (safe for concurrent tile-parallel invocation)
virtual bool isStateless() const { return false; }
int ksize;
int anchor;
@@ -104,6 +106,8 @@ public:
virtual void operator()(const uchar** src, uchar* dst, int dststep, int dstcount, int width) = 0;
//! resets the internal buffers, if any
virtual void reset();
//! returns true if the filter holds no mutable state between calls (safe for concurrent tile-parallel invocation)
virtual bool isStateless() const { return false; }
int ksize;
int anchor;
@@ -132,6 +136,8 @@ public:
virtual void operator()(const uchar** src, uchar* dst, int dststep, int dstcount, int width, int cn) = 0;
//! resets the internal buffers, if any
virtual void reset();
//! returns true if the filter holds no mutable state between calls (safe for concurrent tile-parallel invocation)
virtual bool isStateless() const { return false; }
Size ksize;
Point anchor;
@@ -251,6 +257,9 @@ public:
int remainingInputRows() const;
int remainingOutputRows() const;
//! returns true if the engine's filters are entirely stateless and thus safe for 2D parallel block chunking
bool isStateless() const;
int srcType;
int dstType;
int bufType;
+11 -5
View File
@@ -43,6 +43,7 @@
#include "precomp.hpp"
#include <limits.h>
#include "opencv2/core/hal/intrin.hpp"
#include "filter.hpp"
/****************************************************************************************\
Basic Morphological Operations: Erosion & Dilation
@@ -495,6 +496,8 @@ template<class Op, class VecOp> struct MorphRowFilter : public BaseRowFilter
anchor = _anchor;
}
bool isStateless() const CV_OVERRIDE { return true; }
void operator()(const uchar* src, uchar* dst, int width, int cn) CV_OVERRIDE
{
CV_INSTRUMENT_REGION();
@@ -551,6 +554,8 @@ template<class Op, class VecOp> struct MorphColumnFilter : public BaseColumnFilt
anchor = _anchor;
}
bool isStateless() const CV_OVERRIDE { return true; }
void operator()(const uchar** _src, uchar* dst, int dststep, int count, int width) CV_OVERRIDE
{
CV_INSTRUMENT_REGION();
@@ -638,7 +643,7 @@ template<class Op, class VecOp> struct MorphColumnFilter : public BaseColumnFilt
};
template<class Op, class VecOp> struct MorphFilter : BaseFilter
template<class Op, class VecOp> struct MorphFilter : public BaseFilter
{
typedef typename Op::rtype T;
@@ -651,16 +656,18 @@ template<class Op, class VecOp> struct MorphFilter : BaseFilter
std::vector<uchar> coeffs; // we do not really the values of non-zero
// kernel elements, just their locations
preprocess2DKernel( _kernel, coords, coeffs );
ptrs.resize( coords.size() );
}
bool isStateless() const CV_OVERRIDE { return true; }
void operator()(const uchar** src, uchar* dst, int dststep, int count, int width, int cn) CV_OVERRIDE
{
CV_INSTRUMENT_REGION();
const Point* pt = &coords[0];
const T** kp = (const T**)&ptrs[0];
int i, k, nz = (int)coords.size();
AutoBuffer<const T*> _kp(nz);
const T** kp = _kp.data();
Op op;
width *= cn;
@@ -671,7 +678,7 @@ template<class Op, class VecOp> struct MorphFilter : BaseFilter
for( k = 0; k < nz; k++ )
kp[k] = (const T*)src[pt[k].y] + pt[k].x*cn;
i = vecOp(&ptrs[0], nz, dst, width);
i = vecOp((uchar**)kp, nz, dst, width);
#if CV_ENABLE_UNROLLED
for( ; i <= width - 4; i += 4 )
{
@@ -700,7 +707,6 @@ template<class Op, class VecOp> struct MorphFilter : BaseFilter
}
std::vector<Point> coords;
std::vector<uchar*> ptrs;
VecOp vecOp;
};