potok/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-07-30 15:53:03 +04:00
Code Packages Releases Wiki Activity

Merge branch 4.x

Browse Source
This commit is contained in:
Alexander Smorkalov
2024-01-15 17:23:10 +03:00
parent 788c7252dc dad8af6b17
commit c739117a7c
531 changed files with 20900 additions and 12934 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/3d/include/opencv2/3d.hpp
+2 -3
View File
@@ -828,7 +828,7 @@ and a rotation matrix.
It optionally returns three rotation matrices, one for each axis, and the three Euler angles in
degrees (as the return value) that could be used in OpenGL. Note, there is always more than one
sequence of rotations about the three principal axes that results in the same orientation of an
object, e.g. see @cite Slabaugh . Returned tree rotation matrices and corresponding three Euler angles
object, e.g. see @cite Slabaugh . Returned three rotation matrices and corresponding three Euler angles
are only one of the possible solutions.
*/
CV_EXPORTS_W Vec3d RQDecomp3x3( InputArray src, OutputArray mtxR, OutputArray mtxQ,
@@ -854,7 +854,7 @@ matrix and the position of a camera.
It optionally returns three rotation matrices, one for each axis, and three Euler angles that could
be used in OpenGL. Note, there is always more than one sequence of rotations about the three
principal axes that results in the same orientation of an object, e.g. see @cite Slabaugh . Returned
tree rotation matrices and corresponding three Euler angles are only one of the possible solutions.
three rotation matrices and corresponding three Euler angles are only one of the possible solutions.
The function is based on #RQDecomp3x3 .
*/
@@ -2322,7 +2322,6 @@ void initInverseRectificationMap( InputArray cameraMatrix, InputArray distCoeffs
InputArray R, InputArray newCameraMatrix,
const Size& size, int m1type, OutputArray map1, OutputArray map2 );
//! initializes maps for #remap for wide-angle
CV_EXPORTS
float initWideAngleProjMap(InputArray cameraMatrix, InputArray distCoeffs,
modules/calib/include/opencv2/calib.hpp
+10 -1
View File
@@ -411,7 +411,8 @@ enum { CALIB_CB_ADAPTIVE_THRESH = 1,
CALIB_CB_EXHAUSTIVE = 16,
CALIB_CB_ACCURACY = 32,
CALIB_CB_LARGER = 64,
CALIB_CB_MARKER = 128
CALIB_CB_MARKER = 128,
CALIB_CB_PLAIN = 256
};
enum { CALIB_CB_SYMMETRIC_GRID = 1,
@@ -502,6 +503,10 @@ square-like shape) to filter out false quads extracted at the contour retrieval
- @ref CALIB_CB_FAST_CHECK Run a fast check on the image that looks for chessboard corners,
and shortcut the call if none is found. This can drastically speed up the call in the
degenerate condition when no chessboard is observed.
- @ref CALIB_CB_PLAIN All other flags are ignored. The input image is taken as is.
No image processing is done to improve to find the checkerboard. This has the effect of speeding up the
execution of the function but could lead to not recognizing the checkerboard if the image
is not previously binarized in the appropriate manner.
The function attempts to determine whether the input image is a view of the chessboard pattern and
locate the internal chessboard corners. The function returns a non-zero value if all of the corners
@@ -843,6 +848,10 @@ The algorithm performs the following steps:
\f$f_y\f$ (ratios of 10:1 or more)), then you are probably using patternSize=cvSize(rows,cols)
instead of using patternSize=cvSize(cols,rows) in @ref findChessboardCorners.
@note
The function may throw exceptions, if unsupported combination of parameters is provided or
the system is underconstrained.
@sa
calibrateCameraRO, findChessboardCorners, solvePnP, initCameraMatrix2D, stereoCalibrate,
undistort
modules/calib/src/calibinit.cpp
+87 -53
View File
@@ -46,7 +46,7 @@
Here is the copyright notice from the original Vladimir's code:
===============================================================
The algorithms developed and implemented by Vezhnevets Vldimir
The algorithms developed and implemented by Vezhnevets Vladimir
aka Dead Moroz (vvp@graphics.cs.msu.ru)
See http://graphics.cs.msu.su/en/research/calibration/opencv.html
for detailed information.
@@ -54,7 +54,7 @@
Reliability additions and modifications made by Philip Gruebele.
<a href="mailto:pgruebele@cox.net">pgruebele@cox.net</a>
Some further improvements for detection of partially ocluded boards at non-ideal
Some further improvements for detection of partially occluded boards at non-ideal
lighting conditions have been made by Alex Bovyrin and Kurt Kolonige
\************************************************************************************/
@@ -72,6 +72,7 @@
#include "precomp.hpp"
#include "circlesgrid.hpp"
#include "opencv2/imgproc/imgproc_c.h"
#include "opencv2/flann.hpp"
#include <stack>
@@ -240,7 +241,7 @@ public:
void findConnectedQuads(std::vector<ChessBoardQuad*>& out_group, int group_idx);
int checkQuadGroup(std::vector<ChessBoardQuad*>& quad_group, std::vector<ChessBoardCorner*>& out_corners);
int checkQuadGroup(const std::vector<ChessBoardQuad*>& quad_group, std::vector<ChessBoardCorner*>& out_corners);
int cleanFoundConnectedQuads(std::vector<ChessBoardQuad*>& quad_group);
@@ -297,7 +298,7 @@ static void icvSmoothHistogram256(const ArrayContainer& piHist, ArrayContainer&
CV_DbgAssert(iIdx >= 0 && iIdx < 256);
iSmooth += piHist[iIdx];
}
piHistSmooth[i] = iSmooth/(2*iWidth+1);
piHistSmooth[i] = iSmooth/(iIdx_max-iIdx_min+1);
}
}
/***************************************************************************************************/
@@ -325,7 +326,7 @@ static void icvGradientOfHistogram256(const ArrayContainer& piHist, ArrayContain
piHistGrad[255] = 0;
}
/***************************************************************************************************/
//PERFORM SMART IMAGE THRESHOLDING BASED ON ANALYSIS OF INTENSTY HISTOGRAM
//PERFORM SMART IMAGE THRESHOLDING BASED ON ANALYSIS OF INTENSITY HISTOGRAM
static void icvBinarizationHistogramBased(Mat & img)
{
CV_Assert(img.channels() == 1 && img.depth() == CV_8U);
@@ -479,8 +480,7 @@ bool findChessboardCorners(InputArray image_, Size pattern_size,
bool found = false;
const int min_dilations = 0;
const int max_dilations = 7;
const bool is_plain = (flags & CALIB_CB_PLAIN) != 0;
int type = image_.type(), depth = CV_MAT_DEPTH(type), cn = CV_MAT_CN(type);
Mat img = image_.getMat();
@@ -496,6 +496,9 @@ bool findChessboardCorners(InputArray image_, Size pattern_size,
std::vector<Point2f> out_corners;
if (is_plain)
CV_CheckType(type, depth == CV_8U && cn == 1, "Only 8-bit grayscale images are supported whith CALIB_CB_PLAIN flag enable");
if (img.channels() != 1)
{
cvtColor(img, img, COLOR_BGR2GRAY);
@@ -504,10 +507,11 @@ bool findChessboardCorners(InputArray image_, Size pattern_size,
int prev_sqr_size = 0;
Mat thresh_img_new = img.clone();
icvBinarizationHistogramBased(thresh_img_new); // process image in-place
if(!is_plain)
icvBinarizationHistogramBased(thresh_img_new); // process image in-place
SHOW("New binarization", thresh_img_new);
if (flags & CALIB_CB_FAST_CHECK)
if (flags & CALIB_CB_FAST_CHECK && !is_plain)
{
//perform new method for checking chessboard using a binary image.
//image is binarised using a threshold dependent on the image histogram
@@ -523,6 +527,9 @@ bool findChessboardCorners(InputArray image_, Size pattern_size,
ChessBoardDetector detector(pattern_size);
const int min_dilations = 0;
const int max_dilations = is_plain ? 0 : 7;
// Try our standard "1" dilation, but if the pattern is not found, iterate the whole procedure with higher dilations.
// This is necessary because some squares simply do not separate properly with a single dilation. However,
// we want to use the minimum number of dilations possible since dilations cause the squares to become smaller,
@@ -530,7 +537,8 @@ bool findChessboardCorners(InputArray image_, Size pattern_size,
for (int dilations = min_dilations; dilations <= max_dilations; dilations++)
{
//USE BINARY IMAGE COMPUTED USING icvBinarizationHistogramBased METHOD
dilate( thresh_img_new, thresh_img_new, Mat(), Point(-1, -1), 1 );
if(!is_plain)
dilate( thresh_img_new, thresh_img_new, Mat(), Point(-1, -1), 1 );
// So we can find rectangles that go to the edge, we draw a white line around the image edge.
// Otherwise FindContours will miss those clipped rectangle contours.
@@ -551,7 +559,7 @@ bool findChessboardCorners(InputArray image_, Size pattern_size,
DPRINTF("Chessboard detection result 0: %d", (int)found);
// revert to old, slower, method if detection failed
if (!found)
if (!found && !is_plain)
{
if (flags & CALIB_CB_NORMALIZE_IMAGE)
{
@@ -662,7 +670,6 @@ bool findChessboardCorners(InputArray image_, Size pattern_size,
return found;
}
//
// Checks that each board row and column is pretty much monotonous curve:
// It analyzes each row and each column of the chessboard as following:
@@ -671,7 +678,7 @@ bool findChessboardCorners(InputArray image_, Size pattern_size,
// of the neighbor corners in the same row/column.
//
// This function has been created as temporary workaround for the bug in current implementation
// of cvFindChessboardCornes that produces absolutely unordered sets of corners.
// of cvFindChessboardCorners that produces absolutely unordered sets of corners.
//
bool ChessBoardDetector::checkBoardMonotony(const std::vector<Point2f>& corners)
{
@@ -1213,9 +1220,9 @@ int ChessBoardDetector::cleanFoundConnectedQuads(std::vector<ChessBoardQuad*>& q
// We iteratively remove the point which reduces the size of
// the bounding box of the blobs the most
// (since we want the rectangle to be as small as possible)
// remove the quadrange that causes the biggest reduction
// remove the quadrangle that causes the biggest reduction
// in pattern size until we have the correct number
for (; quad_count > count; quad_count--)
while (quad_count > count)
{
double min_box_area = DBL_MAX;
int min_box_area_index = -1;
@@ -1321,7 +1328,7 @@ void ChessBoardDetector::findConnectedQuads(std::vector<ChessBoardQuad*>& out_gr
}
int ChessBoardDetector::checkQuadGroup(std::vector<ChessBoardQuad*>& quad_group, std::vector<ChessBoardCorner*>& out_corners)
int ChessBoardDetector::checkQuadGroup(const std::vector<ChessBoardQuad*>& quad_group, std::vector<ChessBoardCorner*>& out_corners)
{
const int ROW1 = 1000000;
const int ROW2 = 2000000;
@@ -1419,6 +1426,7 @@ int ChessBoardDetector::checkQuadGroup(std::vector<ChessBoardQuad*>& quad_group,
ChessBoardCorner* cur = first;
ChessBoardCorner* right = NULL;
ChessBoardCorner* below = NULL;
out_corners.clear();
out_corners.push_back(cur);
for (int k = 0; k < 4; ++k)
@@ -1588,7 +1596,24 @@ finalize:
void ChessBoardDetector::findQuadNeighbors()
{
const float thresh_scale = 1.f;
const int all_corners_count = all_quads_count * 4;
std::vector<Point2f> all_quads_pts;
all_quads_pts.reserve(all_corners_count);
for (int idx = 0; idx < all_quads_count; idx++)
{
const ChessBoardQuad& cur_quad = (const ChessBoardQuad&)all_quads[idx];
for (int i = 0; i < 4; i++)
all_quads_pts.push_back(cur_quad.corners[i]->pt);
}
const cvflann::KDTreeSingleIndexParams index_params;
flann::GenericIndex<flann::L2_Simple<float>> all_quads_pts_index(Mat(all_quads_pts).reshape(1, all_corners_count), index_params);
// find quad neighbors
std::vector<int> neighbors_indices(all_corners_count);
std::vector<float> neighbors_dists(all_corners_count);
for (int idx = 0; idx < all_quads_count; idx++)
{
ChessBoardQuad& cur_quad = (ChessBoardQuad&)all_quads[idx];
@@ -1611,36 +1636,41 @@ void ChessBoardDetector::findQuadNeighbors()
Point2f pt = cur_quad.corners[i]->pt;
// find the closest corner in all other quadrangles
for (int k = 0; k < all_quads_count; k++)
std::vector<float> query = Mat(pt);
float radius = cur_quad.edge_len * thresh_scale + 1;
const cvflann::SearchParams search_params(-1);
int neighbors_count = all_quads_pts_index.radiusSearch(query, neighbors_indices, neighbors_dists, radius, search_params);
for (int neighbor_idx_idx = 0; neighbor_idx_idx < neighbors_count; neighbor_idx_idx++)
{
const int neighbor_idx = neighbors_indices[neighbor_idx_idx];
const int k = neighbor_idx >> 2;
if (k == idx)
continue;
ChessBoardQuad& q_k = all_quads[k];
const int j = neighbor_idx & 3;
if (q_k.neighbors[j])
continue;
for (int j = 0; j < 4; j++)
const float dist = normL2Sqr<float>(pt - q_k.corners[j]->pt);
if (dist < min_dist &&
dist <= cur_quad.edge_len * thresh_scale &&
dist <= q_k.edge_len * thresh_scale)
{
if (q_k.neighbors[j])
continue;
float dist = normL2Sqr<float>(pt - q_k.corners[j]->pt);
if (dist < min_dist &&
dist <= cur_quad.edge_len*thresh_scale &&
dist <= q_k.edge_len*thresh_scale )
// check edge lengths, make sure they're compatible
// edges that are different by more than 1:4 are rejected.
// edge_len is squared edge length, so we compare them
// with squared constant 16 = 4^2
if (q_k.edge_len > 16 * cur_quad.edge_len ||
cur_quad.edge_len > 16 * q_k.edge_len)
{
// check edge lengths, make sure they're compatible
// edges that are different by more than 1:4 are rejected
float ediff = cur_quad.edge_len - q_k.edge_len;
if (ediff > 32*cur_quad.edge_len ||
ediff > 32*q_k.edge_len)
{
DPRINTF("Incompatible edge lengths");
continue;
}
closest_corner_idx = j;
closest_quad = &q_k;
min_dist = dist;
DPRINTF("Incompatible edge lengths");
continue;
}
closest_corner_idx = j;
closest_quad = &q_k;
min_dist = dist;
}
}
@@ -1681,26 +1711,29 @@ void ChessBoardDetector::findQuadNeighbors()
// check whether the closest corner to closest_corner
// is different from cur_quad->corners[i]->pt
for (j = 0; j < all_quads_count; j++ )
query = Mat(closest_corner.pt);
radius = min_dist + 1;
neighbors_count = all_quads_pts_index.radiusSearch(query, neighbors_indices, neighbors_dists, radius, search_params);
int neighbor_idx_idx = 0;
for (; neighbor_idx_idx < neighbors_count; neighbor_idx_idx++)
{
const int neighbor_idx = neighbors_indices[neighbor_idx_idx];
j = neighbor_idx >> 2;
ChessBoardQuad* q = &const_cast<ChessBoardQuad&>(all_quads[j]);
if (j == idx || q == closest_quad)
continue;
int k = 0;
for (; k < 4; k++ )
const int k = neighbor_idx & 3;
CV_DbgAssert(q);
if (!q->neighbors[k])
{
CV_DbgAssert(q);
if (!q->neighbors[k])
{
if (normL2Sqr<float>(closest_corner.pt - q->corners[k]->pt) < min_dist)
break;
}
if (normL2Sqr<float>(closest_corner.pt - q->corners[k]->pt) < min_dist)
break;
}
if (k < 4)
break;
}
if (j < all_quads_count)
if (neighbor_idx_idx < neighbors_count)
continue;
closest_corner.pt = (pt + closest_corner.pt) * 0.5f;
@@ -1730,8 +1763,8 @@ void ChessBoardDetector::generateQuads(const Mat& image_, int flags)
all_quads.deallocate();
all_corners.deallocate();
// empiric bound for minimal allowed perimeter for squares
int min_size = 25; //cvRound( image->cols * image->rows * .03 * 0.01 * 0.92 );
// empiric bound for minimal allowed area for squares
const int min_area = 25; //cvRound( image->cols * image->rows * .03 * 0.01 * 0.92 );
bool filterQuads = (flags & CALIB_CB_FILTER_QUADS) != 0;
@@ -1759,7 +1792,7 @@ void ChessBoardDetector::generateQuads(const Mat& image_, int flags)
const std::vector<Point>& contour = contours[idx];
Rect contour_rect = boundingRect(contour);
if (contour_rect.area() < min_size)
if (contour_rect.area() < min_area)
continue;
std::vector<Point> approx_contour;
@@ -1803,7 +1836,7 @@ void ChessBoardDetector::generateQuads(const Mat& image_, int flags)
// than rectangular and which are big enough
double d3 = sqrt(normL2Sqr<double>(pt[0] - pt[1]));
double d4 = sqrt(normL2Sqr<double>(pt[1] - pt[2]));
if (!(d3*4 > d4 && d4*4 > d3 && d3*d4 < area*1.5 && area > min_size &&
if (!(d3*4 > d4 && d4*4 > d3 && d3*d4 < area*1.5 && area > min_area &&
d1 >= 0.15 * p && d2 >= 0.15 * p))
continue;
}
@@ -1912,6 +1945,7 @@ bool ChessBoardDetector::processQuads(std::vector<Point2f>& out_corners, int &pr
if (count > 0 || (-count > (int)out_corners.size()))
{
// copy corners to output array
out_corners.clear();
out_corners.reserve(n);
for (int i = 0; i < n; ++i)
out_corners.push_back(corner_group[i]->pt);
modules/calib/test/test_chesscorners.cpp
+45 -3
View File
@@ -73,7 +73,7 @@ void show_points( const Mat& gray, const Mat& expected, const vector<Point2f>& a
#define show_points(...)
#endif
enum Pattern { CHESSBOARD,CHESSBOARD_SB,CIRCLES_GRID, ASYMMETRIC_CIRCLES_GRID};
enum Pattern { CHESSBOARD, CHESSBOARD_SB, CHESSBOARD_PLAIN, CIRCLES_GRID, ASYMMETRIC_CIRCLES_GRID};
class CV_ChessboardDetectorTest : public cvtest::BaseTest
{
@@ -149,6 +149,25 @@ void CV_ChessboardDetectorTest::run( int /*start_from */)
case CHESSBOARD_SB:
checkByGeneratorHighAccuracy(); // not supported by CHESSBOARD
/* fallthrough */
case CHESSBOARD_PLAIN:
checkByGenerator();
if (ts->get_err_code() != cvtest::TS::OK)
{
break;
}
run_batch("negative_list.dat");
if (ts->get_err_code() != cvtest::TS::OK)
{
break;
}
run_batch("chessboard_list.dat");
if (ts->get_err_code() != cvtest::TS::OK)
{
break;
}
break;
case CHESSBOARD:
checkByGenerator();
if (ts->get_err_code() != cvtest::TS::OK)
@@ -191,6 +210,7 @@ void CV_ChessboardDetectorTest::run_batch( const string& filename )
{
case CHESSBOARD:
case CHESSBOARD_SB:
case CHESSBOARD_PLAIN:
folder = string(ts->get_data_path()) + "cv/cameracalibration/";
break;
case CIRCLES_GRID:
@@ -215,6 +235,9 @@ void CV_ChessboardDetectorTest::run_batch( const string& filename )
int progress = 0;
int max_idx = (int)board_list.size()/2;
if(filename.compare("chessboard_list.dat") == 0 && pattern == CHESSBOARD_PLAIN)
max_idx = 7;
double sum_error = 0.0;
int count = 0;
@@ -247,6 +270,7 @@ void CV_ChessboardDetectorTest::run_batch( const string& filename )
size_t count_exp = static_cast<size_t>(expected.cols * expected.rows);
Size pattern_size = expected.size();
Mat ori;
vector<Point2f> v;
int flags = 0;
switch( pattern )
@@ -254,14 +278,30 @@ void CV_ChessboardDetectorTest::run_batch( const string& filename )
case CHESSBOARD:
flags = CALIB_CB_ADAPTIVE_THRESH | CALIB_CB_NORMALIZE_IMAGE;
break;
case CHESSBOARD_PLAIN: {
flags = CALIB_CB_PLAIN;
ori = gray.clone();
int min_size = cvRound((gray.cols * gray.rows * 0.05) / ((pattern_size.width+1) * (pattern_size.height+1)));
if(min_size%2==0) min_size += 1;
adaptiveThreshold(gray, gray, 255, ADAPTIVE_THRESH_MEAN_C, THRESH_BINARY, min_size, 0);
dilate(gray, gray, Mat(), Point(-1, -1), 1);
break;
}
case CIRCLES_GRID:
case CHESSBOARD_SB:
case ASYMMETRIC_CIRCLES_GRID:
default:
flags = 0;
}
bool result = findChessboardCornersWrapper(gray, pattern_size,v,flags);
if(result && sharpness && (pattern == CHESSBOARD_SB || pattern == CHESSBOARD))
if(result && pattern == CHESSBOARD_PLAIN) {
gray = ori;
cornerSubPix(gray, v, Size(6,6), Size(-1,-1), TermCriteria(TermCriteria::EPS + TermCriteria::COUNT, 30, 0.1));
}
if(result && sharpness && (pattern == CHESSBOARD_SB || pattern == CHESSBOARD || pattern == CHESSBOARD_PLAIN))
{
Scalar s= estimateChessboardSharpness(gray,pattern_size,v);
if(fabs(s[0] - sharpness) > 0.1)
@@ -287,7 +327,7 @@ void CV_ChessboardDetectorTest::run_batch( const string& filename )
double err = calcError(v, expected);
max_rough_error = MAX( max_rough_error, err );
#endif
if( pattern == CHESSBOARD )
if( pattern == CHESSBOARD || pattern == CHESSBOARD_PLAIN )
cornerSubPix( gray, v, Size(5, 5), Size(-1,-1), TermCriteria(TermCriteria::EPS|TermCriteria::MAX_ITER, 30, 0.1));
//find4QuadCornerSubpix(gray, v, Size(5, 5));
show_points( gray, expected, v, result );
@@ -381,6 +421,7 @@ bool CV_ChessboardDetectorTest::findChessboardCornersWrapper(InputArray image, S
switch(pattern)
{
case CHESSBOARD:
case CHESSBOARD_PLAIN:
return findChessboardCorners(image,patternSize,corners,flags);
case CHESSBOARD_SB:
// check default settings until flags have been specified
@@ -631,6 +672,7 @@ bool CV_ChessboardDetectorTest::checkByGeneratorHighAccuracy()
TEST(Calib3d_ChessboardDetector, accuracy) { CV_ChessboardDetectorTest test( CHESSBOARD ); test.safe_run(); }
TEST(Calib3d_ChessboardDetector2, accuracy) { CV_ChessboardDetectorTest test( CHESSBOARD_SB ); test.safe_run(); }
TEST(Calib3d_ChessboardDetector3, accuracy) { CV_ChessboardDetectorTest test( CHESSBOARD_PLAIN ); test.safe_run(); }
TEST(Calib3d_CirclesPatternDetector, accuracy) { CV_ChessboardDetectorTest test( CIRCLES_GRID ); test.safe_run(); }
TEST(Calib3d_AsymmetricCirclesPatternDetector, accuracy) { CV_ChessboardDetectorTest test( ASYMMETRIC_CIRCLES_GRID ); test.safe_run(); }
#ifdef HAVE_OPENCV_FLANN
modules/calib/test/test_chesscorners_timing.cpp
+3 -3
View File
@@ -138,13 +138,13 @@ void CV_ChessboardDetectorTimingTest::run( int start_from )
}
int num_pixels = gray.cols*gray.rows;
float check_chessboard_time = float(_time01 - _time0)/(float)cv::getTickFrequency(); // in us
float check_chessboard_time = float(_time01 - _time0)/(float)cv::getTickFrequency(); // in s
ts->printf(cvtest::TS::LOG, " cvCheckChessboard time s: %f, us per pixel: %f\n",
check_chessboard_time*1e-6, check_chessboard_time/num_pixels);
check_chessboard_time, check_chessboard_time*1e6/num_pixels);
float find_chessboard_time = float(_time1 - _time01)/(float)cv::getTickFrequency();
ts->printf(cvtest::TS::LOG, " cvFindChessboard time s: %f, us per pixel: %f\n",
find_chessboard_time*1e-6, find_chessboard_time/num_pixels);
find_chessboard_time, find_chessboard_time*1e6/num_pixels);
progress = update_progress( progress, idx-1, max_idx, 0 );
}
modules/core/CMakeLists.txt
+24 -23
View File
@@ -1,27 +1,27 @@
set(the_description "The Core Functionality")
ocv_add_dispatched_file(mathfuncs_core SSE2 AVX AVX2)
ocv_add_dispatched_file(stat SSE4_2 AVX2)
ocv_add_dispatched_file(arithm SSE2 SSE4_1 AVX2 VSX3)
ocv_add_dispatched_file(convert SSE2 AVX2 VSX3)
ocv_add_dispatched_file(convert_scale SSE2 AVX2)
ocv_add_dispatched_file(count_non_zero SSE2 AVX2)
ocv_add_dispatched_file(has_non_zero SSE2 AVX2)
ocv_add_dispatched_file(matmul SSE2 SSE4_1 AVX2 AVX512_SKX NEON_DOTPROD)
ocv_add_dispatched_file(mean SSE2 AVX2)
ocv_add_dispatched_file(merge SSE2 AVX2)
ocv_add_dispatched_file(nan_mask SSE2 AVX2)
ocv_add_dispatched_file(split SSE2 AVX2)
ocv_add_dispatched_file(sum SSE2 AVX2)
ocv_add_dispatched_file(mathfuncs_core SSE2 AVX AVX2 LASX)
ocv_add_dispatched_file(stat SSE4_2 AVX2 LASX)
ocv_add_dispatched_file(arithm SSE2 SSE4_1 AVX2 VSX3 LASX)
ocv_add_dispatched_file(convert SSE2 AVX2 VSX3 LASX)
ocv_add_dispatched_file(convert_scale SSE2 AVX2 LASX)
ocv_add_dispatched_file(count_non_zero SSE2 AVX2 LASX)
ocv_add_dispatched_file(has_non_zero SSE2 AVX2 LASX )
ocv_add_dispatched_file(matmul SSE2 SSE4_1 AVX2 AVX512_SKX NEON_DOTPROD LASX)
ocv_add_dispatched_file(mean SSE2 AVX2 LASX)
ocv_add_dispatched_file(merge SSE2 AVX2 LASX)
ocv_add_dispatched_file(nan_mask SSE2 AVX2 LASX)
ocv_add_dispatched_file(split SSE2 AVX2 LASX)
ocv_add_dispatched_file(sum SSE2 AVX2 LASX)
# dispatching for accuracy tests
ocv_add_dispatched_file_force_all(test_intrin128 TEST SSE2 SSE3 SSSE3 SSE4_1 SSE4_2 AVX FP16 AVX2 AVX512_SKX)
ocv_add_dispatched_file_force_all(test_intrin256 TEST AVX2 AVX512_SKX)
ocv_add_dispatched_file_force_all(test_intrin256 TEST AVX2 AVX512_SKX LASX)
ocv_add_dispatched_file_force_all(test_intrin512 TEST AVX512_SKX)
set(PARALLEL_ENABLE_PLUGINS_DEFAULT ON)
if(EMSCRIPTEN OR IOS OR WINRT)
if(EMSCRIPTEN OR IOS OR XROS OR WINRT)
set(PARALLEL_ENABLE_PLUGINS_DEFAULT OFF)
endif()
# parallel backends configuration
@@ -50,13 +50,6 @@ if(DEFINED WINRT AND NOT DEFINED ENABLE_WINRT_MODE_NATIVE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /ZW")
endif()
if(HAVE_CUDA)
if(NOT HAVE_opencv_cudev)
message(FATAL_ERROR "CUDA: OpenCV requires enabled 'cudev' module from 'opencv_contrib' repository: https://github.com/opencv/opencv_contrib")
endif()
ocv_warnings_disable(CMAKE_CXX_FLAGS -Wundef -Wenum-compare -Wunused-function -Wshadow)
endif()
if(CV_TRACE AND HAVE_ITT)
add_definitions(-DOPENCV_WITH_ITT=1)
endif()
@@ -154,7 +147,6 @@ elseif(HAVE_CXX11 OR DEFINED OPENCV_ALLOCATOR_STATS_COUNTER_TYPE)
endif()
endif()
if(PARALLEL_ENABLE_PLUGINS)
ocv_append_source_file_compile_definitions(${CMAKE_CURRENT_SOURCE_DIR}/src/parallel/parallel.cpp "PARALLEL_ENABLE_PLUGINS=1")
if(OPENCV_DEBUG_POSTFIX)
@@ -162,6 +154,15 @@ if(PARALLEL_ENABLE_PLUGINS)
endif()
endif()
if(HAVE_CUDA)
if(NOT HAVE_opencv_cudev)
message(FATAL_ERROR "CUDA: OpenCV requires enabled 'cudev' module from 'opencv_contrib' repository: https://github.com/opencv/opencv_contrib")
endif()
if(ENABLE_CUDA_FIRST_CLASS_LANGUAGE)
ocv_module_include_directories(${CUDAToolkit_INCLUDE_DIRS})
endif()
ocv_warnings_disable(CMAKE_CXX_FLAGS -Wundef -Wenum-compare -Wunused-function -Wshadow)
endif()
ocv_create_module(${extra_libs})
modules/core/include/opencv2/core.hpp
+4
View File
@@ -3201,6 +3201,10 @@ public:
* @overload
*/
CV_WRAP void write(FileStorage& fs, const String& name) const;
#if CV_VERSION_MAJOR < 5
/** @deprecated */
void write(const Ptr<FileStorage>& fs, const String& name = String()) const;
#endif
/** @brief Reads algorithm parameters from a file storage
*/
modules/core/include/opencv2/core/bindings_utils.hpp
+38 -33
View File
@@ -75,20 +75,6 @@ String dumpString(const String& argument)
return cv::format("String: %s", argument.c_str());
}
CV_WRAP static inline
String testOverloadResolution(int value, const Point& point = Point(42, 24))
{
return format("overload (int=%d, point=(x=%d, y=%d))", value, point.x,
point.y);
}
CV_WRAP static inline
String testOverloadResolution(const Rect& rect)
{
return format("overload (rect=(x=%d, y=%d, w=%d, h=%d))", rect.x, rect.y,
rect.width, rect.height);
}
CV_WRAP static inline
String dumpRect(const Rect& argument)
{
@@ -111,6 +97,42 @@ String dumpRotatedRect(const RotatedRect& argument)
argument.size.height, argument.angle);
}
CV_WRAP static inline
String dumpRange(const Range& argument)
{
if (argument == Range::all())
{
return "range: all";
}
else
{
return format("range: (s=%d, e=%d)", argument.start, argument.end);
}
}
CV_EXPORTS_W String dumpVectorOfInt(const std::vector<int>& vec);
CV_EXPORTS_W String dumpVectorOfDouble(const std::vector<double>& vec);
CV_EXPORTS_W String dumpVectorOfRect(const std::vector<Rect>& vec);
//! @cond IGNORED
CV_WRAP static inline
String testOverloadResolution(int value, const Point& point = Point(42, 24))
{
return format("overload (int=%d, point=(x=%d, y=%d))", value, point.x,
point.y);
}
CV_WRAP static inline
String testOverloadResolution(const Rect& rect)
{
return format("overload (rect=(x=%d, y=%d, w=%d, h=%d))", rect.x, rect.y,
rect.width, rect.height);
}
CV_WRAP static inline
RotatedRect testRotatedRect(float x, float y, float w, float h, float angle)
{
@@ -126,19 +148,6 @@ std::vector<RotatedRect> testRotatedRectVector(float x, float y, float w, float
return result;
}
CV_WRAP static inline
String dumpRange(const Range& argument)
{
if (argument == Range::all())
{
return "range: all";
}
else
{
return format("range: (s=%d, e=%d)", argument.start, argument.end);
}
}
CV_WRAP static inline
int testOverwriteNativeMethod(int argument)
{
@@ -151,12 +160,6 @@ String testReservedKeywordConversion(int positional_argument, int lambda = 2, in
return format("arg=%d, lambda=%d, from=%d", positional_argument, lambda, from);
}
CV_EXPORTS_W String dumpVectorOfInt(const std::vector<int>& vec);
CV_EXPORTS_W String dumpVectorOfDouble(const std::vector<double>& vec);
CV_EXPORTS_W String dumpVectorOfRect(const std::vector<Rect>& vec);
CV_WRAP static inline
void generateVectorOfRect(size_t len, CV_OUT std::vector<Rect>& vec)
{
@@ -323,6 +326,8 @@ private:
typedef OriginalClassName::Params OriginalClassName_Params;
} // namespace nested
//! @endcond IGNORED
namespace fs {
CV_EXPORTS_W cv::String getCacheDirectoryForDownloads();
} // namespace fs
modules/core/include/opencv2/core/cv_cpu_helper.h
+42
View File
@@ -441,6 +441,48 @@
#endif
#define __CV_CPU_DISPATCH_CHAIN_NEON_DOTPROD(fn, args, mode, ...) CV_CPU_CALL_NEON_DOTPROD(fn, args); __CV_EXPAND(__CV_CPU_DISPATCH_CHAIN_ ## mode(fn, args, __VA_ARGS__))
#if !defined CV_DISABLE_OPTIMIZATION && defined CV_ENABLE_INTRINSICS && defined CV_CPU_COMPILE_NEON_FP16
# define CV_TRY_NEON_FP16 1
# define CV_CPU_FORCE_NEON_FP16 1
# define CV_CPU_HAS_SUPPORT_NEON_FP16 1
# define CV_CPU_CALL_NEON_FP16(fn, args) return (cpu_baseline::fn args)
# define CV_CPU_CALL_NEON_FP16_(fn, args) return (opt_NEON_FP16::fn args)
#elif !defined CV_DISABLE_OPTIMIZATION && defined CV_ENABLE_INTRINSICS && defined CV_CPU_DISPATCH_COMPILE_NEON_FP16
# define CV_TRY_NEON_FP16 1
# define CV_CPU_FORCE_NEON_FP16 0
# define CV_CPU_HAS_SUPPORT_NEON_FP16 (cv::checkHardwareSupport(CV_CPU_NEON_FP16))
# define CV_CPU_CALL_NEON_FP16(fn, args) if (CV_CPU_HAS_SUPPORT_NEON_FP16) return (opt_NEON_FP16::fn args)
# define CV_CPU_CALL_NEON_FP16_(fn, args) if (CV_CPU_HAS_SUPPORT_NEON_FP16) return (opt_NEON_FP16::fn args)
#else
# define CV_TRY_NEON_FP16 0
# define CV_CPU_FORCE_NEON_FP16 0
# define CV_CPU_HAS_SUPPORT_NEON_FP16 0
# define CV_CPU_CALL_NEON_FP16(fn, args)
# define CV_CPU_CALL_NEON_FP16_(fn, args)
#endif
#define __CV_CPU_DISPATCH_CHAIN_NEON_FP16(fn, args, mode, ...) CV_CPU_CALL_NEON_FP16(fn, args); __CV_EXPAND(__CV_CPU_DISPATCH_CHAIN_ ## mode(fn, args, __VA_ARGS__))
#if !defined CV_DISABLE_OPTIMIZATION && defined CV_ENABLE_INTRINSICS && defined CV_CPU_COMPILE_NEON_BF16
# define CV_TRY_NEON_BF16 1
# define CV_CPU_FORCE_NEON_BF16 1
# define CV_CPU_HAS_SUPPORT_NEON_BF16 1
# define CV_CPU_CALL_NEON_BF16(fn, args) return (cpu_baseline::fn args)
# define CV_CPU_CALL_NEON_BF16_(fn, args) return (opt_NEON_BF16::fn args)
#elif !defined CV_DISABLE_OPTIMIZATION && defined CV_ENABLE_INTRINSICS && defined CV_CPU_DISPATCH_COMPILE_NEON_BF16
# define CV_TRY_NEON_BF16 1
# define CV_CPU_FORCE_NEON_BF16 0
# define CV_CPU_HAS_SUPPORT_NEON_BF16 (cv::checkHardwareSupport(CV_CPU_NEON_BF16))
# define CV_CPU_CALL_NEON_BF16(fn, args) if (CV_CPU_HAS_SUPPORT_NEON_BF16) return (opt_NEON_BF16::fn args)
# define CV_CPU_CALL_NEON_BF16_(fn, args) if (CV_CPU_HAS_SUPPORT_NEON_BF16) return (opt_NEON_BF16::fn args)
#else
# define CV_TRY_NEON_BF16 0
# define CV_CPU_FORCE_NEON_BF16 0
# define CV_CPU_HAS_SUPPORT_NEON_BF16 0
# define CV_CPU_CALL_NEON_BF16(fn, args)
# define CV_CPU_CALL_NEON_BF16_(fn, args)
#endif
#define __CV_CPU_DISPATCH_CHAIN_NEON_BF16(fn, args, mode, ...) CV_CPU_CALL_NEON_BF16(fn, args); __CV_EXPAND(__CV_CPU_DISPATCH_CHAIN_ ## mode(fn, args, __VA_ARGS__))
#if !defined CV_DISABLE_OPTIMIZATION && defined CV_ENABLE_INTRINSICS && defined CV_CPU_COMPILE_MSA
# define CV_TRY_MSA 1
# define CV_CPU_FORCE_MSA 1
modules/core/include/opencv2/core/fast_math.hpp
+1 -1
View File
@@ -68,7 +68,7 @@
// nothing, intrinsics/asm code is not supported
#else
#if ((defined _MSC_VER && defined _M_X64) \
|| (defined __GNUC__ && defined __x86_64__ && defined __SSE2__)) \
|| (defined __GNUC__ && defined __SSE2__)) \
&& !defined(OPENCV_SKIP_INCLUDE_EMMINTRIN_H)
#include <emmintrin.h>
#endif
modules/core/include/opencv2/core/hal/intrin.hpp
-6
View File
@@ -246,12 +246,6 @@ using namespace CV_CPU_OPTIMIZATION_HAL_NAMESPACE;
#include "opencv2/core/hal/intrin_lsx.hpp"
#elif CV_LASX
#if !defined(CV_FORCE_SIMD128_CPP)
#define CV_FORCE_SIMD128_CPP 1
#endif
#include "opencv2/core/hal/intrin_cpp.hpp"
#else
#include "opencv2/core/hal/intrin_cpp.hpp"
modules/core/include/opencv2/core/hal/intrin_lasx.hpp
+24 -23
View File
@@ -1419,20 +1419,6 @@ inline v_uint32x8 v_popcount(const v_int32x8& a)
inline v_uint64x4 v_popcount(const v_int64x4& a)
{ return v_popcount(v_reinterpret_as_u64(a)); }
/** Mask **/
#define OPENCV_HAL_IMPL_REINTERPRET_INT(ft, tt) \
inline tt reinterpret_int(ft x) { union { ft l; tt i; } v; v.l = x; return v.i; }
OPENCV_HAL_IMPL_REINTERPRET_INT(uchar, schar)
OPENCV_HAL_IMPL_REINTERPRET_INT(schar, schar)
OPENCV_HAL_IMPL_REINTERPRET_INT(ushort, short)
OPENCV_HAL_IMPL_REINTERPRET_INT(short, short)
OPENCV_HAL_IMPL_REINTERPRET_INT(unsigned, int)
OPENCV_HAL_IMPL_REINTERPRET_INT(int, int)
OPENCV_HAL_IMPL_REINTERPRET_INT(float, int)
OPENCV_HAL_IMPL_REINTERPRET_INT(uint64, int64)
OPENCV_HAL_IMPL_REINTERPRET_INT(int64, int64)
OPENCV_HAL_IMPL_REINTERPRET_INT(double, int64)
inline int v_signmask(const v_int8x32& a)
{
__m256i result = __lasx_xvmskltz_b(a.val);
@@ -2151,7 +2137,8 @@ template<int n> inline
void v_rshr_pack_store(uchar* ptr, const v_uint16x16& a)
{
__m256i res = __lasx_xvssrlrni_bu_h(a.val, a.val, n);
__lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0);
__lasx_xvstelm_d(res, ptr, 0, 0);
__lasx_xvstelm_d(res, ptr, 8, 2);
}
template<int n> inline
@@ -2165,7 +2152,8 @@ template<int n> inline
void v_rshr_pack_u_store(uchar* ptr, const v_int16x16& a)
{
__m256i res = __lasx_xvssrarni_bu_h(a.val, a.val, n);
__lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0);
__lasx_xvstelm_d(res, ptr, 0, 0);
__lasx_xvstelm_d(res, ptr, 8, 2);
}
template<int n> inline
@@ -2179,7 +2167,8 @@ template<int n> inline
void v_rshr_pack_store(schar* ptr, const v_int16x16& a)
{
__m256i res = __lasx_xvssrarni_b_h(a.val, a.val, n);
__lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0);
__lasx_xvstelm_d(res, ptr, 0, 0);
__lasx_xvstelm_d(res, ptr, 8, 2);
}
// 32
@@ -2198,7 +2187,8 @@ inline void v_pack_store(short* ptr, const v_int32x8& a)
inline void v_pack_store(ushort* ptr, const v_uint32x8& a)
{
__m256i res = __lasx_xvssrlrni_hu_w(a.val, a.val, 0);
__lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0);
__lasx_xvstelm_d(res, ptr, 0, 0);
__lasx_xvstelm_d(res, ptr, 8, 2);
}
inline void v_pack_u_store(ushort* ptr, const v_int32x8& a)
@@ -2212,7 +2202,8 @@ template<int n> inline
void v_rshr_pack_store(ushort* ptr, const v_uint32x8& a)
{
__m256i res = __lasx_xvssrlrni_hu_w(a.val, a.val, n);
__lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0);
__lasx_xvstelm_d(res, ptr, 0, 0);
__lasx_xvstelm_d(res, ptr, 8, 2);
}
template<int n> inline
@@ -2223,7 +2214,8 @@ template<int n> inline
void v_rshr_pack_u_store(ushort* ptr, const v_int32x8& a)
{
__m256i res = __lasx_xvssrarni_hu_w(a.val, a.val, n);
__lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0);
__lasx_xvstelm_d(res, ptr, 0, 0);
__lasx_xvstelm_d(res, ptr, 8, 2);
}
template<int n> inline
@@ -2234,7 +2226,8 @@ template<int n> inline
void v_rshr_pack_store(short* ptr, const v_int32x8& a)
{
__m256i res = __lasx_xvssrarni_h_w(a.val, a.val, n);
__lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0);
__lasx_xvstelm_d(res, ptr, 0, 0);
__lasx_xvstelm_d(res, ptr, 8, 2);
}
// 64
@@ -2263,7 +2256,11 @@ v_uint32x8 v_rshr_pack(const v_uint64x4& a, const v_uint64x4& b)
template<int n> inline
void v_rshr_pack_store(unsigned* ptr, const v_uint64x4& a)
{ __lsx_vst(_v256_shuffle_odd_64(__lasx_xvsrlrni_w_d(a.val, a.val, n)), ptr, 0); }
{
__m256i res = __lasx_xvsrlrni_w_d(a.val, a.val, n);
__lasx_xvstelm_d(res, ptr, 0, 0);
__lasx_xvstelm_d(res, ptr, 8, 2);
}
template<int n> inline
v_int32x8 v_rshr_pack(const v_int64x4& a, const v_int64x4& b)
@@ -2271,7 +2268,11 @@ v_int32x8 v_rshr_pack(const v_int64x4& a, const v_int64x4& b)
template<int n> inline
void v_rshr_pack_store(int* ptr, const v_int64x4& a)
{ __lsx_vst(_v256_shuffle_odd_64(__lasx_xvsrarni_w_d(a.val, a.val, n)), ptr, 0); }
{
__m256i res = __lasx_xvsrarni_w_d(a.val, a.val, n);
__lasx_xvstelm_d(res, ptr, 0, 0);
__lasx_xvstelm_d(res, ptr, 8, 2);
}
// pack boolean
inline v_uint8x32 v_pack_b(const v_uint16x16& a, const v_uint16x16& b)
modules/core/include/opencv2/core/hal/intrin_neon.hpp
+3 -5
View File
@@ -2007,11 +2007,9 @@ inline v_int32x4 v_round(const v_float32x4& a)
#else
inline v_int32x4 v_round(const v_float32x4& a)
{
static const int32x4_t v_sign = vdupq_n_s32(1 << 31),
v_05 = vreinterpretq_s32_f32(vdupq_n_f32(0.5f));
int32x4_t v_addition = vorrq_s32(v_05, vandq_s32(v_sign, vreinterpretq_s32_f32(a.val)));
return v_int32x4(vcvtq_s32_f32(vaddq_f32(a.val, vreinterpretq_f32_s32(v_addition))));
// See https://github.com/opencv/opencv/pull/24271#issuecomment-1867318007
float32x4_t delta = vdupq_n_f32(12582912.0f);
return v_int32x4(vcvtq_s32_f32(vsubq_f32(vaddq_f32(a.val, delta), delta)));
}
#endif
inline v_int32x4 v_floor(const v_float32x4& a)
modules/core/include/opencv2/core/hal/intrin_rvv.hpp
+4 -1
View File
@@ -32,6 +32,8 @@
namespace cv
{
//! @cond IGNORED
CV_CPU_OPTIMIZATION_HAL_NAMESPACE_BEGIN
#define CV_SIMD128 1
@@ -3336,7 +3338,8 @@ inline void v_cleanup() {}
CV_CPU_OPTIMIZATION_HAL_NAMESPACE_END
//! @endcond
}
} // namespace cv
#endif
modules/core/include/opencv2/core/hal/intrin_rvv071.hpp
+1 -1
View File
@@ -2536,5 +2536,5 @@ CV_CPU_OPTIMIZATION_HAL_NAMESPACE_END
//! @endcond
}
} // namespace cv
#endif
modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp
+61 -30
View File
@@ -38,6 +38,9 @@
namespace cv
{
//! @cond IGNORED
CV_CPU_OPTIMIZATION_HAL_NAMESPACE_BEGIN
#define CV_SIMD_SCALABLE 1
@@ -445,29 +448,7 @@ OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_float64, vfloat64m1_t, double, VTraits<v_floa
#define OPENCV_HAL_IMPL_RVV_LUT(_Tpvec, _Tp, suffix) \
inline _Tpvec v_lut(const _Tp* tab, const int* idx) \
{ \
vuint32##suffix##_t vidx = vmul(vreinterpret_u32##suffix(vle32_v_i32##suffix(idx, VTraits<_Tpvec>::vlanes())), sizeof(_Tp), VTraits<_Tpvec>::vlanes()); \
return vloxei32(tab, vidx, VTraits<_Tpvec>::vlanes()); \
} \
inline _Tpvec v_lut_pairs(const _Tp* tab, const int* idx) \
{ \
std::vector<uint> idx_; \
for (int i = 0; i < VTraits<v_int16>::vlanes(); ++i) { \
idx_.push_back(idx[i]); \
idx_.push_back(idx[i]+1); \
} \
vuint32##suffix##_t vidx = vmul(vle32_v_u32##suffix(idx_.data(), VTraits<_Tpvec>::vlanes()), sizeof(_Tp), VTraits<_Tpvec>::vlanes()); \
return vloxei32(tab, vidx, VTraits<_Tpvec>::vlanes()); \
} \
inline _Tpvec v_lut_quads(const _Tp* tab, const int* idx) \
{ \
std::vector<uint> idx_; \
for (int i = 0; i < VTraits<v_int32>::vlanes(); ++i) { \
idx_.push_back(idx[i]); \
idx_.push_back(idx[i]+1); \
idx_.push_back(idx[i]+2); \
idx_.push_back(idx[i]+3); \
} \
vuint32##suffix##_t vidx = vmul(vle32_v_u32##suffix(idx_.data(), VTraits<_Tpvec>::vlanes()), sizeof(_Tp), VTraits<_Tpvec>::vlanes()); \
auto vidx = vmul(vreinterpret_u32##suffix(vle32_v_i32##suffix(idx, VTraits<_Tpvec>::vlanes())), sizeof(_Tp), VTraits<_Tpvec>::vlanes()); \
return vloxei32(tab, vidx, VTraits<_Tpvec>::vlanes()); \
}
OPENCV_HAL_IMPL_RVV_LUT(v_int8, schar, m4)
@@ -479,6 +460,55 @@ OPENCV_HAL_IMPL_RVV_LUT(v_float32, float, m1)
OPENCV_HAL_IMPL_RVV_LUT(v_float64, double, mf2)
#endif
#define OPENCV_HAL_IMPL_RVV_LUT_PAIRS(_Tpvec, _Tp, suffix1, suffix2, v_trunc) \
inline _Tpvec v_lut_pairs(const _Tp* tab, const int* idx) \
{ \
auto v0 = vle32_v_u32##suffix1((unsigned*)idx, VTraits<_Tpvec>::vlanes()/2); \
auto v1 = vadd(v0, 1, VTraits<_Tpvec>::vlanes()/2); \
auto w0 = vwcvtu_x(v0, VTraits<_Tpvec>::vlanes()/2); \
auto w1 = vwcvtu_x(v1, VTraits<_Tpvec>::vlanes()/2); \
auto sh1 = vslide1up(v_trunc(vreinterpret_u32##suffix2(w1)),0, VTraits<_Tpvec>::vlanes()); \
auto vid = vor(sh1, v_trunc(vreinterpret_u32##suffix2(w0)), VTraits<_Tpvec>::vlanes()); \
auto vidx = vmul(vid, sizeof(_Tp), VTraits<_Tpvec>::vlanes()); \
return vloxei32(tab, vidx, VTraits<_Tpvec>::vlanes()); \
}
OPENCV_HAL_IMPL_RVV_LUT_PAIRS(v_int8, schar, m2, m4, OPENCV_HAL_NOP)
OPENCV_HAL_IMPL_RVV_LUT_PAIRS(v_int16, short, m1, m2, OPENCV_HAL_NOP)
OPENCV_HAL_IMPL_RVV_LUT_PAIRS(v_int32, int, mf2, m1, OPENCV_HAL_NOP)
OPENCV_HAL_IMPL_RVV_LUT_PAIRS(v_float32, float, mf2, m1, OPENCV_HAL_NOP)
OPENCV_HAL_IMPL_RVV_LUT_PAIRS(v_int64, int64_t, mf2, m1, vlmul_trunc_u32mf2)
#if CV_SIMD_SCALABLE_64F
OPENCV_HAL_IMPL_RVV_LUT_PAIRS(v_float64, double, mf2, m1, vlmul_trunc_u32mf2)
#endif
#define OPENCV_HAL_IMPL_RVV_LUT_QUADS(_Tpvec, _Tp, suffix0, suffix1, suffix2, v_trunc) \
inline _Tpvec v_lut_quads(const _Tp* tab, const int* idx) \
{ \
auto v0 = vle32_v_u32##suffix0((unsigned*)idx, VTraits<_Tpvec>::vlanes()/4); \
auto v1 = vadd(v0, 1, VTraits<_Tpvec>::vlanes()/4); \
auto v2 = vadd(v0, 2, VTraits<_Tpvec>::vlanes()/4); \
auto v3 = vadd(v0, 3, VTraits<_Tpvec>::vlanes()/4); \
auto w0 = vwcvtu_x(v0, VTraits<_Tpvec>::vlanes()/4); \
auto w1 = vwcvtu_x(v1, VTraits<_Tpvec>::vlanes()/4); \
auto w2 = vwcvtu_x(v2, VTraits<_Tpvec>::vlanes()/4); \
auto w3 = vwcvtu_x(v3, VTraits<_Tpvec>::vlanes()/4); \
auto sh2 = vslide1up(vreinterpret_u32##suffix1(w2),0, VTraits<_Tpvec>::vlanes()/2); \
auto sh3 = vslide1up(vreinterpret_u32##suffix1(w3),0, VTraits<_Tpvec>::vlanes()/2); \
auto vid0 = vor(sh2, vreinterpret_u32##suffix1(w0), VTraits<_Tpvec>::vlanes()/2); \
auto vid1 = vor(sh3, vreinterpret_u32##suffix1(w1), VTraits<_Tpvec>::vlanes()/2); \
auto wid0 = vwcvtu_x(v_trunc(vid0), VTraits<_Tpvec>::vlanes()/2); \
auto wid1 = vwcvtu_x(v_trunc(vid1), VTraits<_Tpvec>::vlanes()/2); \
auto shwid1 = vslide1up(vreinterpret_u32##suffix2(wid1),0, VTraits<_Tpvec>::vlanes()); \
auto vid = vor(shwid1, vreinterpret_u32##suffix2(wid0), VTraits<_Tpvec>::vlanes()); \
auto vidx = vmul(vid, sizeof(_Tp), VTraits<_Tpvec>::vlanes()); \
return vloxei32(tab, vidx, VTraits<_Tpvec>::vlanes()); \
}
OPENCV_HAL_IMPL_RVV_LUT_QUADS(v_int8, schar, m1, m2, m4, OPENCV_HAL_NOP)
OPENCV_HAL_IMPL_RVV_LUT_QUADS(v_int16, short, mf2 , m1, m2, OPENCV_HAL_NOP)
OPENCV_HAL_IMPL_RVV_LUT_QUADS(v_int32, int, mf2, m1, m1, vlmul_trunc_u32mf2)
OPENCV_HAL_IMPL_RVV_LUT_QUADS(v_float32, float, mf2, m1, m1, vlmul_trunc_u32mf2)
#define OPENCV_HAL_IMPL_RVV_LUT_VEC(_Tpvec, _Tp) \
inline _Tpvec v_lut(const _Tp* tab, const v_int32& vidx) \
{ \
@@ -509,7 +539,6 @@ inline v_uint32 v_lut_pairs(const unsigned* tab, const int* idx) { return v_rein
inline v_uint32 v_lut_quads(const unsigned* tab, const int* idx) { return v_reinterpret_as_u32(v_lut_quads((int*)tab, idx)); }
inline v_uint64 v_lut(const uint64* tab, const int* idx) { return v_reinterpret_as_u64(v_lut((const int64_t *)tab, idx)); }
inline v_uint64 v_lut_pairs(const uint64* tab, const int* idx) { return v_reinterpret_as_u64(v_lut_pairs((const int64_t *)tab, idx)); }
inline v_uint64 v_lut_quads(const uint64* tab, const int* idx) { return v_reinterpret_as_u64(v_lut_quads((const int64_t*)tab, idx)); }
////////////// Pack boolean ////////////////////
inline v_uint8 v_pack_b(const v_uint16& a, const v_uint16& b)
@@ -1390,23 +1419,23 @@ OPENCV_HAL_IMPL_RVV_REVERSE(v_float64, 64)
#define OPENCV_HAL_IMPL_RVV_EXPAND(_Tp, _Tpwvec, _Tpwvec_m2, _Tpvec, width, suffix, suffix2, cvt) \
inline void v_expand(const _Tpvec& a, _Tpwvec& b0, _Tpwvec& b1) \
{ \
_Tpwvec_m2 temp = cvt(a, vsetvlmax_e##width##m1()); \
_Tpwvec_m2 temp = cvt(a, VTraits<_Tpvec>::vlanes()); \
b0 = vget_##suffix##m1(temp, 0); \
b1 = vget_##suffix##m1(temp, 1); \
} \
inline _Tpwvec v_expand_low(const _Tpvec& a) \
{ \
_Tpwvec_m2 temp = cvt(a, vsetvlmax_e##width##m1()); \
_Tpwvec_m2 temp = cvt(a, VTraits<_Tpvec>::vlanes()); \
return vget_##suffix##m1(temp, 0); \
} \
inline _Tpwvec v_expand_high(const _Tpvec& a) \
{ \
_Tpwvec_m2 temp = cvt(a, vsetvlmax_e##width##m1()); \
_Tpwvec_m2 temp = cvt(a, VTraits<_Tpvec>::vlanes()); \
return vget_##suffix##m1(temp, 1); \
} \
inline _Tpwvec v_load_expand(const _Tp* ptr) \
{ \
return cvt(vle##width##_v_##suffix2##mf2(ptr, vsetvlmax_e##width##m1()), vsetvlmax_e##width##m1()); \
return cvt(vle##width##_v_##suffix2##mf2(ptr, VTraits<_Tpvec>::vlanes()), VTraits<_Tpvec>::vlanes()); \
}
OPENCV_HAL_IMPL_RVV_EXPAND(uchar, v_uint16, vuint16m2_t, v_uint8, 8, u16, u8, vwcvtu_x)
@@ -1759,8 +1788,8 @@ inline int v_scan_forward(const v_float64& a)
// mask: {0,0,0,1, ...} -> {T,T,T,F, ...}
#define OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(_Tpvec, v_trunc) \
inline _Tpvec v_pack_triplets(const _Tpvec& vec) { \
size_t vl = vsetvlmax_e8m1(); \
vuint32m1_t one = vmv_v_x_u32m1(1, vl/4); \
size_t vl = __cv_rvv_e8m1_nlanes; \
vuint32m1_t one = vmv_v_x_u32m1(1, __cv_rvv_e32m1_nlanes); \
vuint8m1_t zero = vmv_v_x_u8m1(0, vl); \
vuint8m1_t mask = vreinterpret_u8m1(one); \
return vcompress(vmseq(v_trunc(vslideup(zero, mask, 3, vl)), 0, vl), vec, vec, VTraits<_Tpvec>::vlanes()); \
@@ -2126,6 +2155,8 @@ inline void v_cleanup() {}
CV_CPU_OPTIMIZATION_HAL_NAMESPACE_END
//! @endcond
} //namespace cv
#endif //OPENCV_HAL_INTRIN_RVV_SCALABLE_HPP
modules/core/include/opencv2/core/matx.hpp
+1 -1
View File
@@ -781,7 +781,7 @@ inline Matx<_Tp, m, n>::operator Matx<T2, m, n>() const
template<typename _Tp, int m, int n> template<int m1, int n1> inline
Matx<_Tp, m1, n1> Matx<_Tp, m, n>::reshape() const
{
CV_StaticAssert(m1*n1 == m*n, "Input and destnarion matrices must have the same number of elements");
CV_StaticAssert(m1*n1 == m*n, "Input and destination matrices must have the same number of elements");
return (const Matx<_Tp, m1, n1>&)*this;
}
modules/core/perf/perf_arithm.cpp
+1 -2
View File
@@ -1,6 +1,6 @@
#include "perf_precomp.hpp"
#include "opencv2/core/softfloat.hpp"
#include <numeric>
#include "opencv2/core/softfloat.hpp"
namespace opencv_test
{
@@ -452,7 +452,6 @@ INSTANTIATE_TEST_CASE_P(/*nothing*/ , BinaryOpTest,
)
);
///////////// PatchNaNs ////////////////////////
template<typename _Tp>
modules/core/src/arithm.cpp
+1 -1
View File
@@ -1646,7 +1646,7 @@ static bool ocl_inRange( InputArray _src, InputArray _lowerb,
if (kercn % cn != 0)
kercn = cn;
int colsPerWI = kercn / cn;
String opts = format("%s-D cn=%d -D srcT=%s -D srcT1=%s -D dstT=%s -D kercn=%d -D depth=%d%s -D colsPerWI=%d",
String opts = format("%s-D CN=%d -D SRC_T=%s -D SRC_T1=%s -D DST_T=%s -D KERCN=%d -D DEPTH=%d%s -D COLS_PER_WI=%d",
haveScalar ? "-D HAVE_SCALAR " : "", cn, ocl::typeToStr(CV_MAKE_TYPE(sdepth, kercn)),
ocl::typeToStr(sdepth), ocl::typeToStr(CV_8UC(colsPerWI)), kercn, sdepth,
doubleSupport ? " -D DOUBLE_SUPPORT" : "", colsPerWI);
modules/core/src/arithm.simd.hpp
+1 -1
View File
@@ -861,7 +861,7 @@ static void cmp_loop_nosimd(const double* src1, size_t step1, const double* src2
}
// todo: try to avoid define dispatcher functions using macros with these such cases
DEFINE_SIMD_ALL(cmp)
DEFINE_SIMD_ALL(cmp, void)
//=========================================================================
// scaling helpers for single and dual source
modules/core/src/opencl/inrange.cl
+24 -24
View File
@@ -52,7 +52,7 @@
__kernel void inrange(__global const uchar * src1ptr, int src1_step, int src1_offset,
__global uchar * dstptr, int dst_step, int dst_offset, int dst_rows, int dst_cols,
#ifdef HAVE_SCALAR
__global const srcT1 * src2, __global const srcT1 * src3,
__global const SRC_T1 * src2, __global const SRC_T1 * src3,
#else
__global const uchar * src2ptr, int src2_step, int src2_offset,
__global const uchar * src3ptr, int src3_step, int src3_offset,
@@ -64,56 +64,56 @@ __kernel void inrange(__global const uchar * src1ptr, int src1_step, int src1_of
if (x < dst_cols)
{
int src1_index = mad24(y0, src1_step, mad24(x, (int)sizeof(srcT1) * kercn, src1_offset));
int dst_index = mad24(y0, dst_step, mad24(x, colsPerWI, dst_offset));
int src1_index = mad24(y0, src1_step, mad24(x, (int)sizeof(SRC_T1) * KERCN, src1_offset));
int dst_index = mad24(y0, dst_step, mad24(x, COLS_PER_WI, dst_offset));
#ifndef HAVE_SCALAR
int src2_index = mad24(y0, src2_step, mad24(x, (int)sizeof(srcT1) * kercn, src2_offset));
int src3_index = mad24(y0, src3_step, mad24(x, (int)sizeof(srcT1) * kercn, src3_offset));
int src2_index = mad24(y0, src2_step, mad24(x, (int)sizeof(SRC_T1) * KERCN, src2_offset));
int src3_index = mad24(y0, src3_step, mad24(x, (int)sizeof(SRC_T1) * KERCN, src3_offset));
#endif
for (int y = y0, y1 = min(dst_rows, y0 + rowsPerWI); y < y1; ++y, src1_index += src1_step, dst_index += dst_step)
{
#if kercn >= cn && kercn == 4 && depth <= 4 && !defined HAVE_SCALAR
srcT src1 = *(__global const srcT *)(src1ptr + src1_index);
srcT src2 = *(__global const srcT *)(src2ptr + src2_index);
srcT src3 = *(__global const srcT *)(src3ptr + src3_index);
__global dstT * dst = (__global dstT *)(dstptr + dst_index);
#if cn == 1
dst[0] = src2 > src1 || src3 < src1 ? (dstT)(0) : (dstT)(255);
#elif cn == 2
dst[0] = (dstT)(src2.xy > src1.xy || src3.xy < src1.xy ||
src2.zw > src1.zw || src3.zw < src1.zw ? (dstT)(0) : (dstT)(255);
#elif cn == 4
dst[0] = (dstT)(src2.x > src1.x || src3.x < src1.x ||
#if KERCN >= CN && KERCN == 4 && DEPTH <= 4 && !defined HAVE_SCALAR
SRC_T src1 = *(__global const SRC_T *)(src1ptr + src1_index);
SRC_T src2 = *(__global const SRC_T *)(src2ptr + src2_index);
SRC_T src3 = *(__global const SRC_T *)(src3ptr + src3_index);
__global DST_T * dst = (__global DST_T *)(dstptr + dst_index);
#if CN == 1
dst[0] = src2 > src1 || src3 < src1 ? (DST_T)(0) : (DST_T)(255);
#elif CN == 2
dst[0] = (DST_T)(src2.xy > src1.xy || src3.xy < src1.xy ||
src2.zw > src1.zw || src3.zw < src1.zw ? (DST_T)(0) : (DST_T)(255);
#elif CN == 4
dst[0] = (DST_T)(src2.x > src1.x || src3.x < src1.x ||
src2.y > src1.y || src3.y < src1.y ||
src2.z > src1.z || src3.z < src1.z ||
src2.w > src1.w || src3.w < src1.w ? 0 : 255);
#endif
#else
__global const srcT1 * src1 = (__global const srcT1 *)(src1ptr + src1_index);
__global const SRC_T1 * src1 = (__global const SRC_T1 *)(src1ptr + src1_index);
__global uchar * dst = dstptr + dst_index;
#ifndef HAVE_SCALAR
__global const srcT1 * src2 = (__global const srcT1 *)(src2ptr + src2_index);
__global const srcT1 * src3 = (__global const srcT1 *)(src3ptr + src3_index);
__global const SRC_T1 * src2 = (__global const SRC_T1 *)(src2ptr + src2_index);
__global const SRC_T1 * src3 = (__global const SRC_T1 *)(src3ptr + src3_index);
#endif
#pragma unroll
for (int px = 0; px < colsPerWI; ++px, src1 += cn
for (int px = 0; px < COLS_PER_WI; ++px, src1 += CN
#ifndef HAVE_SCALAR
, src2 += cn, src3 += cn
, src2 += CN, src3 += CN
#endif
)
{
dst[px] = 255;
for (int c = 0; c < cn; ++c)
for (int c = 0; c < CN; ++c)
if (src2[c] > src1[c] || src3[c] < src1[c])
{
dst[px] = 0;
break;
}
}
#endif // kercn >= cn
#endif // KERCN >= CN
#ifndef HAVE_SCALAR
src2_index += src2_step;
src3_index += src3_step;
modules/core/src/system.cpp
+10 -5
View File
@@ -154,6 +154,12 @@ void* allocSingletonNewBuffer(size_t size) { return malloc(size); }
# endif
#endif
#if defined __loongarch64
#include "sys/auxv.h"
#define LA_HWCAP_LSX (1<<4)
#define LA_HWCAP_LASX (1<<5)
#endif
#if defined _WIN32 || defined WINCE
#ifndef _WIN32_WINNT // This is needed for the declaration of TryEnterCriticalSection in winbase.h with Visual Studio 2005 (and older?)
#define _WIN32_WINNT 0x0400 // http://msdn.microsoft.com/en-us/library/ms686857(VS.85).aspx
@@ -704,12 +710,11 @@ struct HWFeatures
have[CV_CPU_RVV] = true;
#endif
#if defined __loongarch_sx
have[CV_CPU_LSX] = true;
#endif
#if defined __loongarch64 && defined __linux__
int flag = (int)getauxval(AT_HWCAP);
#if defined __loongarch_asx
have[CV_CPU_LASX] = true;
have[CV_CPU_LSX] = (flag & LA_HWCAP_LSX) != 0;
have[CV_CPU_LASX] = (flag & LA_HWCAP_LASX) != 0;
#endif
bool skip_baseline_check = false;
modules/core/test/ocl/test_image2d.cpp
+1 -1
View File
@@ -83,7 +83,7 @@ TEST(Image2D, turnOffOpenCL)
}
else
std::cout << "CV_8UC1 is not supported for OpenCL images. Test skipped." << std::endl;
// reset state to the previous one
cv::ocl::setUseOpenCL(useOCL);
}
modules/core/test/test_operations.cpp
+50
View File
@@ -1571,4 +1571,54 @@ TEST(Core_Arithm, scalar_handling_19599) // https://github.com/opencv/opencv/is
EXPECT_EQ(1, c.rows);
}
// https://github.com/opencv/opencv/issues/24163
typedef tuple<perf::MatDepth,int,int,int> Arith_Regression24163Param;
typedef testing::TestWithParam<Arith_Regression24163Param> Core_Arith_Regression24163;
#if defined __riscv
TEST_P(Core_Arith_Regression24163, DISABLED_test_for_ties_to_even)
#else
TEST_P(Core_Arith_Regression24163, test_for_ties_to_even)
#endif
{
const int matDepth = get<0>(GetParam());
const int matHeight= get<1>(GetParam());
const int matWidth = 3; // Fixed
const int alpha = get<2>(GetParam());
const int beta = get<3>(GetParam());
// If alpha and/or beta are negative, and matDepth is unsigned, test is passed.
if( ( (alpha < 0) || (beta < 0) )
&&
( (matDepth != CV_8S) && (matDepth != CV_16S) && (matDepth != CV_32S) ) )
{
throw SkipTestException( cv::format("Test is skipped(matDepth is not signed, alpha = %d, beta = %d)", alpha, beta) );
}
const int matType = CV_MAKE_TYPE(matDepth, 1);
const Size matSize(matWidth, matHeight);
const Mat src1(matSize, matType, Scalar(alpha,alpha,alpha,alpha));
const Mat src2(matSize, matType, Scalar(beta, beta, beta, beta));
const Mat result = ( src1 + src2 ) / 2;
// Expected that default is FE_TONEAREST(Ties to Even).
const int mean = lrint( static_cast<double>(alpha + beta) / 2.0 );
const Mat expected(matSize, matType, Scalar(mean,mean,mean,mean));
// Compare result and extected.
ASSERT_EQ(expected.size(), result.size());
EXPECT_EQ(0, cvtest::norm(expected, result, NORM_INF)) <<
"result=" << std::endl << result << std::endl <<
"expected=" << std::endl << expected;
}
INSTANTIATE_TEST_CASE_P(/* */, Core_Arith_Regression24163,
testing::Combine(
testing::Values(perf::MatDepth(CV_8U), CV_8S, CV_16U, CV_16S, CV_32S), // MatType
testing::Values( 3, 4, 5, 6), // MatHeight
testing::Values(-2,-1, 0, 1, 2), // src1
testing::Values( -1, 0, 1 ) // src2
)
);
}} // namespace
modules/dnn/CMakeLists.txt
+8 -2
View File
@@ -6,9 +6,9 @@ set(the_description "Deep neural network module. It allows to load models from d
ocv_add_dispatched_file_force_all("layers/layers_common" AVX AVX2 AVX512_SKX RVV LASX)
ocv_add_dispatched_file_force_all("int8layers/layers_common" AVX2 AVX512_SKX LASX)
ocv_add_dispatched_file_force_all("layers/cpu_kernels/conv_block" AVX AVX2)
ocv_add_dispatched_file_force_all("layers/cpu_kernels/conv_block" AVX AVX2 NEON NEON_FP16)
ocv_add_dispatched_file_force_all("layers/cpu_kernels/conv_depthwise" AVX AVX2 RVV LASX)
ocv_add_dispatched_file_force_all("layers/cpu_kernels/conv_winograd_f63" AVX AVX2)
ocv_add_dispatched_file_force_all("layers/cpu_kernels/conv_winograd_f63" AVX AVX2 NEON_FP16)
ocv_add_dispatched_file_force_all("layers/cpu_kernels/fast_gemm_kernels" AVX AVX2 NEON LASX)
ocv_add_module(dnn opencv_core opencv_imgproc WRAP python java objc js)
@@ -164,6 +164,12 @@ if(OPENCV_DNN_CUDA AND HAVE_CUDA AND HAVE_CUBLAS AND HAVE_CUDNN)
endif()
endforeach()
unset(CC_LIST)
if(ENABLE_CUDA_FIRST_CLASS_LANGUAGE)
list(APPEND libs ${CUDNN_LIBRARIES} CUDA::cublas${CUDA_LIB_EXT})
if(NOT CUDA_VERSION VERSION_LESS 10.1)
list(APPEND libs CUDA::cublasLt${CUDA_LIB_EXT})
endif()
endif()
else()
set(sources_options ${sources_options} EXCLUDE_CUDA)
endif()
modules/dnn/include/opencv2/dnn/all_layers.hpp
+19 -2
View File
@@ -291,7 +291,7 @@ CV__DNN_INLINE_NS_BEGIN
static Ptr<BaseConvolutionLayer> create(const LayerParams& params);
bool fusedActivation = false;
bool fusedAdd = false;
bool useWinograd = false; // Flag whether to use Winograd to speed up 3x3 convolution.
bool useWinograd = true; // Flag whether to use Winograd to speed up 3x3 convolution.
};
class CV_EXPORTS ConvolutionLayerInt8 : public BaseConvolutionLayer
@@ -303,7 +303,7 @@ CV__DNN_INLINE_NS_BEGIN
// quantization type flag. The perChannel default is true, that means it contains the parameters
// of per-Channel quantization. Otherwise, that means this layer contains per-Tensor quantized parameters.
bool per_channel;
bool useWinograd = true; // Flag whether to use Winograd to speed up 3x3 convolution.
bool useWinograd = false; // Flag whether to use Winograd to speed up 3x3 convolution.
static Ptr<BaseConvolutionLayer> create(const LayerParams& params);
};
@@ -1160,12 +1160,29 @@ CV__DNN_INLINE_NS_BEGIN
static Ptr<GemmLayer> create(const LayerParams& params);
};
class CV_EXPORTS MatMulLayer : public Layer {
public:
static Ptr<MatMulLayer> create(const LayerParams &params);
};
class CV_EXPORTS ExpandLayer : public Layer
{
public:
static Ptr<ExpandLayer> create(const LayerParams &params);
};
class CV_EXPORTS InstanceNormLayer : public Layer {
public:
float epsilon;
static Ptr<InstanceNormLayer> create(const LayerParams &params);
};
class CV_EXPORTS AttentionLayer : public Layer {
public:
static Ptr<AttentionLayer> create(const LayerParams &params);
};
//! @}
//! @}
CV__DNN_INLINE_NS_END
modules/dnn/include/opencv2/dnn/dnn.hpp
+19 -1
View File
@@ -1127,7 +1127,7 @@ CV__DNN_INLINE_NS_BEGIN
CV_WRAP Image2BlobParams();
CV_WRAP Image2BlobParams(const Scalar& scalefactor, const Size& size = Size(), const Scalar& mean = Scalar(),
bool swapRB = false, int ddepth = CV_32F, DataLayout datalayout = DNN_LAYOUT_NCHW,
ImagePaddingMode mode = DNN_PMODE_NULL);
ImagePaddingMode mode = DNN_PMODE_NULL, Scalar borderValue = 0.0);
CV_PROP_RW Scalar scalefactor; //!< scalefactor multiplier for input image values.
CV_PROP_RW Size size; //!< Spatial size for output image.
@@ -1136,6 +1136,21 @@ CV__DNN_INLINE_NS_BEGIN
CV_PROP_RW int ddepth; //!< Depth of output blob. Choose CV_32F or CV_8U.
CV_PROP_RW DataLayout datalayout; //!< Order of output dimensions. Choose DNN_LAYOUT_NCHW or DNN_LAYOUT_NHWC.
CV_PROP_RW ImagePaddingMode paddingmode; //!< Image padding mode. @see ImagePaddingMode.
CV_PROP_RW Scalar borderValue; //!< Value used in padding mode for padding.
/** @brief Get rectangle coordinates in original image system from rectangle in blob coordinates.
* @param rBlob rect in blob coordinates.
* @param size original input image size.
* @returns rectangle in original image coordinates.
*/
CV_WRAP Rect blobRectToImageRect(const Rect &rBlob, const Size &size);
/** @brief Get rectangle coordinates in original image system from rectangle in blob coordinates.
* @param rBlob rect in blob coordinates.
* @param rImg result rect in image coordinates.
* @param size original input image size.
*/
CV_WRAP void blobRectsToImageRects(const std::vector<Rect> &rBlob, CV_OUT std::vector<Rect>& rImg, const Size& size);
};
/** @brief Creates 4-dimensional blob from image with given params.
@@ -1373,6 +1388,9 @@ CV__DNN_INLINE_NS_BEGIN
/// @sa Net::setPreferableTarget
CV_WRAP Model& setPreferableTarget(dnn::Target targetId);
/// @sa Net::enableWinograd
CV_WRAP Model& enableWinograd(bool useWinograd);
CV_DEPRECATED_EXTERNAL
operator Net&() const { return getNetwork_(); }
modules/dnn/include/opencv2/dnn/version.hpp
+1 -1
View File
@@ -6,7 +6,7 @@
#define OPENCV_DNN_VERSION_HPP
/// Use with major OpenCV version only.
#define OPENCV_DNN_API_VERSION 20230620
#define OPENCV_DNN_API_VERSION 20231225
#if !defined CV_DOXYGEN && !defined CV_STATIC_ANALYSIS && !defined CV_DNN_DONT_ADD_INLINE_NS
#define CV__DNN_INLINE_NS __CV_CAT(dnn5_v, OPENCV_DNN_API_VERSION)
modules/dnn/misc/python/test/test_dnn.py
+28
View File
@@ -127,6 +127,34 @@ class dnn_test(NewOpenCVTests):
targets = cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_OPENCV)
self.assertTrue(cv.dnn.DNN_TARGET_CPU in targets)
def test_blobRectsToImageRects(self):
paramNet = cv.dnn.Image2BlobParams()
paramNet.size = (226, 226)
paramNet.ddepth = cv.CV_32F
paramNet.mean = [0.485, 0.456, 0.406]
paramNet.scalefactor = [0.229, 0.224, 0.225]
paramNet.swapRB = False
paramNet.datalayout = cv.dnn.DNN_LAYOUT_NCHW
paramNet.paddingmode = cv.dnn.DNN_PMODE_LETTERBOX
rBlob = np.zeros(shape=(20, 4), dtype=np.int32)
rImg = paramNet.blobRectsToImageRects(rBlob, (356, 356))
self.assertTrue(type(rImg[0, 0])==np.int32)
self.assertTrue(rImg.shape==(20, 4))
def test_blobRectToImageRect(self):
paramNet = cv.dnn.Image2BlobParams()
paramNet.size = (226, 226)
paramNet.ddepth = cv.CV_32F
paramNet.mean = [0.485, 0.456, 0.406]
paramNet.scalefactor = [0.229, 0.224, 0.225]
paramNet.swapRB = False
paramNet.datalayout = cv.dnn.DNN_LAYOUT_NCHW
paramNet.paddingmode = cv.dnn.DNN_PMODE_LETTERBOX
rBlob = np.zeros(shape=(20, 4), dtype=np.int32)
rImg = paramNet.blobRectToImageRect((0, 0, 0, 0), (356, 356))
self.assertTrue(type(rImg[0])==int)
def test_blobFromImage(self):
np.random.seed(324)
modules/dnn/perf/perf_convolution.cpp
+672 -567
View File
File diff suppressed because it is too large Load Diff
modules/dnn/perf/perf_einsum.cpp
+109
View File
@@ -0,0 +1,109 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#include "perf_precomp.hpp"
namespace opencv_test {
struct EinsumParams {
int inputSize;
int outputSize;
std::string equation;
std::vector<MatShape> einsumInpShapes;
EinsumParams(std::string equation_, std::vector<MatShape> einsumInpShapes_ = std::vector<MatShape>())
{
inputSize = einsumInpShapes_.size();
equation = equation_;
einsumInpShapes = einsumInpShapes_;
}
};
static inline void PrintTo(const EinsumParams& params, ::std::ostream* os) {
(*os) << "Equation=" << params.equation << " ";
(*os) << "InputShape={";
for(int i = 0; i < params.einsumInpShapes.size(); i++)
{
(*os) << "{";
for(int j = 0; j < params.einsumInpShapes[i].size(); j++)
{
(*os) << params.einsumInpShapes[i][j] << ((j < params.einsumInpShapes[i].size() - 1) ? ", " : "");
}
(*os) << ((i < params.einsumInpShapes.size() - 1) ? "}, " : "}");
}
(*os) << "}";
}
// test cases
static const EinsumParams testEinsumConfigs[] = {
// TODO: Add tests with one input after ellips merge
{"ij, jk -> ik", {{2, 3}, {3, 2}}},
{"ij, jk -> ik", {{20, 30}, {30, 20}}},
{"ij, jk -> ik", {{113, 127}, {127, 113}}},
{"imkj, injs -> imnks", {{1, 4, 7, 9}, {1, 5, 9, 8}}},
{"imkj, injs -> imnks", {{1, 4, 70, 90}, {1, 5, 90, 80}}},
{"imkj, injs -> imnks", {{1, 4, 73, 91}, {1, 5, 91, 57}}},
{"ij -> i", {{30, 40}}},
{"ij -> i", {{113, 374}}},
{"...ij -> ...i", {{30, 40}}},
{"...ij -> ...i", {{113, 374}}},
{"...ij, ...jk -> ...ik", {{40, 50}, {50, 80}}},
{"...ij, ...jk -> ...ik", {{47, 51}, {51, 83}}},
};
class Layer_Einsum: public TestBaseWithParam<EinsumParams> {};
PERF_TEST_P_(Layer_Einsum, einsum) {
const EinsumParams& params = GetParam();
LayerParams lp;
lp.type = "Einsum";
lp.name = "testEinsum";
lp.set("equation", params.equation);
lp.set("inputSize", params.inputSize);
lp.set("outputSize", 1);
CV_CheckFalse(params.einsumInpShapes.empty(), "ERROR no inputs shapes provided");
for (int i = 0; i < params.einsumInpShapes.size(); i++) {
lp.set("inputShapes" + cv::format("%d", i), DictValue::arrayInt(params.einsumInpShapes[i].begin(), params.einsumInpShapes[i].size()));
}
Net net;
std::vector<Mat> inputs;
std::vector<std::string> input_names;
int id = net.addLayer(lp.name, lp.type, lp);
for (int i = 0; i < params.inputSize; ++i) {
// create inputs
inputs.emplace_back(Mat(params.einsumInpShapes[i].size(), params.einsumInpShapes[i].data(), CV_32FC1));
// connect each input to the layer
net.connect(0, i, id, i);
// create input names dynamically, assuming input naming follows a consistent pattern
input_names.emplace_back("input" + std::to_string(i + 1));
}
//warm up
std::vector<Mat> outputs;
net.setInputsNames(input_names);
for (int i = 0; i < input_names.size(); i++){
net.setInput(inputs[i], input_names[i]);
}
net.forward(outputs, "testEinsum");
TEST_CYCLE()
{
net.forward(outputs, "testEinsum");
}
SANITY_CHECK_NOTHING();
}
INSTANTIATE_TEST_CASE_P(/**/, Layer_Einsum, testing::ValuesIn(testEinsumConfigs));
}; //namespace
modules/dnn/perf/perf_gemm.cpp
+168 -2
View File
@@ -5,6 +5,8 @@
#include "perf_precomp.hpp"
#include <opencv2/dnn/shape_utils.hpp>
#include <numeric>
namespace opencv_test {
struct GemmParam_t {
@@ -71,6 +73,18 @@ static const GemmParam_t test_gemm_configs[] = {
*/
};
static const GemmParam_t test_matmul_configs[] = {
// vision transformer cases
{ {12, 197, 197}, {12, 197, 64} },
{ {12, 197, 64 }, {12, 64, 197} },
{ {12, 50, 64}, {12, 64, 50} },
{ {12, 50, 50}, {12, 50, 64} },
{ {16, 197, 197}, {16, 197, 64} },
{ {16, 197, 64 }, {16, 64, 197} },
{ {16, 50, 64}, {16, 64, 50} },
{ {16, 50, 50}, {16, 50, 64} },
};
struct GemmParamId
{
enum {
@@ -88,6 +102,21 @@ struct GemmParamId
}
};
struct MatMulParamId {
enum {
MATMUL_0 = 0,
MATMUL_LAST = sizeof(test_matmul_configs) / sizeof(test_matmul_configs[0])
};
int val_;
MatMulParamId(int val = 0) : val_(val) {}
operator int() const { return val_; }
static ::testing::internal::ParamGenerator<MatMulParamId> all() {
enum { NUM = (int)MATMUL_LAST };
MatMulParamId v_[NUM]; for (int i = 0; i < NUM; i++) { v_[i] = MatMulParamId(i); }
return ::testing::ValuesIn(v_, v_ + NUM);
}
};
static inline void PrintTo(const GemmParamId& v, std::ostream* os)
{
CV_Assert((int)v >= 0); CV_Assert((int)v < GemmParamId::GEMM_LAST);
@@ -138,7 +167,7 @@ PERF_TEST_P_(Gemm, gemm)
Mat A(static_cast<int>(a_shape.size()), a_shape.data(), CV_32F);
randu(A, -1.0f, 1.0f);
Mat B(static_cast<int>(b_shape.size()), b_shape.data(), CV_32F);
randu(A, -1.0f, 1.0f);
randu(B, -1.0f, 1.0f);
LayerParams lp;
lp.type = "Gemm";
@@ -197,7 +226,7 @@ PERF_TEST_P_(Gemm, innerproduct)
Mat A(static_cast<int>(a_shape.size()), a_shape.data(), CV_32F);
randu(A, -1.0f, 1.0f);
Mat B(static_cast<int>(b_shape.size()), b_shape.data(), CV_32F);
randu(A, -1.0f, 1.0f);
randu(B, -1.0f, 1.0f);
LayerParams lp;
lp.type = "InnerProduct";
@@ -241,9 +270,146 @@ PERF_TEST_P_(Gemm, innerproduct)
SANITY_CHECK_NOTHING();
}
static inline void PrintTo(const MatMulParamId& v, std::ostream* os)
{
CV_Assert((int)v >= 0); CV_Assert((int)v < MatMulParamId::MATMUL_LAST);
const GemmParam_t& p = test_matmul_configs[(int)v];
auto print_shape = [os](const std::vector<int>& shape, const std::string tag) {
if (shape.empty()) {
return ;
}
*os << tag << "=[";
for (size_t i = 0; i < shape.size(); ++i) {
if (i == shape.size() - 1) {
*os << shape[i] << "]";
break;
}
*os << shape[i] << ", ";
}
};
print_shape(p.a_shape, "A");
print_shape(p.b_shape, ", B");
print_shape(p.c_shape, ", C");
*os << ", trans_a=" << p.trans_a << ", trans_b=" << p.trans_b;
}
using MatMulTestParam_t = tuple<MatMulParamId, tuple<Backend, Target>>;
using MatMul = TestBaseWithParam<MatMulTestParam_t>;
PERF_TEST_P_(MatMul, matmul)
{
int test_id = (int)get<0>(GetParam());
ASSERT_GE(test_id, 0); ASSERT_LT(test_id, MatMulParamId::MATMUL_LAST);
const GemmParam_t& params = test_matmul_configs[test_id];
auto a_shape = params.a_shape;
auto b_shape = params.b_shape;
auto trans_a = params.trans_a;
auto trans_b = params.trans_b;
float alpha = 1.f;
float beta = 1.f;
Backend backend_id = get<0>(get<1>(GetParam()));
Target target_id = get<1>(get<1>(GetParam()));
Mat A(a_shape, CV_32F);
randu(A, -1.0f, 1.0f);
Mat B(b_shape, CV_32F);
randu(B, -1.0f, 1.0f);
LayerParams lp;
lp.type = "MatMul";
lp.name = "testLayer";
lp.set("transA", trans_a);
lp.set("transB", trans_b);
lp.set("alpha", alpha);
lp.set("beta", beta);
lp.blobs.push_back(B);
Net net;
net.addLayerToPrev(lp.name, lp.type, lp);
net.setPreferableBackend(backend_id);
net.setPreferableTarget(target_id);
// warmup
{
std::vector<std::string> input_names{"A"};
net.setInputsNames(input_names);
net.setInput(A, input_names[0]);
Mat out = net.forward();
}
TEST_CYCLE()
{
Mat res = net.forward();
}
SANITY_CHECK_NOTHING();
}
PERF_TEST_P_(MatMul, innerproduct)
{
int test_id = (int)get<0>(GetParam());
ASSERT_GE(test_id, 0); ASSERT_LT(test_id, MatMulParamId::MATMUL_LAST);
const GemmParam_t& params = test_matmul_configs[test_id];
auto a_shape = params.a_shape;
auto b_shape = params.b_shape;
Backend backend_id = get<0>(get<1>(GetParam()));
Target target_id = get<1>(get<1>(GetParam()));
Mat A(a_shape, CV_32F);
randu(A, -1.0f, 1.0f);
Mat B(b_shape, CV_32F);
randu(B, -1.0f, 1.0f);
LayerParams lp;
lp.type = "InnerProduct";
lp.name = "testLayer";
lp.set("axis", (int)(a_shape.size() - 1));
lp.set("bias_term", false);
// pre-transpose
std::vector<int> order(b_shape.size());
std::iota(order.begin(), order.end(), 0);
std::swap(order.back(), order[b_shape.size() - 2]);
Mat B_transposed;
transposeND(B, order, B_transposed);
lp.blobs.push_back(B_transposed);
lp.set("num_output", int(B_transposed.total(0, b_shape.size() - 1)));
lp.set("is_matmul", true);
Net net;
net.addLayerToPrev(lp.name, lp.type, lp);
net.setPreferableBackend(backend_id);
net.setPreferableTarget(target_id);
// warmup
{
std::vector<std::string> input_names{"A"};
net.setInputsNames(input_names);
net.setInput(A, input_names[0]);
Mat out = net.forward();
}
TEST_CYCLE()
{
Mat res = net.forward();
}
SANITY_CHECK_NOTHING();
}
INSTANTIATE_TEST_CASE_P(/**/, Gemm, Combine(
GemmParamId::all(),
dnnBackendsAndTargets(false, false) // defined in ../test/test_common.hpp
));
INSTANTIATE_TEST_CASE_P(/**/, MatMul, Combine(
MatMulParamId::all(),
dnnBackendsAndTargets(false, false) // defined in ../test/test_common.hpp
));
} // namespace
modules/dnn/perf/perf_layer.cpp
+169 -1
View File
@@ -683,7 +683,119 @@ PERF_TEST_P_(Layer_GatherElements, GatherElements)
test_layer({2700, 1, 2914}, {2700, 1, 81}, 2);
}
INSTANTIATE_TEST_CASE_P(/**/, Layer_Slice, dnnBackendsAndTargets(/* withInferenceEngine = */false, /* obsolete_withHalide = */false));
struct Layer_InstanceNorm : public TestBaseWithParam<tuple<Backend, Target> >
{
void test_layer(const std::vector<int>& x_shape)
{
int backendId = get<0>(GetParam());
int targetId = get<1>(GetParam());
Mat x(x_shape, CV_32FC1);
Mat scale(x_shape[1], 1, CV_32FC1);
Mat b(x_shape[1], 1, CV_32FC1);
randu(x, 0.f, 1.f);
randu(scale, 0.f, 1.f);
randu(b, 0.f, 1.f);
Net net;
LayerParams lp;
lp.type = "InstanceNormalization";
lp.name = "testLayer";
int id = net.addLayerToPrev(lp.name, lp.type, lp);
net.connect(0, 0, id, 0);
net.connect(0, 1, id, 1);
net.connect(0, 2, id, 2);
// warmup
{
std::vector<String> inpNames{"x", "scale", "b"};
net.setInputsNames(inpNames);
net.setInput(x, inpNames[0]);
net.setInput(scale, inpNames[1]);
net.setInput(b, inpNames[2]);
net.setPreferableBackend(backendId);
net.setPreferableTarget(targetId);
Mat out = net.forward();
}
TEST_CYCLE()
{
Mat res = net.forward();
}
SANITY_CHECK_NOTHING();
}
int N = 2;
int C = 64;
int H = 180;
int W = 240;
};
PERF_TEST_P_(Layer_InstanceNorm, InstanceNorm)
{
test_layer({N, C, H, W});
}
struct Layer_Attention : public TestBaseWithParam<tuple<Backend, Target>> {
void test_layer(const std::vector<int> x_shape, const std::vector<int> qkv_hidden_sizes, const int num_heads) {
int backendId = get<0>(GetParam());
int targetId = get<1>(GetParam());
auto qk_hidden_size = qkv_hidden_sizes[0];
auto v_hidden_size = qkv_hidden_sizes[2];
auto input_hidden_size = x_shape[2];
auto hidden_size = qk_hidden_size + qk_hidden_size + v_hidden_size;
Mat x(x_shape, CV_32F);
Mat weight(std::vector<int>{input_hidden_size, hidden_size}, CV_32F);
Mat bias(std::vector<int>{hidden_size}, CV_32F);
randu(x, 0.f, 1.f);
randu(weight, 0.f, 1.f);
randu(bias, 0.f, 1.f);
LayerParams lp;
lp.type = "Attention";
lp.name = "testLayer";
lp.set("num_heads", num_heads);
lp.set("qkv_hidden_sizes", DictValue::arrayInt(qkv_hidden_sizes.data(), qkv_hidden_sizes.size()));
Net net;
int id = net.addLayerToPrev(lp.name, lp.type, lp);
net.connect(0, 0, id, 0);
net.connect(0, 1, id, 1);
net.connect(0, 2, id, 2);
{
std::vector<std::string> input_names{"x", "weight", "bias"};
net.setInputsNames(input_names);
net.setInput(x, input_names[0]);
net.setInput(weight, input_names[1]);
net.setInput(bias, input_names[2]);
net.setPreferableBackend(backendId);
net.setPreferableTarget(targetId);
Mat out = net.forward();
}
TEST_CYCLE()
{
Mat out = net.forward();
}
SANITY_CHECK_NOTHING();
}
};
PERF_TEST_P_(Layer_Attention, VisionTransformer) {
test_layer({1, 197, 768}, {768, 768, 768}, 12);
}
INSTANTIATE_TEST_CASE_P(/**/, Layer_Slice, dnnBackendsAndTargets(false, false));
INSTANTIATE_TEST_CASE_P(/**/, Layer_NaryEltwise, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
#ifdef HAVE_CUDA
INSTANTIATE_TEST_CASE_P(CUDA, Layer_NaryEltwise, testing::Values(std::make_tuple(DNN_BACKEND_CUDA, DNN_TARGET_CUDA)));
@@ -693,6 +805,8 @@ INSTANTIATE_TEST_CASE_P(/**/, Layer_ScatterND, testing::Values(std::make_tuple(D
INSTANTIATE_TEST_CASE_P(/**/, Layer_LayerNorm, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
INSTANTIATE_TEST_CASE_P(/**/, Layer_LayerNormExpanded, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
INSTANTIATE_TEST_CASE_P(/**/, Layer_GatherElements, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
INSTANTIATE_TEST_CASE_P(/**/, Layer_InstanceNorm, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
INSTANTIATE_TEST_CASE_P(/**/, Layer_Attention, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
typedef TestBaseWithParam<tuple<Vec4i, int, bool, tuple<Backend, Target> > > Layer_FullyConnected;
@@ -715,6 +829,9 @@ PERF_TEST_P_(Layer_FullyConnected, fc)
int backendId = get<0>(get<3>(GetParam()));
int targetId = get<1>(get<3>(GetParam()));
if (inpShape.size() == 4 && inpShape[0] == 5 && inpShape[1] == 16 && inpShape[2] == 512 && inpShape[3] == 128 && outDims >= 512)
applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);
std::vector<int> weightShape;
if (isMatMul) {
weightShape = inpShape;
@@ -758,4 +875,55 @@ INSTANTIATE_TEST_CASE_P(/**/, Layer_FullyConnected, Combine(
dnnBackendsAndTargets()
));
typedef TestBaseWithParam<tuple<std::vector<int>, int, tuple<Backend, Target> > > Layer_Softmax;
PERF_TEST_P_(Layer_Softmax, softmax_3d) {
std::vector<int> shape = get<0>(GetParam());
int axis = get<1>(GetParam());
int backendId = get<0>(get<2>(GetParam()));
int targetId = get<1>(get<2>(GetParam()));
Mat data(shape, CV_32FC1);
Scalar mean = 0.f;
Scalar std = 1.f;
randn(data, mean, std);
Net net;
LayerParams lp;
lp.type = "Softmax";
lp.name = "testLayer";
lp.set("axis", axis);
net.addLayerToPrev(lp.name, lp.type, lp);
// warmup
{
net.setInput(data);
net.setPreferableBackend(backendId);
net.setPreferableTarget(targetId);
Mat out = net.forward();
}
TEST_CYCLE() {
Mat res = net.forward();
}
SANITY_CHECK_NOTHING();
}
INSTANTIATE_TEST_CASE_P(/**/, Layer_Softmax, Combine(
Values( // input size
std::vector<int>({16, 50, 50}),
std::vector<int>({16, 197, 197}),
std::vector<int>({16, 1024, 1024})
),
Values(0, 1, 2), // axis
dnnBackendsAndTargets(/* withInferenceEngine= */ false,
/* withHalide= */ false,
/* withCpuOCV= */ true,
/* withVkCom= */ false,
/* withCUDA= */ false,
/* withNgraph= */ false,
/* withWebnn= */ false,
/* withCann= */ false) // only test on CPU
));
} // namespace
modules/dnn/perf/perf_net.cpp
+40 -9
View File
@@ -82,7 +82,6 @@ public:
}
};
PERF_TEST_P_(DNNTestNetwork, AlexNet)
{
processNet("dnn/bvlc_alexnet.caffemodel", "dnn/bvlc_alexnet.prototxt", cv::Size(227, 227));
@@ -111,6 +110,8 @@ PERF_TEST_P_(DNNTestNetwork, Inception_5h)
PERF_TEST_P_(DNNTestNetwork, SSD)
{
applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);
processNet("dnn/VGG_ILSVRC2016_SSD_300x300_iter_440000.caffemodel", "dnn/ssd_vgg16.prototxt", cv::Size(300, 300));
}
@@ -136,6 +137,8 @@ PERF_TEST_P_(DNNTestNetwork, DenseNet_121)
PERF_TEST_P_(DNNTestNetwork, OpenPose_pose_mpi_faster_4_stages)
{
applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && (target == DNN_TARGET_MYRIAD || target == DNN_TARGET_HDDL))
throw SkipTestException("");
// The same .caffemodel but modified .prototxt
@@ -150,12 +153,17 @@ PERF_TEST_P_(DNNTestNetwork, opencv_face_detector)
PERF_TEST_P_(DNNTestNetwork, Inception_v2_SSD_TensorFlow)
{
applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);
processNet("dnn/ssd_inception_v2_coco_2017_11_17.pb", "ssd_inception_v2_coco_2017_11_17.pbtxt", cv::Size(300, 300));
}
PERF_TEST_P_(DNNTestNetwork, YOLOv3)
{
applyTestTag(CV_TEST_TAG_MEMORY_2GB);
applyTestTag(
CV_TEST_TAG_MEMORY_2GB,
CV_TEST_TAG_DEBUG_VERYLONG
);
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL)
throw SkipTestException("Test is disabled in OpenVINO 2020.4");
@@ -174,7 +182,10 @@ PERF_TEST_P_(DNNTestNetwork, YOLOv3)
PERF_TEST_P_(DNNTestNetwork, YOLOv4)
{
applyTestTag(CV_TEST_TAG_MEMORY_2GB);
applyTestTag(
CV_TEST_TAG_MEMORY_2GB,
CV_TEST_TAG_DEBUG_VERYLONG
);
if (target == DNN_TARGET_MYRIAD) // not enough resources
throw SkipTestException("");
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure
@@ -206,15 +217,23 @@ PERF_TEST_P_(DNNTestNetwork, YOLOv5) {
processNet("", "dnn/yolov5n.onnx", inp);
}
PERF_TEST_P_(DNNTestNetwork, YOLOv8) {
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
PERF_TEST_P_(DNNTestNetwork, YOLOv8)
{
applyTestTag(
CV_TEST_TAG_MEMORY_512MB,
CV_TEST_TAG_DEBUG_LONG
);
Mat sample = imread(findDataFile("dnn/dog416.png"));
Mat inp = blobFromImage(sample, 1.0 / 255.0, Size(640, 640), Scalar(), true);
processNet("", "dnn/yolov8n.onnx", inp);
}
PERF_TEST_P_(DNNTestNetwork, YOLOX) {
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
applyTestTag(
CV_TEST_TAG_MEMORY_512MB,
CV_TEST_TAG_DEBUG_VERYLONG
);
Mat sample = imread(findDataFile("dnn/dog416.png"));
Mat inp = blobFromImage(sample, 1.0 / 255.0, Size(640, 640), Scalar(), true);
processNet("", "dnn/yolox_s.onnx", inp);
@@ -222,16 +241,22 @@ PERF_TEST_P_(DNNTestNetwork, YOLOX) {
PERF_TEST_P_(DNNTestNetwork, EAST_text_detection)
{
applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);
processNet("dnn/frozen_east_text_detection.pb", "", cv::Size(320, 320));
}
PERF_TEST_P_(DNNTestNetwork, FastNeuralStyle_eccv16)
{
applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);
processNet("", "dnn/mosaic-9.onnx", cv::Size(224, 224));
}
PERF_TEST_P_(DNNTestNetwork, Inception_v2_Faster_RCNN)
{
applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019010000)
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019)
throw SkipTestException("Test is disabled in OpenVINO 2019R1");
@@ -317,17 +342,16 @@ PERF_TEST_P_(DNNTestNetwork, CRNN) {
processNet("", "dnn/text_recognition_CRNN_EN_2021sep.onnx", inp);
}
PERF_TEST_P_(DNNTestNetwork, ViTTrack) {
PERF_TEST_P_(DNNTestNetwork, VitTrack) {
Mat inp1(cv::Size(128, 128), CV_32FC3);
Mat inp2(cv::Size(256, 256), CV_32FC3);
randu(inp1, 0.0f, 1.0f);
randu(inp2, 0.0f, 1.0f);
inp1 = blobFromImage(inp1, 1.0, Size(), Scalar(), false);
inp2 = blobFromImage(inp2, 1.0, Size(), Scalar(), false);
processNet("", "dnn/onnx/models/vitTracker.onnx", {std::make_tuple(inp1, "template"), std::make_tuple(inp2, "search")});
processNet("", "dnn/onnx/models/object_tracking_vittrack_2023sep.onnx", {std::make_tuple(inp1, "template"), std::make_tuple(inp2, "search")});
}
PERF_TEST_P_(DNNTestNetwork, EfficientDet_int8)
{
if (target != DNN_TARGET_CPU || (backend != DNN_BACKEND_OPENCV &&
@@ -339,6 +363,13 @@ PERF_TEST_P_(DNNTestNetwork, EfficientDet_int8)
processNet("", "dnn/tflite/coco_efficientdet_lite0_v1_1.0_quant_2021_09_06.tflite", inp);
}
PERF_TEST_P_(DNNTestNetwork, VIT_B_32)
{
applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);
processNet("", "dnn/onnx/models/vit_b_32.onnx", cv::Size(224, 224));
}
INSTANTIATE_TEST_CASE_P(/*nothing*/, DNNTestNetwork, dnnBackendsAndTargets());
} // namespace
modules/dnn/src/caffe/caffe_importer.cpp
+5
View File
@@ -499,6 +499,11 @@ public:
{
type = "Convolution";
}
else if (type == "Softmax"){
// set default axis to 1
if(!layerParams.has("axis"))
layerParams.set("axis", 1);
}
int id = dstNet.addLayer(name, type, layerParams);
modules/dnn/src/cuda/eltwise_ops.cu
+7
View File
@@ -319,7 +319,13 @@ void eltwise_div_2(const Stream& stream, TensorSpan<T> output, TensorView<T> x,
eltwise_op<T, DivFunctor<T>>(stream, output, x, y);
}
template <class T>
void eltwise_sub_2(const Stream& stream, TensorSpan<T> output, TensorView<T> x, TensorView<T> y) {
eltwise_op<T, SubFunctor<T>>(stream, output, x, y);
}
#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530)
template void eltwise_sub_2(const Stream& stream, TensorSpan<__half> output, TensorView<__half> x, TensorView<__half> y);
template void eltwise_div_2(const Stream& stream, TensorSpan<__half> output, TensorView<__half> x, TensorView<__half> y);
template void eltwise_prod_2(const Stream& stream, TensorSpan<__half> output, TensorView<__half> x, TensorView<__half> y);
template void eltwise_sum_coeff_2(const Stream&, TensorSpan<__half>, __half, TensorView<__half>, __half, TensorView<__half>);
@@ -327,6 +333,7 @@ void eltwise_div_2(const Stream& stream, TensorSpan<T> output, TensorView<T> x,
template void eltwise_max_2(const Stream& stream, TensorSpan<__half> output, TensorView<__half> x, TensorView<__half> y);
template void eltwise_min_2(const Stream& stream, TensorSpan<__half> output, TensorView<__half> x, TensorView<__half> y);
#endif
template void eltwise_sub_2(const Stream& stream, TensorSpan<float> output, TensorView<float> x, TensorView<float> y);
template void eltwise_div_2(const Stream& stream, TensorSpan<float> output, TensorView<float> x, TensorView<float> y);
template void eltwise_prod_2(const Stream& stream, TensorSpan<float> output, TensorView<float> x, TensorView<float> y);
template void eltwise_sum_coeff_2(const Stream&, TensorSpan<float>, float, TensorView<float>, float, TensorView<float>);
modules/dnn/src/cuda/functors.hpp
+12
View File
@@ -741,6 +741,18 @@ struct DivFunctor {
CUDA4DNN_DEVICE T operator()(T x, T y) { return x / y; }
};
template <class T>
struct SubFunctor {
struct Params {
CUDA4DNN_HOST_DEVICE Params() { }
};
CUDA4DNN_DEVICE SubFunctor() { }
CUDA4DNN_DEVICE SubFunctor(const Params& params) { }
CUDA4DNN_DEVICE T operator()(T x, T y) { return x - y; }
};
template <class T>
struct SignFunctor {
struct Params {
modules/dnn/src/cuda/mvn.cu
+83
View File
@@ -66,6 +66,38 @@ namespace raw {
output[idx] = (static_cast<float>(input[idx]) - means[outer_idx]) * scale[outer_idx];
}
}
template <class T>
__global__ void normalize_mean_variance_channelwise(Span<T> output, View<T> input, View<T> scale, View<T> bias, View<float> means, View<float> inv_stddev, size_type inner_size, size_type C) {
for (auto idx : grid_stride_range(output.size())) {
const index_type outer_idx = idx / inner_size;
const index_type c = outer_idx % C;
auto s = static_cast<float>(scale[c]) * inv_stddev[outer_idx];
auto b = static_cast<float>(bias[c]);
output[idx] = (static_cast<float>(input[idx]) - means[outer_idx]) * s + b;
}
}
template <class T>
__global__ void normalize_mean_variance_layernorm(Span<T> output, View<T> input, View<T> scale, View<float> means, View<float> inv_stddev, size_type inner_size) {
for (auto idx : grid_stride_range(output.size())) {
const index_type outer_idx = idx / inner_size;
const index_type inner_idx = idx % inner_size;
auto s = static_cast<float>(scale[inner_idx]) * inv_stddev[outer_idx];
output[idx] = (static_cast<float>(input[idx]) - means[outer_idx]) * s;
}
}
template <class T>
__global__ void normalize_mean_variance_layernorm_with_bias(Span<T> output, View<T> input, View<T> scale, View<T> bias, View<float> means, View<float> inv_stddev, size_type inner_size) {
for (auto idx : grid_stride_range(output.size())) {
const index_type outer_idx = idx / inner_size;
const index_type inner_idx = idx % inner_size;
auto s = static_cast<float>(scale[inner_idx]) * inv_stddev[outer_idx];
auto b = static_cast<float>(bias[inner_idx]);
output[idx] = (static_cast<float>(input[idx]) - means[outer_idx]) * s + b;
}
}
}
template <class T>
@@ -142,4 +174,55 @@ template void normalize_mean_variance(const Stream&, Span<__half>, View<__half>,
#endif
template void normalize_mean_variance(const Stream&, Span<float>, View<float>, View<float>, View<float>, std::size_t);
template <class T>
void normalize_mean_variance_channelwise(const Stream& stream, Span<T> output, View<T> input, View<T> scale, View<T> bias, View<float> means, View<float> inv_stddev, std::size_t inner_size, std::size_t C)
{
CV_Assert(input.size() == output.size());
CV_Assert(input.size() / inner_size == means.size());
CV_Assert(means.size() == inv_stddev.size());
auto kernel = raw::normalize_mean_variance_channelwise<T>;
auto policy = make_policy(kernel, output.size(), 0, stream);
launch_kernel(kernel, policy, output, input, scale, bias, means, inv_stddev, inner_size, C);
}
#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530)
template void normalize_mean_variance_channelwise(const Stream&, Span<__half> /*output*/, View<__half> /*input*/, View<__half> /*scale*/, View<__half> /*bias*/, View<float> /*means*/, View<float> /*inv_stddev*/, std::size_t, std::size_t);
#endif
template void normalize_mean_variance_channelwise(const Stream&, Span<float> /*output*/, View<float> /*input*/, View<float> /*scale*/, View<float> /*bias*/, View<float> /*means*/, View<float> /*inv_stddev*/, std::size_t, std::size_t);
template <class T>
void normalize_mean_variance_layernorm(const Stream& stream, Span<T> output, View<T> input, View<T> scale, View<float> means, View<float> inv_stddev, std::size_t inner_size)
{
CV_Assert(input.size() == output.size());
CV_Assert(input.size() / inner_size == means.size());
CV_Assert(means.size() == inv_stddev.size());
auto kernel = raw::normalize_mean_variance_layernorm<T>;
auto policy = make_policy(kernel, output.size(), 0, stream);
launch_kernel(kernel, policy, output, input, scale, means, inv_stddev, inner_size);
}
#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530)
template void normalize_mean_variance_layernorm(const Stream&, Span<__half> /*output*/, View<__half> /*input*/, View<__half> /*scale*/, View<float> /*means*/, View<float> /*inv_stddev*/, std::size_t);
#endif
template void normalize_mean_variance_layernorm(const Stream&, Span<float> /*output*/, View<float> /*input*/, View<float> /*scale*/, View<float> /*means*/, View<float> /*inv_stddev*/, std::size_t);
template <class T>
void normalize_mean_variance_layernorm(const Stream& stream, Span<T> output, View<T> input, View<T> scale, View<T> bias, View<float> means, View<float> inv_stddev, std::size_t inner_size)
{
CV_Assert(input.size() == output.size());
CV_Assert(input.size() / inner_size == means.size());
CV_Assert(means.size() == inv_stddev.size());
auto kernel = raw::normalize_mean_variance_layernorm_with_bias<T>;
auto policy = make_policy(kernel, output.size(), 0, stream);
launch_kernel(kernel, policy, output, input, scale, bias, means, inv_stddev, inner_size);
}
#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530)
template void normalize_mean_variance_layernorm(const Stream&, Span<__half> /*output*/, View<__half> /*input*/, View<__half> /*scale*/, View<__half> /*bias*/, View<float> /*means*/, View<float> /*inv_stddev*/, std::size_t);
#endif
template void normalize_mean_variance_layernorm(const Stream&, Span<float> /*output*/, View<float> /*input*/, View<float> /*scale*/, View<float> /*bias*/, View<float> /*means*/, View<float> /*inv_stddev*/, std::size_t);
}}}} /* namespace cv::dnn::cuda4dnn::kernels */
modules/dnn/src/cuda4dnn/csl/cublas.hpp
+140
View File
@@ -8,6 +8,7 @@
#include "error.hpp"
#include "stream.hpp"
#include "pointer.hpp"
#include "memory.hpp"
#include <opencv2/core.hpp>
@@ -363,6 +364,145 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace csl { namespace cu
);
}
/** @brief Strided batched GEMM for colummn-major matrices
*
* \f$ C_i = \alpha A_i B_i + \beta C_i \f$ for a stack of matrices A, B and C indexed by i
*
* @tparam T matrix element type (must be `half` or `float`)
*
* @param handle valid cuBLAS Handle
* @param trans_a use transposed matrix of A_i for computation
* @param trans_b use transposed matrix of B_i for computation
* @param M number of rows in C
* @param N number of columns in C
* @param K common dimension of A (or trans A) and B (or trans B)
* @param alpha scale factor for A B
* @param[in] A pointer to stack of column-major matrices A in device memory
* @param lda leading dimension of matrix A
* @param A_offsets offsets to get A slices
* @param[in] B pointer to stack of column-major matrices B in device memory
* @param ldb leading dimension of matrix B
* @param B_offsets offsets to get B slices
* @param beta scale factor for C
* @param[in,out] C pointer to stack of column-major matrices C in device memory
* @param ldc leading dimension of matrix C
* @param C_offsets offsets to get C slices
* @param batchCount number of matrices in the batch
*
* Exception Guarantee: Basic
*/
template <class T>
void gemmBatched(const Handle &handle,
bool trans_a, bool trans_b,
std::size_t M, std::size_t N, std::size_t K,
T alpha,
const DevicePtr<const T> A, std::size_t lda, std::vector<std::size_t> A_offsets,
const DevicePtr<const T> B, std::size_t ldb, std::vector<std::size_t> B_offsets,
T beta,
const DevicePtr<T> C, std::size_t ldc, std::vector<std::size_t> C_offsets,
std::size_t batchCount);
template <> inline
void gemmBatched<half>(const Handle &handle,
bool trans_a, bool trans_b,
std::size_t M, std::size_t N, std::size_t K,
half alpha,
const DevicePtr<const half> A, std::size_t lda, std::vector<std::size_t> A_offsets,
const DevicePtr<const half> B, std::size_t ldb, std::vector<std::size_t> B_offsets,
half beta,
const DevicePtr<half> C, std::size_t ldc, std::vector<std::size_t> C_offsets,
std::size_t batchCount) {
CV_Assert(handle);
const auto opa = trans_a ? CUBLAS_OP_T : CUBLAS_OP_N,
opb = trans_b ? CUBLAS_OP_T : CUBLAS_OP_N;
const auto iM = static_cast<int>(M),
iN = static_cast<int>(N),
iK = static_cast<int>(K),
ilda = static_cast<int>(lda),
ildb = static_cast<int>(ldb),
ildc = static_cast<int>(ldc);
const auto batch_count = static_cast<int>(batchCount);
AutoBuffer<half> buffer(3 * batch_count);
auto A_slices = (half**)(buffer.data());
auto B_slices = A_slices + batch_count;
auto C_slices = B_slices + batch_count;
// collect A, B and C slices
for (int i = 0; i < batch_count; i++) {
A_slices[i] = (half*)(A.get()) + A_offsets[i];
B_slices[i] = (half*)(B.get()) + B_offsets[i];
C_slices[i] = (half*)(C.get()) + C_offsets[i];
}
const half **dev_A_slices = 0, **dev_B_slices = 0;
half **dev_C_slices = 0;
cudaMalloc((void**)&dev_A_slices, batch_count * sizeof(half*));
cudaMalloc((void**)&dev_B_slices, batch_count * sizeof(half*));
cudaMalloc((void**)&dev_C_slices, batch_count * sizeof(half*));
cudaMemcpy(dev_A_slices, A_slices, batch_count * sizeof(half*), cudaMemcpyHostToDevice);
cudaMemcpy(dev_B_slices, B_slices, batch_count * sizeof(half*), cudaMemcpyHostToDevice);
cudaMemcpy(dev_C_slices, C_slices, batch_count * sizeof(half*), cudaMemcpyHostToDevice);
CUDA4DNN_CHECK_CUBLAS(cublasHgemmBatched(handle.get(), opa, opb, iM, iN, iK, &alpha, dev_A_slices, ilda, dev_B_slices, ildb, &beta, dev_C_slices, ildc, batch_count));
cudaFree(dev_A_slices);
cudaFree(dev_B_slices);
cudaFree(dev_C_slices);
}
template <> inline
void gemmBatched<float>(const Handle &handle,
bool trans_a, bool trans_b,
std::size_t M, std::size_t N, std::size_t K,
float alpha,
const DevicePtr<const float> A, std::size_t lda, std::vector<std::size_t> A_offsets,
const DevicePtr<const float> B, std::size_t ldb, std::vector<std::size_t> B_offsets,
float beta,
const DevicePtr<float> C, std::size_t ldc, std::vector<std::size_t> C_offsets,
std::size_t batchCount) {
CV_Assert(handle);
const auto opa = trans_a ? CUBLAS_OP_T : CUBLAS_OP_N,
opb = trans_b ? CUBLAS_OP_T : CUBLAS_OP_N;
const auto iM = static_cast<int>(M),
iN = static_cast<int>(N),
iK = static_cast<int>(K),
ilda = static_cast<int>(lda),
ildb = static_cast<int>(ldb),
ildc = static_cast<int>(ldc);
const auto batch_count = static_cast<int>(batchCount);
AutoBuffer<float> buffer(3 * batch_count);
auto A_slices = (float**)(buffer.data());
auto B_slices = A_slices + batch_count;
auto C_slices = B_slices + batch_count;
// collect A, B and C slices
for (int i = 0; i < batch_count; i++) {
A_slices[i] = (float*)(A.get()) + A_offsets[i];
B_slices[i] = (float*)(B.get()) + B_offsets[i];
C_slices[i] = (float*)(C.get()) + C_offsets[i];
}
const float **dev_A_slices = 0, **dev_B_slices = 0;
float **dev_C_slices = 0;
cudaMalloc((void**)&dev_A_slices, batch_count * sizeof(float*));
cudaMalloc((void**)&dev_B_slices, batch_count * sizeof(float*));
cudaMalloc((void**)&dev_C_slices, batch_count * sizeof(float*));
cudaMemcpy(dev_A_slices, A_slices, batch_count * sizeof(float*), cudaMemcpyHostToDevice);
cudaMemcpy(dev_B_slices, B_slices, batch_count * sizeof(float*), cudaMemcpyHostToDevice);
cudaMemcpy(dev_C_slices, C_slices, batch_count * sizeof(float*), cudaMemcpyHostToDevice);
// cuBLAS is column-major
CUDA4DNN_CHECK_CUBLAS(cublasSgemmBatched(handle.get(), opa, opb, iM, iN, iK, &alpha, dev_A_slices, ilda, dev_B_slices, ildb, &beta, dev_C_slices, ildc, batch_count));
cudaFree(dev_A_slices);
cudaFree(dev_B_slices);
cudaFree(dev_C_slices);
}
}}}}} /* namespace cv::dnn::cuda4dnn::csl::cublas */
#endif /* OPENCV_DNN_SRC_CUDA4DNN_CSL_CUBLAS_HPP */
modules/dnn/src/cuda4dnn/csl/tensor_ops.hpp
+25
View File
@@ -152,6 +152,31 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace csl {
batch_size);
}
/** @brief performs generalized matrix-multiplication for a strided batch of matrices
*
* Pre-conditions:
* - A, B and C must be rank three tensors with dimensions (batch, rows, cols)
* - the last two axes of \p A and \p B must meet the mathematical requirements for matrix multiplication
* - \p C must be large enough to hold the result and the matrices must not overlap in memory
*
* Exception Guarantee: Basic
*/
template <class T> inline
void gemmBatched(const cublas::Handle& handle, std::size_t batch,
T beta, TensorSpan<T> C, const std::vector<std::size_t> C_offsets, T alpha,
bool trans_a, TensorView<T> A, const std::vector<std::size_t> A_offsets,
bool trans_b, TensorView<T> B, const std::vector<std::size_t> B_offsets) {
const auto M = C.get_axis_size(-2),
N = C.get_axis_size(-1),
K = A.get_axis_size(trans_a ? -2 : -1);
const auto lda = A.get_axis_size(-1),
ldb = B.get_axis_size(-1),
ldc = N;
// collect pointers and run cublasSgemmBatched / cublasHgemmBatched
csl::cublas::gemmBatched<T>(handle, trans_b, trans_a, N, M, K, 1.f, B.get(), ldb, B_offsets, A.get(), lda, A_offsets, 0.f, C.get(), ldc, C_offsets, batch);
}
/** @brief performs element-wise addition with broadcasting
*
* Pre-conditions:
modules/dnn/src/cuda4dnn/kernels/eltwise_ops.hpp
+3
View File
@@ -30,6 +30,9 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels {
template <class T>
void eltwise_div_2(const csl::Stream& stream, csl::TensorSpan<T> output, csl::TensorView<T> x, csl::TensorView<T> y);
template <class T>
void eltwise_sub_2(const csl::Stream& stream, csl::TensorSpan<T> output, csl::TensorView<T> x, csl::TensorView<T> y);
}}}} /* namespace cv::dnn::cuda4dnn::kernels */
#endif /* OPENCV_DNN_SRC_CUDA4DNN_KERNELS_ELTWISE_OPS_HPP */
modules/dnn/src/cuda4dnn/kernels/mvn.hpp
+9
View File
@@ -26,6 +26,15 @@ void normalize_mean(const csl::Stream& stream, csl::Span<T> output, csl::View<T>
template <class T>
void normalize_mean_variance(const csl::Stream& stream, csl::Span<T> output, csl::View<T> input, csl::View<float> means, csl::View<float> scale, std::size_t inner_size);
template <class T>
void normalize_mean_variance_channelwise(const csl::Stream &stream, csl::Span<T> output, csl::View<T> input, csl::View<T> scale, csl::View<T> bias, csl::View<float> means, csl::View<float> inv_stddev, std::size_t inner_size, std::size_t C);
template <class T>
void normalize_mean_variance_layernorm(const csl::Stream &stream, csl::Span<T> output, csl::View<T> input, csl::View<T> scale, csl::View<float> means, csl::View<float> inv_stddev, std::size_t inner_size);
template <class T>
void normalize_mean_variance_layernorm(const csl::Stream &stream, csl::Span<T> output, csl::View<T> input, csl::View<T> scale, csl::View<T> bias, csl::View<float> means, csl::View<float> inv_stddev, std::size_t inner_size);
}}}} /* namespace cv::dnn::cuda4dnn::kernels */
#endif /* OPENCV_DNN_SRC_CUDA4DNN_KERNELS_MVN_HPP */
modules/dnn/src/cuda4dnn/primitives/eltwise.hpp
+3
View File
@@ -27,6 +27,7 @@ namespace cv { namespace dnn { namespace cuda4dnn {
PRODUCT,
DIV,
MIN,
SUB,
};
class EltwiseOpBase : public CUDABackendNode {
@@ -88,6 +89,7 @@ namespace cv { namespace dnn { namespace cuda4dnn {
else
kernels::eltwise_sum_coeff_2<T>(stream, output, coeffs[0], input_x, coeffs[1], input_y);
break;
case EltwiseOpType::SUB: kernels::eltwise_sub_2<T>(stream, output, input_x, input_y); break;
}
}
else
@@ -119,6 +121,7 @@ namespace cv { namespace dnn { namespace cuda4dnn {
kernels::eltwise_sum_coeff_2<T>(stream, output, coeff_x, output, coeffs[i], input);
}
break;
case EltwiseOpType::SUB: kernels::eltwise_sub_2<T>(stream, output, output, input); break;
}
}
}
modules/dnn/src/cuda4dnn/primitives/instance_norm.hpp
+86
View File
@@ -0,0 +1,86 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#ifndef OPENCV_DNN_SRC_CUDA4DNN_PRIMITIVES_INSTANCE_NORM_HPP
#define OPENCV_DNN_SRC_CUDA4DNN_PRIMITIVES_INSTANCE_NORM_HPP
#include "../../op_cuda.hpp"
#include "../csl/stream.hpp"
#include "../csl/span.hpp"
#include "../csl/tensor.hpp"
#include "../csl/workspace.hpp"
#include "../kernels/fill_copy.hpp"
#include "../kernels/mvn.hpp"
#include <opencv2/core.hpp>
#include <cstddef>
#include <vector>
#include <utility>
namespace cv { namespace dnn { namespace cuda4dnn {
template <class T>
class InstanceNormOp final : public CUDABackendNode {
public:
using wrapper_type = GetCUDABackendWrapperType<T>;
InstanceNormOp(csl::Stream stream_, float epsilon_, size_t loops)
: stream(std::move(stream_)), epsilon(epsilon_) {
csl::WorkspaceBuilder builder;
builder.require<float>(loops);
builder.require<float>(loops);
scratch_mem_in_bytes = builder.required_workspace_size();
}
void forward(const std::vector<cv::Ptr<BackendWrapper>>& inputs,
const std::vector<cv::Ptr<BackendWrapper>>& outputs,
csl::Workspace& workspace) override {
auto input_wrapper = inputs[0].dynamicCast<wrapper_type>();
auto scale_wrapper = inputs[1].dynamicCast<wrapper_type>();
auto bias_wrapper = inputs[2].dynamicCast<wrapper_type>();
auto input = input_wrapper->getView();
auto scale = scale_wrapper->getView();
auto bias = bias_wrapper->getView();
auto output_wrapper = outputs[0].dynamicCast<wrapper_type>();
auto output = output_wrapper->getSpan();
auto C = input.get_axis_size(1);
auto loops = input.size_range(0, 2);
auto norm_size = input.size_range(2, input.rank());
if (norm_size == 1) {
kernels::fill<T>(stream, output, 0.f);
return;
} else {
auto ws_allocator = csl::WorkspaceAllocator(workspace);
auto mean = ws_allocator.get_span<float>(loops);
kernels::fill<float>(stream, mean, 0.f);
auto stdev = ws_allocator.get_span<float>(loops);
kernels::fill<float>(stream, stdev, 0.f);
kernels::reduce_mean_sqr_sum<T>(stream, mean, stdev, input, norm_size);
kernels::compute_normalization_scale(stream, stdev, mean, stdev, norm_size, epsilon);
kernels::normalize_mean_variance_channelwise<T>(stream, output, input, scale, bias, mean, stdev, norm_size, C);
}
}
std::size_t get_workspace_memory_in_bytes() const noexcept override { return scratch_mem_in_bytes; }
private:
csl::Stream stream;
float epsilon;
std::size_t scratch_mem_in_bytes;
};
}}} // cv::dnn::cuda4dnn
#endif // OPENCV_DNN_SRC_CUDA4DNN_PRIMITIVES_INSTANCE_NORM_HPP
modules/dnn/src/cuda4dnn/primitives/layer_norm.hpp
+93
View File
@@ -0,0 +1,93 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#ifndef OPENCV_DNN_SRC_CUDA4DNN_PRIMITIVES_LAYER_NORM_HPP
#define OPENCV_DNN_SRC_CUDA4DNN_PRIMITIVES_LAYER_NORM_HPP
#include "../../op_cuda.hpp"
#include "../csl/stream.hpp"
#include "../csl/span.hpp"
#include "../csl/tensor.hpp"
#include "../csl/workspace.hpp"
#include "../kernels/fill_copy.hpp"
#include "../kernels/mvn.hpp"
#include <opencv2/core.hpp>
#include <cstddef>
#include <vector>
#include <utility>
namespace cv { namespace dnn { namespace cuda4dnn {
template <class T>
class LayerNormOp final : public CUDABackendNode {
public:
using wrapper_type = GetCUDABackendWrapperType<T>;
LayerNormOp(csl::Stream stream_, int normalized_axis, float epsilon_, size_t loops)
: stream(std::move(stream_)), epsilon(epsilon_) {
CV_CheckGE(normalized_axis, 0, "LayerNorm/CUDA: axis needs to be normalized");
axis = static_cast<size_t>(normalized_axis);
csl::WorkspaceBuilder builder;
builder.require<float>(loops);
builder.require<float>(loops);
scratch_mem_in_bytes = builder.required_workspace_size();
}
void forward(const std::vector<cv::Ptr<BackendWrapper>>& inputs,
const std::vector<cv::Ptr<BackendWrapper>>& outputs,
csl::Workspace& workspace) override {
auto input_wrapper = inputs[0].dynamicCast<wrapper_type>();
auto scale_wrapper = inputs[1].dynamicCast<wrapper_type>();
auto input = input_wrapper->getView();
auto scale = scale_wrapper->getView();
auto output_wrapper = outputs[0].dynamicCast<wrapper_type>();
auto output = output_wrapper->getSpan();
auto loops = input.size_range(0, axis);
auto norm_size = input.size_range(axis, input.rank());
if (norm_size == 1) {
kernels::fill<T>(stream, output, 0.f);
return;
} else {
auto ws_allocator = csl::WorkspaceAllocator(workspace);
auto mean = ws_allocator.get_span<float>(loops);
kernels::fill<float>(stream, mean, 0.f);
auto inv_stddev = ws_allocator.get_span<float>(loops);
kernels::fill<float>(stream, inv_stddev, 0.f);
kernels::reduce_mean_sqr_sum<T>(stream, mean, inv_stddev, input, norm_size);
kernels::compute_normalization_scale(stream, inv_stddev, mean, inv_stddev, norm_size, epsilon);
if (inputs.size() == 3) {
auto bias_wrapper = inputs[2].dynamicCast<wrapper_type>();
auto bias = bias_wrapper->getView();
kernels::normalize_mean_variance_layernorm<T>(stream, output, input, scale, bias, mean, inv_stddev, norm_size);
} else {
kernels::normalize_mean_variance_layernorm<T>(stream, output, input, scale, mean, inv_stddev, norm_size);
}
}
}
std::size_t get_workspace_memory_in_bytes() const noexcept override { return scratch_mem_in_bytes; }
private:
csl::Stream stream;
float epsilon;
size_t axis;
std::size_t scratch_mem_in_bytes;
};
}}} // cv::dnn::cuda4dnn
#endif // OPENCV_DNN_SRC_CUDA4DNN_PRIMITIVES_LAYER_NORM_HPP
modules/dnn/src/cuda4dnn/primitives/matmul_broadcast.hpp
+79
View File
@@ -0,0 +1,79 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#ifndef OPENCV_DNN_SRC_CUDA4DNN_PRIMITIVES_MATMUL_BROADCAST_HPP
#define OPENCV_DNN_SRC_CUDA4DNN_PRIMITIVES_MATMUL_BROADCAST_HPP
#include "../../op_cuda.hpp"
#include "../csl/stream.hpp"
#include "../csl/cublas.hpp"
#include "../csl/tensor.hpp"
#include "../csl/tensor_ops.hpp"
#include <opencv2/core.hpp>
#include <utility>
namespace cv { namespace dnn { namespace cuda4dnn {
template <class T>
class MatMulBroadcastOp final : public CUDABackendNode {
public:
using wrapper_type = GetCUDABackendWrapperType<T>;
MatMulBroadcastOp(csl::Stream stream_, csl::cublas::Handle handle, const Mat &B, bool _transA, bool _transB,
const std::vector<size_t> &A_offsets_, const std::vector<size_t> &B_offsets_, std::vector<size_t> &C_offsets_,
size_t batch_)
: stream(std::move(stream_)), cublasHandle(std::move(handle)), A_offsets(A_offsets_), B_offsets(B_offsets_), C_offsets(C_offsets_), batch(batch_)
{
if (!B.empty()) {
input_B_tensor = csl::makeTensorHeader<T>(B);
csl::copyMatToTensor<T>(B, input_B_tensor, stream);
}
transA = _transA;
transB = _transB;
}
void forward(
const std::vector<cv::Ptr<BackendWrapper>>& inputs,
const std::vector<cv::Ptr<BackendWrapper>>& outputs,
csl::Workspace& workspace) override
{
CV_Assert(((inputs.size() == 2 && input_B_tensor.empty()) ||
(inputs.size() == 1 && !input_B_tensor.empty())) && outputs.size() == 1);
auto input_A_wrapper = inputs[0].dynamicCast<wrapper_type>();
auto input_A = input_A_wrapper->getView();
csl::TensorView<T> input_B;
if (input_B_tensor.empty()) {
auto input_B_wrapper = inputs[1].dynamicCast<wrapper_type>();
input_B = input_B_wrapper->getView();
} else {
input_B = csl::TensorView<T>(input_B_tensor);
}
auto output_wrapper = outputs[0].dynamicCast<wrapper_type>();
auto output = output_wrapper->getSpan();
csl::tensor_ops::gemmBatched<T>(cublasHandle, batch, 0.f, output, C_offsets, 1.f, transA, input_A, A_offsets, transB, input_B, B_offsets);
}
private:
csl::Stream stream;
csl::cublas::Handle cublasHandle;
csl::Tensor<T> input_B_tensor;
bool transA, transB;
std::vector<size_t> A_offsets;
std::vector<size_t> B_offsets;
std::vector<size_t> C_offsets;
size_t batch;
};
}}} /* namespace cv::dnn::cuda4dnn */
#endif /* OPENCV_DNN_SRC_CUDA4DNN_PRIMITIVES_MATMUL_BROADCAST_HPP */
modules/dnn/src/cuda4dnn/primitives/scale_shift.hpp
+3
View File
@@ -128,6 +128,9 @@ namespace cv { namespace dnn { namespace cuda4dnn {
/* the scale shift operation might require broadcasting */
const int end_axis = [&] {
if (num_parameters == 1) {
return static_cast<int>(axis + 1);
}
for (int endAxis = axis + 1; endAxis <= input.rank(); endAxis++) {
if (input.size_range(axis, endAxis) == mid_size)
return endAxis;
modules/dnn/src/darknet/darknet_io.cpp
+62 -4
View File
@@ -309,11 +309,56 @@ namespace cv {
fused_layer_names.push_back(last_layer);
}
void setCrop(int crop_height, int crop_width, int inp_height, int inp_width, bool noadjust)
{
cv::dnn::LayerParams crop_param;
crop_param.name = "CropLayer-name";
std::vector<int> begin = {0, 0, (inp_height - crop_height) / 2, (inp_width - crop_width) / 2};
std::vector<int> sizes = {-1, -1, crop_height, crop_width};
crop_param.set("begin", DictValue::arrayInt(&begin[0], begin.size()));
crop_param.set("size", DictValue::arrayInt(&sizes[0], sizes.size()));
crop_param.type = "Slice";
darknet::LayerParameter lp;
std::string layer_name = cv::format("crop_%d", layer_id);
lp.layer_name = layer_name;
lp.layer_type = crop_param.type;
lp.layerParams = crop_param;
lp.bottom_indexes.push_back(last_layer);
last_layer = layer_name;
net->layers.push_back(lp);
layer_id++;
if (!noadjust)
{
cv::dnn::LayerParams params;
params.set("bias_term", true);
params.blobs = {
Mat(1, 1, CV_32F, Scalar(2)),
Mat(1, 1, CV_32F, Scalar(-1))
};
darknet::LayerParameter lp;
std::string layer_name = cv::format("adjust_crop_%d", layer_id);
lp.layer_name = layer_name;
lp.layer_type = "Scale";
lp.layerParams = params;
lp.bottom_indexes.push_back(last_layer);
last_layer = layer_name;
net->layers.push_back(lp);
layer_id++;
}
fused_layer_names.push_back(last_layer);
}
void setSoftmax()
{
cv::dnn::LayerParams softmax_param;
softmax_param.name = "Softmax-name";
softmax_param.type = "Softmax";
// set default axis to 1
if(!softmax_param.has("axis"))
softmax_param.set("axis", 1);
darknet::LayerParameter lp;
std::string layer_name = cv::format("softmax_%d", layer_id);
@@ -682,8 +727,8 @@ namespace cv {
MatShape tensor_shape(3);
tensor_shape[0] = net->channels;
tensor_shape[1] = net->width;
tensor_shape[2] = net->height;
tensor_shape[1] = net->height;
tensor_shape[2] = net->width;
net->out_channels_vec.resize(net->layers_cfg.size());
layers_counter = -1;
@@ -760,6 +805,19 @@ namespace cv {
tensor_shape[1] = 1;
tensor_shape[2] = 1;
}
else if (layer_type == "crop")
{
int crop_height = getParam<int>(layer_params, "crop_height", 0);
int crop_width = getParam<int>(layer_params, "crop_width", 0);
bool noadjust = getParam<int>(layer_params, "noadjust", false);
CV_CheckGT(crop_height, 0, "");
CV_CheckGT(crop_width, 0, "");
setParams.setCrop(crop_height, crop_width, tensor_shape[1], tensor_shape[2], noadjust);
tensor_shape[1] = crop_height;
tensor_shape[2] = crop_width;
}
else if (layer_type == "softmax")
{
int groups = getParam<int>(layer_params, "groups", 1);
@@ -934,8 +992,8 @@ namespace cv {
MatShape tensor_shape(3);
tensor_shape[0] = net->channels;
tensor_shape[1] = net->width;
tensor_shape[2] = net->height;
tensor_shape[1] = net->height;
tensor_shape[2] = net->width;
int cv_layers_counter = -1;
int darknet_layers_counter = -1;
modules/dnn/src/dnn_read.cpp
+3 -1
View File
@@ -58,7 +58,9 @@ Net readNet(const String& _framework, const std::vector<uchar>& bufferModel,
const std::vector<uchar>& bufferConfig)
{
String framework = toLowerCase(_framework);
if (framework == "caffe")
if (framework == "onnx")
return readNetFromONNX(bufferModel);
else if (framework == "caffe")
return readNetFromCaffe(bufferConfig, bufferModel);
else if (framework == "tensorflow")
return readNetFromTensorflow(bufferModel, bufferConfig);
modules/dnn/src/dnn_utils.cpp
+64 -4
View File
@@ -17,9 +17,9 @@ Image2BlobParams::Image2BlobParams():scalefactor(Scalar::all(1.0)), size(Size())
{}
Image2BlobParams::Image2BlobParams(const Scalar& scalefactor_, const Size& size_, const Scalar& mean_, bool swapRB_,
int ddepth_, DataLayout datalayout_, ImagePaddingMode mode_):
scalefactor(scalefactor_), size(size_), mean(mean_), swapRB(swapRB_), ddepth(ddepth_),
datalayout(datalayout_), paddingmode(mode_)
int ddepth_, DataLayout datalayout_, ImagePaddingMode mode_, Scalar borderValue_):
scalefactor(scalefactor_), size(size_), mean(mean_), swapRB(swapRB_), ddepth(ddepth_),
datalayout(datalayout_), paddingmode(mode_), borderValue(borderValue_)
{}
void getVector(InputArrayOfArrays images_, std::vector<Mat>& images) {
@@ -196,7 +196,7 @@ void blobFromImagesWithParamsImpl(InputArrayOfArrays images_, Tmat& blob_, const
int bottom = size.height - top - rh;
int left = (size.width - rw)/2;
int right = size.width - left - rw;
copyMakeBorder(images[i], images[i], top, bottom, left, right, BORDER_CONSTANT);
copyMakeBorder(images[i], images[i], top, bottom, left, right, BORDER_CONSTANT, param.borderValue);
}
else
{
@@ -382,6 +382,66 @@ void imagesFromBlob(const cv::Mat& blob_, OutputArrayOfArrays images_)
}
}
Rect Image2BlobParams::blobRectToImageRect(const Rect &r, const Size &oriImage)
{
CV_Assert(!oriImage.empty());
std::vector<Rect> rImg, rBlob;
rBlob.push_back(Rect(r));
rImg.resize(1);
this->blobRectsToImageRects(rBlob, rImg, oriImage);
return Rect(rImg[0]);
}
void Image2BlobParams::blobRectsToImageRects(const std::vector<Rect> &rBlob, std::vector<Rect>& rImg, const Size& imgSize)
{
Size size = this->size;
rImg.resize(rBlob.size());
if (size != imgSize)
{
if (this->paddingmode == DNN_PMODE_CROP_CENTER)
{
float resizeFactor = std::max(size.width / (float)imgSize.width,
size.height / (float)imgSize.height);
for (int i = 0; i < rBlob.size(); i++)
{
rImg[i] = Rect((rBlob[i].x + 0.5 * (imgSize.width * resizeFactor - size.width)) / resizeFactor,
(rBlob[i].y + 0.5 * (imgSize.height * resizeFactor - size.height)) / resizeFactor,
rBlob[i].width / resizeFactor,
rBlob[i].height / resizeFactor);
}
}
else if (this->paddingmode == DNN_PMODE_LETTERBOX)
{
float resizeFactor = std::min(size.width / (float)imgSize.width,
size.height / (float)imgSize.height);
int rh = int(imgSize.height * resizeFactor);
int rw = int(imgSize.width * resizeFactor);
int top = (size.height - rh) / 2;
int left = (size.width - rw) / 2;
for (int i = 0; i < rBlob.size(); i++)
{
rImg[i] = Rect((rBlob[i].x - left) / resizeFactor,
(rBlob[i].y - top) / resizeFactor,
rBlob[i].width / resizeFactor,
rBlob[i].height / resizeFactor);
}
}
else if (this->paddingmode == DNN_PMODE_NULL)
{
for (int i = 0; i < rBlob.size(); i++)
{
rImg[i] = Rect(rBlob[i].x * (float)imgSize.width / size.width,
rBlob[i].y * (float)imgSize.height / size.height,
rBlob[i].width * (float)imgSize.width / size.width,
rBlob[i].height * (float)imgSize.height / size.height);
}
}
else
CV_Error(CV_StsBadArg, "Unknown padding mode");
}
}
CV__DNN_INLINE_NS_END
}} // namespace cv::dnn
modules/dnn/src/graph_simplifier.cpp
+134 -53
View File
@@ -77,68 +77,107 @@ int Subgraph::getInputNodeId(const Ptr<ImportGraphWrapper>& net,
}
bool Subgraph::match(const Ptr<ImportGraphWrapper>& net, int nodeId,
std::vector<int>& matchedNodesIds,
std::vector<int>& targetNodesIds)
std::vector<int>& matchedNodesIds)
{
matchedNodesIds.clear();
targetNodesIds.clear();
std::queue<int> nodesToMatch;
std::queue<int> targetNodes;
nodesToMatch.push(nodeId);
targetNodes.push(nodes.size() - 1);
while (!nodesToMatch.empty())
// Collection of all matchings states across branching.
// If there is no commutative ops in the subgraph - there would be just a single map.
std::vector<std::shared_ptr<std::map<int, int>>> matchCandidates;
matchCandidates.push_back(makePtr<std::map<int, int>>());
struct State
{
int nodeToMatch = nodesToMatch.front();
int targetNodeId = targetNodes.front();
nodesToMatch.pop();
targetNodes.pop();
int nodeToMatch;
int targetNodeId;
// Every state refers to current matchings pairs as well as
// matchings from parent branches produced by commutative ops.
std::vector<std::shared_ptr<std::map<int, int>>> matchings;
if (std::find(matchedNodesIds.begin(), matchedNodesIds.end(), nodeToMatch) !=
matchedNodesIds.end())
// When we register a matching pair we should register it in every parent branch.
// This is actual for branching in case of commutative ops only.
void addMatch(std::pair<int, int> match)
{
for (auto& m : matchings)
m->insert(match);
}
};
std::queue<State> states;
states.push({nodeId, (int)nodes.size() - 1, matchCandidates});
while (!states.empty())
{
auto state = states.front();
states.pop();
int nodeToMatch = state.nodeToMatch;
int targetNodeId = state.targetNodeId;
auto matchings = state.matchings.back();
if (matchings->find(targetNodeId) != matchings->end())
continue;
// Empty placeholder matches with any input type
if (nodes[targetNodeId].empty()) {
state.addMatch({targetNodeId, nodeToMatch});
continue;
}
const Ptr<ImportNodeWrapper> node = net->getNode(nodeToMatch);
if (node->getType() != nodes[targetNodeId])
return false;
continue;
std::vector<int>& inputNodes = inputs[targetNodeId];
if (inputNodes.size() != node->getNumInputs())
return false;
continue;
for (int j = 0; j < inputNodes.size(); ++j)
state.addMatch({targetNodeId, nodeToMatch});
bool isCommutative = net->isCommutativeOp(node->getType());
if (isCommutative)
{
if (nodes[inputNodes[j]].empty() || node->getInputName(j).empty()) // Unknown input node type.
continue;
nodeId = getInputNodeId(net, node, j);
const Ptr<ImportNodeWrapper> inpNode = net->getNode(nodeId);
if (inpNode->getType() != "Const" && inpNode->getType() != "Constant")
{
nodesToMatch.push(nodeId);
targetNodes.push(inputNodes[j]);
}
else if (nodes[inputNodes[j]] != "Const" && nodes[inputNodes[j]] != "Constant")
return false;
}
matchedNodesIds.push_back(nodeToMatch);
targetNodesIds.push_back(targetNodeId);
}
if (inputNodes.size() != 2)
CV_Error(Error::StsNotImplemented, "Commutative op fusion with more than 2 inputs");
const int n = matchedNodesIds.size();
std::vector<std::pair<int, int> > elements(n);
for (int i = 0; i < n; ++i)
elements[i] = std::make_pair(matchedNodesIds[i], targetNodesIds[i]);
std::sort(elements.begin(), elements.end());
for (int i = 0; i < n; ++i)
{
matchedNodesIds[i] = elements[i].first;
targetNodesIds[i] = elements[i].second;
auto newMatchings = makePtr<std::map<int, int>>(*matchings);
matchCandidates.push_back(newMatchings);
state.matchings.push_back(newMatchings);
states.push({getInputNodeId(net, node, 0), inputNodes[0], state.matchings});
states.push({getInputNodeId(net, node, 1), inputNodes[1], state.matchings});
state.matchings.pop_back();
newMatchings = makePtr<std::map<int, int>>(*matchings);
matchCandidates.push_back(newMatchings);
state.matchings.push_back(newMatchings);
states.push({getInputNodeId(net, node, 0), inputNodes[1], state.matchings});
states.push({getInputNodeId(net, node, 1), inputNodes[0], state.matchings});
state.matchings.pop_back();
}
else
{
for (int j = 0; j < inputNodes.size(); ++j)
{
nodeId = getInputNodeId(net, node, j);
states.push({nodeId, inputNodes[j], state.matchings});
}
}
}
return true;
for (auto& matchings : matchCandidates)
{
if (matchings->size() != nodes.size())
continue;
matchedNodesIds.resize(matchings->size());
for (int i = 0; i < matchings->size(); ++i)
{
CV_Assert(matchings->find(i) != matchings->end());
matchedNodesIds[i] = matchings->at(i);
}
return true;
}
return false;
}
void Subgraph::replace(const Ptr<ImportGraphWrapper>& net, const std::vector<int>& matchedNodesIds,
const std::vector<int>& targetNodesIds)
void Subgraph::replace(const Ptr<ImportGraphWrapper>& net, const std::vector<int>& matchedNodesIds)
{
// Extract names of input nodes.
std::vector<std::string> inputsNames(fusedNodeInputs.size());
@@ -149,9 +188,9 @@ void Subgraph::replace(const Ptr<ImportGraphWrapper>& net, const std::vector<int
for (int j = 0; j < matchedNodesIds.size() && inpName.empty(); ++j)
{
Ptr<ImportNodeWrapper> node = net->getNode(matchedNodesIds[j]);
std::vector<int>& inpIndices = inputs[targetNodesIds[j]];
std::vector<int>& inpIndices = inputs[j];
CV_Assert(node->getNumInputs() == inpIndices.size());
CV_Assert(inpIndices.empty() || node->getNumInputs() == inpIndices.size());
for (int k = 0; k < inpIndices.size(); ++k)
{
if (inpIndices[k] == fusedNodeInputs[i])
@@ -165,10 +204,7 @@ void Subgraph::replace(const Ptr<ImportGraphWrapper>& net, const std::vector<int
inputsNames[i] = inpName;
}
// Remove matched nodes except the last one. Indices in ascending order are expected.
Ptr<ImportNodeWrapper> node = net->getNode(matchedNodesIds.back());
for (int i = matchedNodesIds.size() - 2; i >= 0; --i)
net->removeNode(matchedNodesIds[i]);
// Modify the last node to be a fused one.
node->setType(fusedNodeOp);
@@ -190,18 +226,63 @@ void simplifySubgraphs(const Ptr<ImportGraphWrapper>& net,
const std::vector<Ptr<Subgraph> >& patterns)
{
int numNodes = net->getNumNodes();
std::vector<int> matchedNodesIds, targetNodesIds;
std::vector<int> matchedNodesIds;
std::vector<int> nodesToRemove;
for (int j = 0; j < patterns.size(); ++j)
{
for (int i = 0; i < numNodes; ++i)
{
if (patterns[j]->match(net, i, matchedNodesIds, targetNodesIds))
if (patterns[j]->match(net, i, matchedNodesIds))
{
patterns[j]->replace(net, matchedNodesIds, targetNodesIds);
numNodes -= matchedNodesIds.size() - 1; // #matchedNodes removed and one added.
patterns[j]->replace(net, matchedNodesIds);
// Remove matched nodes except the last one.
nodesToRemove.insert(nodesToRemove.end(), matchedNodesIds.begin(), matchedNodesIds.end() - 1);
}
}
}
if (nodesToRemove.empty())
return;
// Collect reference counts for every node
std::vector<int> refcounts(net->getNumNodes(), 0);
std::map<std::string, int> nodeIds;
// Register node outputs.
// Every usage of one of the node's outputs should be counted.
for (int nodeId = 0; nodeId < refcounts.size(); ++nodeId) {
for (int i = 0; i < net->getNumOutputs(nodeId); ++i) {
std::string name = net->getOutputName(nodeId, i);
nodeIds[name] = nodeId;
}
}
for (int nodeId = 0; nodeId < refcounts.size(); ++nodeId) {
// Increase counters for node's inputs
auto node = net->getNode(nodeId);
for (int i = 0; i < node->getNumInputs(); ++i) {
std::string inpName = node->getInputName(i);
if (inpName.empty())
continue;
CV_Assert(nodeIds.find(inpName) != nodeIds.end());
refcounts[nodeIds[inpName]] += 1;
}
}
// Remove all fused nodes. Indices expected to be in descending order.
std::sort(nodesToRemove.begin(), nodesToRemove.end(), [](int a, int b) { return a > b; });
for (int nodeId : nodesToRemove) {
if (refcounts[nodeId] == 0) {
// Decrease references to node's inputs and remove node itself
auto node = net->getNode(nodeId);
for (int i = 0; i < node->getNumInputs(); ++i) {
std::string inpName = node->getInputName(i);
refcounts[nodeIds[inpName]] -= 1;
}
net->removeNode(nodeId);
refcounts[nodeId] = -1; // Same node cannot be removed twice
}
}
}
}} // namespace cv::dnn
modules/dnn/src/graph_simplifier.hpp
+4 -4
View File
@@ -44,6 +44,8 @@ public:
virtual std::string getOutputName(int nodeId, int outId) const = 0;
virtual void removeNode(int idx) = 0;
virtual bool isCommutativeOp(const std::string& type) const = 0;
};
class Subgraph // Interface to match and replace subgraphs.
@@ -75,12 +77,10 @@ public:
// Match TensorFlow subgraph starting from <nodeId> with a set of nodes to be fused.
// Const nodes are skipped during matching. Returns true if nodes are matched and can be fused.
virtual bool match(const Ptr<ImportGraphWrapper>& net, int nodeId,
std::vector<int>& matchedNodesIds,
std::vector<int>& targetNodesIds);
std::vector<int>& matchedNodesIds);
// Fuse matched subgraph.
void replace(const Ptr<ImportGraphWrapper>& net, const std::vector<int>& matchedNodesIds,
const std::vector<int>& targetNodesIds);
void replace(const Ptr<ImportGraphWrapper>& net, const std::vector<int>& matchedNodesIds);
virtual void finalize(const Ptr<ImportGraphWrapper>& net,
const Ptr<ImportNodeWrapper>& fusedNode,
modules/dnn/src/init.cpp
+3
View File
@@ -102,6 +102,7 @@ void initializeLayerFactory()
CV_DNN_REGISTER_LAYER_CLASS(LRN, LRNLayer);
CV_DNN_REGISTER_LAYER_CLASS(InnerProduct, InnerProductLayer);
CV_DNN_REGISTER_LAYER_CLASS(Gemm, GemmLayer);
CV_DNN_REGISTER_LAYER_CLASS(MatMul, MatMulLayer);
CV_DNN_REGISTER_LAYER_CLASS(Softmax, SoftmaxLayer);
CV_DNN_REGISTER_LAYER_CLASS(SoftMax, SoftmaxLayer); // For compatibility. See https://github.com/opencv/opencv/issues/16877
CV_DNN_REGISTER_LAYER_CLASS(MVN, MVNLayer);
@@ -160,6 +161,8 @@ void initializeLayerFactory()
CV_DNN_REGISTER_LAYER_CLASS(GatherElements, GatherElementsLayer);
CV_DNN_REGISTER_LAYER_CLASS(LayerNormalization, LayerNormLayer);
CV_DNN_REGISTER_LAYER_CLASS(Expand, ExpandLayer);
CV_DNN_REGISTER_LAYER_CLASS(InstanceNormalization, InstanceNormLayer);
CV_DNN_REGISTER_LAYER_CLASS(Attention, AttentionLayer);
CV_DNN_REGISTER_LAYER_CLASS(Crop, CropLayer);
CV_DNN_REGISTER_LAYER_CLASS(Eltwise, EltwiseLayer);
modules/dnn/src/layers/attention_layer.cpp
+272
View File
@@ -0,0 +1,272 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#include "../precomp.hpp"
#include "cpu_kernels/fast_gemm.hpp"
#include "cpu_kernels/softmax.hpp"
#include <opencv2/dnn/shape_utils.hpp>
namespace cv { namespace dnn {
static void packWeight(size_t num_heads, size_t head_size, size_t input_hidden_size,
const float *weight_data, size_t hidden_size, std::vector<float> &packed_weight, const FastGemmOpt &opt) {
// num_heads * pack(head_size, input_hidden_size)
size_t pack_size = fastGemmPackBSize(head_size, input_hidden_size, opt);
size_t packed_weight_size = num_heads * pack_size;
packed_weight.resize(packed_weight_size, 0.f);
auto *packed_weight_data = packed_weight.data();
for (size_t i = 0; i < num_heads; i++) {
fastGemmPackB(false, head_size, input_hidden_size, weight_data, hidden_size, packed_weight_data, opt);
packed_weight_data += pack_size;
weight_data += head_size;
}
}
// Operator spec: https://github.com/microsoft/onnxruntime/blob/v1.16.1/docs/ContribOperators.md#com.microsoft.Attention
class AttentionLayerImpl CV_FINAL : public AttentionLayer {
public:
AttentionLayerImpl(const LayerParams &params) {
setParamsFrom(params);
CV_CheckTrue(params.has("num_heads"), "DNN/Attention: num_heads is required but missing");
num_heads = params.get<int>("num_heads"); // required, no default value
CV_CheckTrue(params.has("qkv_hidden_sizes"), "DNN/Attention: qkv_hidden_sizes is required but missing");
auto param_qkv_hidden_sizes = params.get("qkv_hidden_sizes");
CV_CheckEQ(param_qkv_hidden_sizes.size(), 3, "DNN/Attention: qkv_hidden_sizes must and only have three elements");
qkv_hidden_sizes.clear();
qkv_hidden_sizes.resize(3);
qkv_hidden_sizes[0] = static_cast<size_t>(param_qkv_hidden_sizes.get<int>(0));
qkv_hidden_sizes[1] = static_cast<size_t>(param_qkv_hidden_sizes.get<int>(1));
/* v_hidden_size needs to be initialized in finalize in case v_slice_end=INT_MAX */
qkv_head_sizes.clear();
qkv_head_sizes.resize(3);
qkv_head_sizes[0] = static_cast<size_t>(qkv_hidden_sizes[0] / num_heads);
qkv_head_sizes[1] = static_cast<size_t>(qkv_hidden_sizes[1] / num_heads);
scale = 1.f / params.get<float>("scale", sqrt(qkv_head_sizes[0]));
output_ndims = params.get<int>("output_ndims", 3);
is_prepacked = false;
}
virtual bool supportBackend(int backendId) CV_OVERRIDE {
return backendId == DNN_BACKEND_OPENCV;
}
virtual bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int requiredOutputs,
std::vector<MatShape> &outputs,
std::vector<MatShape> &internals) const CV_OVERRIDE {
CV_CheckEQ(inputs.size(), static_cast<size_t>(3), "DNN/Attention: three inputs are required");
const auto &input_shape = inputs[0];
const auto &weight_shape = inputs[1];
const auto &bias_shape = inputs[2];
CV_CheckEQ(input_shape.size(), static_cast<size_t>(3), "DNN/Attention: invalid input dimension");
CV_CheckEQ(weight_shape.size(), static_cast<size_t>(2), "DNN/Attention: invalid weight dimension");
CV_CheckEQ(input_shape[2], weight_shape[0], "DNN/Attention: invalid input shape");
CV_CheckEQ(weight_shape[1], bias_shape[0], "DNN/Attention: invalid weight or bias shape");
if (output_ndims == 3) {
outputs.assign(1, inputs[0]);
} else if (output_ndims == 2) {
int batch = input_shape[0], seq_len = input_shape[1], input_hidden_size = input_shape[2];
MatShape output_shape{batch * seq_len, input_hidden_size};
outputs.assign(1, output_shape);
} else {
CV_Error(Error::StsBadArg, format("DNN/Attention: invalid output dimension %zu, valid value is 2 or 3", output_ndims));
}
return false;
}
virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE {
opt.init();
std::vector<Mat> inputs;
inputs_arr.getMatVector(inputs);
const auto input_shape = shape(inputs[0]);
batch_size = static_cast<size_t>(input_shape[0]);
seq_len = static_cast<size_t>(input_shape[1]);
input_hidden_size = static_cast<size_t>(input_shape[2]);
const auto weight_shape = shape(inputs[1]);
hidden_size = weight_shape[1];
qkv_hidden_sizes[2] = hidden_size - qkv_hidden_sizes[0] - qkv_hidden_sizes[1];
qkv_head_sizes[2] = static_cast<size_t>(qkv_hidden_sizes[2] / num_heads);
}
void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE {
CV_TRACE_FUNCTION();
CV_TRACE_ARG_VALUE(name, "name", name.c_str());
if (inputs_arr.depth() == CV_16S)
{
forward_fallback(inputs_arr, outputs_arr, internals_arr);
return;
}
std::vector<Mat> inputs, outputs;
inputs_arr.getMatVector(inputs);
outputs_arr.getMatVector(outputs);
// prepack weights
if (!is_prepacked) {
// prepack
const auto &weight = inputs[1];
const auto *weight_data = weight.ptr<const float>();
packWeight(num_heads, qkv_head_sizes[0], input_hidden_size, weight_data, hidden_size, packed_weight_q, opt);
packWeight(num_heads, qkv_head_sizes[1], input_hidden_size, weight_data + qkv_hidden_sizes[0], hidden_size, packed_weight_k, opt);
packWeight(num_heads, qkv_head_sizes[2], input_hidden_size, weight_data + qkv_hidden_sizes[0] + qkv_hidden_sizes[1], hidden_size, packed_weight_v, opt);
is_prepacked = true;
}
float *packed_weights[3] = {packed_weight_q.data(), packed_weight_k.data(), packed_weight_v.data()};
size_t packed_weights_size[3] = {packed_weight_q.size() / num_heads, packed_weight_k.size() / num_heads, packed_weight_v.size() / num_heads};
Mat gemm_buffer = Mat::zeros(1, int(batch_size * seq_len * hidden_size), CV_32F);
auto *Q = gemm_buffer.ptr<float>();
auto *K = Q + batch_size * seq_len * qkv_hidden_sizes[0];
auto *V = K + batch_size * seq_len * qkv_hidden_sizes[1];
float *QKV[3] = {Q, K, V}; // Q, K, V: [B, N, S, H]
{
const auto &input = inputs[0];
const auto &bias = inputs[2];
const auto *input_data = input.ptr<const float>();
const auto *bias_data = bias.ptr<const float>();
opt.multi_thread = false;
auto fn = [&](const Range &r) {
for (int i = r.start; i < r.end; i++) {
const int batch_index = static_cast<int>((i / 3) / num_heads);
const int head_index = static_cast<int>((i / 3) % num_heads);
const int qkv_index = static_cast<int>(i % 3);
auto *dst = QKV[qkv_index];
size_t head_size = qkv_head_sizes[qkv_index];
int input_offset = batch_index * seq_len * input_hidden_size;
int bias_offset = qkv_index * qkv_hidden_sizes[0] + head_index * head_size;
int dst_offset = (batch_index * num_heads + head_index) * (seq_len * head_size);
// broadcast bias ([NH] -> [BN, SH]) and make copy to dst
const auto *bias_data_src = bias_data + bias_offset;
auto *dst_data = dst + dst_offset;
for (size_t seq_len_idx = 0; seq_len_idx < seq_len; seq_len_idx++) {
std::memcpy(dst_data, bias_data_src, head_size * sizeof(float));
dst_data += head_size;
}
auto *packed_weight = packed_weights[qkv_index] + packed_weights_size[qkv_index] * head_index;
// single-thread gemm kernel
fastGemm(false, seq_len, head_size, input_hidden_size,
1.f, input_data + input_offset, input_hidden_size,
packed_weight, 1.f, dst + dst_offset, head_size, opt);
}
};
size_t loops = 3 * batch_size * num_heads;
double nstripes = loops * seq_len * qkv_head_sizes[0] * input_hidden_size * (1 / 1024.0);
parallel_for_(Range(0, loops), fn, nstripes);
}
// Compute softmax(scale * matmul(Q, K))
std::vector<int> attention_prob_shape{int(batch_size * num_heads), int(seq_len), int(seq_len)};
Mat attention_prob = Mat::zeros(attention_prob_shape.size(), attention_prob_shape.data(), CV_32F);
{
auto *output = attention_prob.ptr<float>();
auto loops = batch_size * num_heads;
auto seq_len_square = seq_len * seq_len;
auto qk_head_size = qkv_head_sizes[0];
auto qk_inner_size = seq_len * qk_head_size;
// Compute scale * matmul(Q, K)
opt.multi_thread = false;
parallel_for_(Range(0, loops), [&] (const Range r) {
for (int i = r.start; i < r.end; i++) {
const int output_offset = i * seq_len_square;
const auto *q = Q + qk_inner_size * i, *k = K + qk_inner_size * i;
fastGemm(false, true, seq_len, qk_head_size, seq_len, qk_head_size,
scale, q, qk_head_size, 1,
k, qk_head_size, 1, 0.f,
output + output_offset, seq_len, opt);
}
}, loops * seq_len * qk_head_size * seq_len * (1 / 1024.0));
// Compute softmax
softmax(attention_prob, attention_prob, attention_prob_shape.size() - 1);
}
// Compute np.matmul(attention_prob, V)
Mat output_buffer = Mat::zeros(1, int(batch_size * num_heads * seq_len * qkv_head_sizes[2]), CV_32F);
{
auto *output = outputs[0].ptr<float>();
auto *output_buff = output_buffer.ptr<float>();
const auto *prob = attention_prob.ptr<const float>();
auto loops = batch_size * num_heads;
auto prob_inner_size = seq_len * seq_len;
auto v_head_size = qkv_head_sizes[2];
auto v_inner_size = seq_len * v_head_size;
opt.multi_thread = false;
parallel_for_(Range(0, loops), [&] (const Range &r) {
for (int i = r.start; i < r.end; i++) {
const int output_offset = i * v_inner_size;
const auto *p = prob + i * prob_inner_size, *v = V + i * v_inner_size;
fastGemm(false, false, seq_len, seq_len, seq_len, v_head_size,
1.f, p, seq_len, 1,
v, v_head_size, 1, 0.f,
output_buff + output_offset, v_head_size, opt);
// tranpose on the fly
const int batch_index = static_cast<int>(i / num_heads);
const int head_index = static_cast<int>(i % num_heads);
auto *src = output_buff + output_offset;
auto *dst = output + (batch_index * seq_len * num_heads + head_index) * v_head_size;
for (int j = 0; j < seq_len; j++) {
std::memcpy(dst, src, v_head_size * sizeof(float));
src += v_head_size;
dst += qkv_hidden_sizes[2];
}
}
}, loops * seq_len * seq_len * v_head_size * (1 / 1024.0));
}
}
private:
size_t num_heads;
std::vector<size_t> qkv_hidden_sizes; // order: {qk_hidden_size, qk_hidden_size, v_hidden_size}
float scale;
size_t output_ndims;
std::vector<size_t> qkv_head_sizes; // order: {qk_head_size, qk_head_size, v_head_size}
size_t batch_size;
size_t seq_len;
size_t input_hidden_size;
size_t hidden_size;
bool is_prepacked;
std::vector<float> packed_weight_q;
std::vector<float> packed_weight_k;
std::vector<float> packed_weight_v;
FastGemmOpt opt;
};
Ptr<AttentionLayer> AttentionLayer::create(const LayerParams &params) {
return makePtr<AttentionLayerImpl>(params);
}
}} // cv::dnn
modules/dnn/src/layers/cpu_kernels/conv_block.simd.hpp
+112 -145
View File
@@ -8,16 +8,26 @@ namespace cv {
namespace dnn {
CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
void convBlock(int np, const float* a, const float* b, float* c, int ldc, bool init_c, int width, const int convMR, const int convNR);
void convBlock_F32(int np, const float* a, const float* b, float* c, int ldc, bool init_c, int width, const int convMR, const int convNR);
#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) && CV_AVX
// FP 16 branch.
void convBlock_F16(int np, const char * _a, const char * _b, char * _c, int ldc, bool init_c, int width,
const int convMR_fp16, const int convNR_fp16);
void convBlockMR1_F16(int np, const char* _a, const char* _b, float *c, const float _bias, bool init_c,
const float minval, const float maxval, bool ifMinMaxAct, const int width, const int convNR_FP16);
#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY)
#if CV_AVX
#if !CV_FMA3 // AVX workaround
#undef _mm256_fmadd_ps
#define _mm256_fmadd_ps(a, b, c) _mm256_add_ps(c, _mm256_mul_ps(a, b))
#endif
void convBlock(int np, const float* a, const float* b, float* c, int ldc, bool init_c, int width, const int convMR, const int convNR)
void convBlock_F32(int np, const float* a, const float* b, float* c, int ldc, bool init_c, int width, const int convMR, const int convNR)
{
CV_Assert(convMR == 4 && convNR == 24);
__m256 c00 = _mm256_set1_ps(0.f), c01 = c00, c02 = c00;
@@ -121,16 +131,11 @@ void convBlock(int np, const float* a, const float* b, float* c, int ldc, bool i
_mm256_zeroupper();
}
#endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
#endif
CV_CPU_OPTIMIZATION_NAMESPACE_END
#if CV_NEON
// NEON code work around.
namespace opt_NEON
{
#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) && CV_NEON
void convBlock(int np, const float* a, const float* b, float* c, int ldc, bool init_c, int width, const int convMR, const int convNR)
void convBlock_F32(int np, const float* a, const float* b, float* c, int ldc, bool init_c, int width, const int convMR, const int convNR)
{
#if CV_NEON_AARCH64
if (convMR == 4 && convNR == 28) // AARCH64
@@ -298,104 +303,104 @@ void convBlock(int np, const float* a, const float* b, float* c, int ldc, bool i
}
else
#endif
if (convMR == 4 && convNR == 12) // ARMv7
{
float32x4_t c0 = vdupq_n_f32(0.f), c1 = c0, c2 = c0;
float32x4_t c3 = vdupq_n_f32(0.f), c4 = c3, c5 = c3;
float32x4_t c6 = vdupq_n_f32(0.f), c7 = c6, c8 = c6;
float32x4_t c9 = vdupq_n_f32(0.f), c10 = c9, c11 = c9;
float32x2_t a0 = vdup_n_f32(0.0f), a1 = a0;
float32x4_t b0 = vdupq_n_f32(0.0f), b1 = vdupq_n_f32(0.0f), b2 = vdupq_n_f32(0.0f);
if (width > 8)
if (convMR == 4 && convNR == 12) // ARMv7
{
for (int p = 0; p < np; p++, a += convMR, b += convNR)
float32x4_t c0 = vdupq_n_f32(0.f), c1 = c0, c2 = c0;
float32x4_t c3 = vdupq_n_f32(0.f), c4 = c3, c5 = c3;
float32x4_t c6 = vdupq_n_f32(0.f), c7 = c6, c8 = c6;
float32x4_t c9 = vdupq_n_f32(0.f), c10 = c9, c11 = c9;
float32x2_t a0 = vdup_n_f32(0.0f), a1 = a0;
float32x4_t b0 = vdupq_n_f32(0.0f), b1 = vdupq_n_f32(0.0f), b2 = vdupq_n_f32(0.0f);
if (width > 8)
{
a0 = vld1_f32(a), a1 = vld1_f32(a+2);
b0 = vld1q_f32(b), b1 = vld1q_f32(b + 4), b2 = vld1q_f32(b + 8);
for (int p = 0; p < np; p++, a += convMR, b += convNR)
{
a0 = vld1_f32(a), a1 = vld1_f32(a+2);
b0 = vld1q_f32(b), b1 = vld1q_f32(b + 4), b2 = vld1q_f32(b + 8);
c0 = vmlaq_lane_f32(c0, b0, a0, 0);
c1 = vmlaq_lane_f32(c1, b1, a0, 0);
c2 = vmlaq_lane_f32(c2, b2, a0, 0);
c0 = vmlaq_lane_f32(c0, b0, a0, 0);
c1 = vmlaq_lane_f32(c1, b1, a0, 0);
c2 = vmlaq_lane_f32(c2, b2, a0, 0);
c3 = vmlaq_lane_f32(c3, b0, a0, 1);
c4 = vmlaq_lane_f32(c4, b1, a0, 1);
c5 = vmlaq_lane_f32(c5, b2, a0, 1);
c3 = vmlaq_lane_f32(c3, b0, a0, 1);
c4 = vmlaq_lane_f32(c4, b1, a0, 1);
c5 = vmlaq_lane_f32(c5, b2, a0, 1);
c6 = vmlaq_lane_f32(c6, b0, a1, 0);
c7 = vmlaq_lane_f32(c7, b1, a1, 0);
c8 = vmlaq_lane_f32(c8, b2, a1, 0);
c6 = vmlaq_lane_f32(c6, b0, a1, 0);
c7 = vmlaq_lane_f32(c7, b1, a1, 0);
c8 = vmlaq_lane_f32(c8, b2, a1, 0);
c9 = vmlaq_lane_f32(c9 , b0, a1, 1);
c10 = vmlaq_lane_f32(c10, b1, a1, 1);
c11 = vmlaq_lane_f32(c11, b2, a1, 1);
c9 = vmlaq_lane_f32(c9 , b0, a1, 1);
c10 = vmlaq_lane_f32(c10, b1, a1, 1);
c11 = vmlaq_lane_f32(c11, b2, a1, 1);
}
}
}
else if (width > 4)
{
for (int p = 0; p < np; p++, a += convMR, b += convNR)
else if (width > 4)
{
a0 = vld1_f32(a), a1 = vld1_f32(a+2);
b0 = vld1q_f32(b), b1 = vld1q_f32(b + 4);
for (int p = 0; p < np; p++, a += convMR, b += convNR)
{
a0 = vld1_f32(a), a1 = vld1_f32(a+2);
b0 = vld1q_f32(b), b1 = vld1q_f32(b + 4);
c0 = vmlaq_lane_f32(c0, b0, a0, 0);
c1 = vmlaq_lane_f32(c1, b1, a0, 0);
c0 = vmlaq_lane_f32(c0, b0, a0, 0);
c1 = vmlaq_lane_f32(c1, b1, a0, 0);
c3 = vmlaq_lane_f32(c3, b0, a0, 1);
c4 = vmlaq_lane_f32(c4, b1, a0, 1);
c3 = vmlaq_lane_f32(c3, b0, a0, 1);
c4 = vmlaq_lane_f32(c4, b1, a0, 1);
c6 = vmlaq_lane_f32(c6, b0, a1, 0);
c7 = vmlaq_lane_f32(c7, b1, a1, 0);
c6 = vmlaq_lane_f32(c6, b0, a1, 0);
c7 = vmlaq_lane_f32(c7, b1, a1, 0);
c9 = vmlaq_lane_f32(c9 , b0, a1, 1);
c10 = vmlaq_lane_f32(c10, b1, a1, 1);
c9 = vmlaq_lane_f32(c9 , b0, a1, 1);
c10 = vmlaq_lane_f32(c10, b1, a1, 1);
}
}
else
{
for (int p = 0; p < np; p++, a += convMR, b += convNR)
{
a0 = vld1_f32(a), a1 = vld1_f32(a+2);
b0 = vld1q_f32(b);
c0 = vmlaq_lane_f32(c0, b0, a0, 0);
c3 = vmlaq_lane_f32(c3, b0, a0, 1);
c6 = vmlaq_lane_f32(c6, b0, a1, 0);
c9 = vmlaq_lane_f32(c9 , b0, a1, 1);
}
}
if (!init_c)
{
c0 = vaddq_f32(c0, vld1q_f32(c));
c1 = vaddq_f32(c1, vld1q_f32(c + 4));
c2 = vaddq_f32(c2, vld1q_f32(c + 8));
c3 = vaddq_f32(c3, vld1q_f32(c + ldc));
c4 = vaddq_f32(c4, vld1q_f32(c + ldc + 4));
c5 = vaddq_f32(c5, vld1q_f32(c + ldc + 8));
c6 = vaddq_f32(c6, vld1q_f32(c + ldc * 2));
c7 = vaddq_f32(c7, vld1q_f32(c + ldc * 2 + 4));
c8 = vaddq_f32(c8, vld1q_f32(c + ldc * 2 + 8));
c9 = vaddq_f32(c9 , vld1q_f32(c + ldc * 3));
c10 = vaddq_f32(c10, vld1q_f32(c + ldc * 3 + 4));
c11 = vaddq_f32(c11, vld1q_f32(c + ldc * 3 + 8));
}
vst1q_f32(c, c0), vst1q_f32(c+4, c1), vst1q_f32(c+8, c2);
vst1q_f32(c + ldc, c3), vst1q_f32(c + ldc + 4, c4), vst1q_f32(c + ldc + 8, c5);
vst1q_f32(c + ldc*2, c6), vst1q_f32(c + ldc*2 + 4, c7), vst1q_f32(c + ldc*2 + 8, c8);
vst1q_f32(c + ldc*3, c9), vst1q_f32(c + ldc*3 + 4, c10), vst1q_f32(c + ldc*3 + 8, c11);
}
else
{
for (int p = 0; p < np; p++, a += convMR, b += convNR)
{
a0 = vld1_f32(a), a1 = vld1_f32(a+2);
b0 = vld1q_f32(b);
c0 = vmlaq_lane_f32(c0, b0, a0, 0);
c3 = vmlaq_lane_f32(c3, b0, a0, 1);
c6 = vmlaq_lane_f32(c6, b0, a1, 0);
c9 = vmlaq_lane_f32(c9 , b0, a1, 1);
}
}
if (!init_c)
{
c0 = vaddq_f32(c0, vld1q_f32(c));
c1 = vaddq_f32(c1, vld1q_f32(c + 4));
c2 = vaddq_f32(c2, vld1q_f32(c + 8));
c3 = vaddq_f32(c3, vld1q_f32(c + ldc));
c4 = vaddq_f32(c4, vld1q_f32(c + ldc + 4));
c5 = vaddq_f32(c5, vld1q_f32(c + ldc + 8));
c6 = vaddq_f32(c6, vld1q_f32(c + ldc * 2));
c7 = vaddq_f32(c7, vld1q_f32(c + ldc * 2 + 4));
c8 = vaddq_f32(c8, vld1q_f32(c + ldc * 2 + 8));
c9 = vaddq_f32(c9 , vld1q_f32(c + ldc * 3));
c10 = vaddq_f32(c10, vld1q_f32(c + ldc * 3 + 4));
c11 = vaddq_f32(c11, vld1q_f32(c + ldc * 3 + 8));
}
vst1q_f32(c, c0), vst1q_f32(c+4, c1), vst1q_f32(c+8, c2);
vst1q_f32(c + ldc, c3), vst1q_f32(c + ldc + 4, c4), vst1q_f32(c + ldc + 8, c5);
vst1q_f32(c + ldc*2, c6), vst1q_f32(c + ldc*2 + 4, c7), vst1q_f32(c + ldc*2 + 8, c8);
vst1q_f32(c + ldc*3, c9), vst1q_f32(c + ldc*3 + 4, c10), vst1q_f32(c + ldc*3 + 8, c11);
}
else
CV_Error(Error::StsNotImplemented, "Unsupported convMR and/or convNR in opt_NEON::convBlock");
CV_Error(Error::StsNotImplemented, "Unsupported convMR and/or convNR in opt_NEON::convBlock");
}
void convBlockMR1_F32(int np, const float * a, const float * b, float *c, const float bias, bool init_c,
const float minval, const float maxval, bool ifMinMaxAct, const int width, const int convNR)
const float minval, const float maxval, bool ifMinMaxAct, const int width, const int convNR)
{
CV_Assert(convNR == 28);
float32x4_t c0 = vdupq_n_f32(bias), c1 = c0, c2 = c0;
@@ -482,22 +487,17 @@ void convBlockMR1_F32(int np, const float * a, const float * b, float *c, const
vst1q_f32(c + 20, c5);
vst1q_f32(c + 24, c6);
}
#if CV_NEON_AARCH64 && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
// Fix conflict between float16_t in arm_neon.h and float16_t in cvdef.h.
typedef __fp16 float16_t;
#ifndef __ARM_FEATURE_FMA // Work around without FMA support.
#define vfmaq_f16(a, b, c) (a + b * c)
#endif
void convBlock_FP16(int np, const char * _a, const char * _b, char * _c, int ldc, bool init_c, int width,
#if defined(CV_NEON_AARCH64) && CV_NEON_AARCH64 && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
void convBlock_F16(int np, const char * _a, const char * _b, char * _c, int ldc, bool init_c, int width,
const int convMR_fp16, const int convNR_fp16)
{
#if 1
typedef __fp16 float16_t;
const float16_t* a = (const float16_t*)_a;
const float16_t* b = (const float16_t*)_b;
float16_t* c = (float16_t*)_c;
CV_Assert(convMR_fp16 == 8 && convNR_fp16 == 24);
float16x8_t c00 = vdupq_n_f16(0), c01 = c00, c02 = c00;
@@ -603,8 +603,8 @@ void convBlock_FP16(int np, const char * _a, const char * _b, char * _c, int ldc
if (!init_c)
{
#undef _FX_UPDATE_CBUF_ROW
#define _FX_UPDATE_CBUF_ROW(row) \
#undef _FX_UPDATE_CBUF_ROW
#define _FX_UPDATE_CBUF_ROW(row) \
c##row##0 = c##row##0 + vld1q_f16(c + row*ldc); \
c##row##1 = c##row##1 + vld1q_f16(c + row*ldc + 8); \
c##row##2 = c##row##2 + vld1q_f16(c + row*ldc + 16)
@@ -619,8 +619,8 @@ void convBlock_FP16(int np, const char * _a, const char * _b, char * _c, int ldc
_FX_UPDATE_CBUF_ROW(7);
}
#undef _FX_STORE_CBUF_ROW
#define _FX_STORE_CBUF_ROW(row) \
#undef _FX_STORE_CBUF_ROW
#define _FX_STORE_CBUF_ROW(row) \
vst1q_f16(c + row*ldc, c##row##0); \
vst1q_f16(c + row*ldc + 8, c##row##1); \
vst1q_f16(c + row*ldc + 16, c##row##2)
@@ -633,46 +633,12 @@ void convBlock_FP16(int np, const char * _a, const char * _b, char * _c, int ldc
_FX_STORE_CBUF_ROW(5);
_FX_STORE_CBUF_ROW(6);
_FX_STORE_CBUF_ROW(7);
#else
// reference only.
const float16_t* a = (const float16_t*)_a;
const float16_t* b = (const float16_t*)_b;
float16_t* c = (float16_t*)_c;
float cbuf[convMR_fp16*convNR_fp16];
memset(cbuf, 0, sizeof(cbuf));
for( int p = 0; p < np; p++ )
{
for( int i = 0; i < convMR_fp16; i++ )
{
float ai = float(a[convMR_fp16*p + i]);
for( int j = 0; j < convNR_fp16; j++ )
cbuf[i*convNR_fp16+j] += float(b[convNR_fp16*p + j]) * ai;
}
}
if (!init_c)
{
for(int i = 0; i < convMR_fp16; i++)
{
for(int j = 0; j < convNR_fp16; j++)
c[i*ldc + j] = float16_t(float(c[i*ldc + j]) + cbuf[i*convNR_fp16 + j]);
}
}
else
{
for(int i = 0; i < convMR_fp16; i++)
{
for(int j = 0; j < convNR_fp16; j++)
c[i*ldc + j] = (float16_t)(cbuf[i*convNR_fp16 + j]);
}
}
#endif
}
void convBlockMR1_FP16(int np, const char* _a, const char* _b, float *c, const float _bias, bool init_c,
const float minval, const float maxval, bool ifMinMaxAct, const int width, const int convNR_FP16)
void convBlockMR1_F16(int np, const char* _a, const char* _b, float *c, const float _bias, bool init_c,
const float minval, const float maxval, bool ifMinMaxAct, const int width, const int convNR_FP16)
{
typedef __fp16 float16_t;
CV_Assert(convNR_FP16 == 24); // CONV_NR_FP16 = 24
const float16_t* a = (const float16_t*)_a;
const float16_t* b = (const float16_t*)_b;
@@ -685,7 +651,7 @@ void convBlockMR1_FP16(int np, const char* _a, const char* _b, float *c, const f
{
for (int p = 0; p < np; p++, a++, b += convNR_FP16)
{
float16x8_t a0= vdupq_n_f16(a[0]);
float16x8_t a0 = vdupq_n_f16(a[0]);
float16x8_t b0 = vld1q_f16(b), b1 = vld1q_f16(b + 8), b2 = vld1q_f16(b + 16);
c0 = vfmaq_f16(c0, a0, b0);
@@ -754,6 +720,7 @@ void convBlockMR1_FP16(int np, const char* _a, const char* _b, float *c, const f
}
#endif
#endif
}
#endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
CV_CPU_OPTIMIZATION_NAMESPACE_END
}} // namespace cv::dnn
modules/dnn/src/layers/cpu_kernels/conv_depthwise.cpp
+1 -1
View File
@@ -92,7 +92,7 @@ void runDepthwise(InputArray _input, OutputArray _output, const Ptr<FastConv>& c
ofstab[k] = dy * Wi + dx;
}
const float *weights0 = conv->weightsBufPtr, *bias = conv->biasBuf.data();
const float *weights0 = conv->getWeights(), *bias = conv->biasBuf.data();
const float* relu = reluslope.data();
CV_Assert(ksize > 1 || (pad_left == 0 && pad_right == 0 && pad_top == 0 && pad_bottom == 0));
modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.cpp
+115 -40
View File
@@ -20,15 +20,15 @@ namespace cv { namespace dnn {
#if CV_NEON || CV_SIMD128 || CV_TRY_AVX2
enum { VEC_ALIGN = 32, DFT_TYPE = CV_32F }; // Memory alignment.
void winofunc_accum_f32(const float* inwptr, const float* wptr, float* outbuf, int Cg, int iblock,
void winofunc_accum_F32(const float* inwptr, const float* wptr, float* outbuf, int Cg, int iblock,
const int winoIblock, const int winoKblock, const int winoAtomF32, const int winoNatomF32);
/*Input transform*/
void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep,
void winofunc_BtXB_8x8_F32(const float* inptr, int inpstep,
float* outptr, int Cg, const int winoIblock, const int winoAtomF32);
/*Output transform*/
void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep, float* bpptr, int bpstep, float* outptr, int outstep,
void winofunc_AtXA_8x8_F32(const float* inptr, int inpstep, float* bpptr, int bpstep, float* outptr, int outstep,
float bias, float minval, float maxval, bool ifMinMaxAct);
int runWinograd63(InputArray _input, InputArray _fusedAddMat, OutputArray _output, const Ptr<FastConv>& conv,
@@ -67,6 +67,28 @@ int runWinograd63(InputArray _input, InputArray _fusedAddMat, OutputArray _outpu
#endif
const int CONV_WINO_NATOMS_F32 = CONV_WINO_AREA / CONV_WINO_ATOM_F32; // for AVX2, it is 8, otherwise, it's 16.
int CONV_WINO_ATOM = CONV_WINO_ATOM_F32;
int CONV_WINO_NATOMS = CONV_WINO_NATOMS_F32;
#ifdef CONV_ARM_FP16
// FP 16
const int CONV_WINO_ATOM_F16 = CONV_WINO_ATOM_F32 * 2;
const int CONV_WINO_NATOMS_F16 = CONV_WINO_AREA / CONV_WINO_ATOM_F16;
#endif
int esz = sizeof(float );
#ifdef CONV_ARM_FP16
const bool useFP16 = conv->useFP16;
if (useFP16)
{
// works at FP 16.
CONV_WINO_ATOM = CONV_WINO_ATOM_F16;
CONV_WINO_NATOMS = CONV_WINO_NATOMS_F16;
esz = sizeof(float16_t);
}
#endif
int Kg_nblocks = (Kg + CONV_WINO_KBLOCK - 1)/CONV_WINO_KBLOCK;
const size_t inp_planesize = (size_t)Hi*Wi;
const size_t out_planesize = (size_t)H0*W0;
@@ -78,9 +100,9 @@ int runWinograd63(InputArray _input, InputArray _fusedAddMat, OutputArray _outpu
size_t totalbufsize = (size_t)N*C*blocks_per_plane_aligned*CONV_WINO_AREA;
AutoBuffer<float> _buf;
_buf.allocate(totalbufsize + VEC_ALIGN);
float* wbuf_all = alignPtr(_buf.data(), VEC_ALIGN);
AutoBuffer<char> _buf;
_buf.allocate((totalbufsize + VEC_ALIGN) * esz);
char* wbuf_all = alignPtr(_buf.data(), VEC_ALIGN * esz);
float* inp = input.ptr<float>();
float* out = output.ptr<float>();
@@ -104,14 +126,15 @@ int runWinograd63(InputArray _input, InputArray _fusedAddMat, OutputArray _outpu
int c = nc0 - n*C;
int g = c / Cg;
c -= g*Cg;
for (int block_id = 0; block_id < blocks_per_plane; block_id += CONV_WINO_IBLOCK)
{
for (int db = 0; db < CONV_WINO_IBLOCK; db++)
{
size_t inwofs = ((n*ngroups + g)*blocks_per_plane_aligned +
block_id)*Cg*CONV_WINO_AREA +
(c*CONV_WINO_IBLOCK + db)*CONV_WINO_ATOM_F32;
float* inwptr = (float*)wbuf_all + inwofs;
(c*CONV_WINO_IBLOCK + db) * CONV_WINO_ATOM;
char* inwptr = wbuf_all + inwofs * esz;
if (block_id + db < blocks_per_plane)
{
@@ -152,27 +175,40 @@ int runWinograd63(InputArray _input, InputArray _fusedAddMat, OutputArray _outpu
inptr = inpbuf;
inpstep = CONV_WINO_SIZE;
}
#if CV_TRY_AVX2
if (conv->useAVX2)
opt_AVX2::winofunc_BtXB_8x8_f32(inptr, inpstep, inwptr, Cg, CONV_WINO_IBLOCK, CONV_WINO_ATOM_F32);
opt_AVX2::winofunc_BtXB_8x8_F32(inptr, inpstep, (float *)inwptr, Cg, CONV_WINO_IBLOCK, CONV_WINO_ATOM);
else
#endif
#if CV_TRY_AVX
if (conv->useAVX)
opt_AVX::winofunc_BtXB_8x8_f32(inptr, inpstep, inwptr, Cg, CONV_WINO_IBLOCK, CONV_WINO_ATOM_F32);
opt_AVX::winofunc_BtXB_8x8_F32(inptr, inpstep, (float *)inwptr, Cg, CONV_WINO_IBLOCK, CONV_WINO_ATOM);
else
#endif
#if CV_NEON && CV_NEON_AARCH64
if (conv->useNEON)
opt_NEON::winofunc_BtXB_8x8_f32(inptr, inpstep, inwptr, Cg, CONV_WINO_IBLOCK, CONV_WINO_ATOM_F32);
{
#ifdef CONV_ARM_FP16
if (useFP16)
{
opt_NEON_FP16::winofunc_BtXB_8x8_F16(inptr, inpstep, inwptr, Cg, CONV_WINO_IBLOCK,
CONV_WINO_ATOM);
}
else
#endif
opt_NEON::winofunc_BtXB_8x8_F32(inptr, inpstep, (float *)inwptr, Cg, CONV_WINO_IBLOCK,
CONV_WINO_ATOM);
}
else
#endif
winofunc_BtXB_8x8_f32(inptr, inpstep, inwptr, Cg, CONV_WINO_IBLOCK, CONV_WINO_ATOM_F32);
winofunc_BtXB_8x8_F32(inptr, inpstep, (float *)inwptr, Cg, CONV_WINO_IBLOCK, CONV_WINO_ATOM);
}
else
{
for (int i = 0; i < CONV_WINO_NATOMS_F32; i++, inwptr += CONV_WINO_IBLOCK*CONV_WINO_ATOM_F32)
memset(inwptr, 0, CONV_WINO_ATOM_F32*sizeof(inwptr[0]));
for (int i = 0; i < CONV_WINO_NATOMS; i++, inwptr += CONV_WINO_IBLOCK * CONV_WINO_ATOM * esz)
memset(inwptr, 0, CONV_WINO_ATOM * esz);
}
}
}
@@ -182,19 +218,37 @@ int runWinograd63(InputArray _input, InputArray _fusedAddMat, OutputArray _outpu
// Phase 2. compute elemwise-weighted sums of transformed blocks,
// apply inverse Winograd transforms to the sums,
// add bias, apply activation function if any and store the results.
char* wptr0 = nullptr;
#ifdef CONV_ARM_FP16
if (useFP16)
{
CV_Assert(!conv->weightsWinoBuf_FP16.empty());
wptr0 = (char *)conv->getWeightsWinoFP16();
}
else
#endif
{
CV_Assert(!conv->weightsWinoBuf.empty());
wptr0 = (char *)conv->getWeightsWino();
}
parallel_for_(Range(0, ntasks), [&](const Range& r0) {
for (int task_id = r0.start; task_id < r0.end; task_id++)
{
size_t out_wbuf_size = CONV_WINO_AREA*CONV_WINO_KBLOCK*CONV_WINO_IBLOCK;
size_t out_wbuf_size = CONV_WINO_AREA * CONV_WINO_KBLOCK * CONV_WINO_IBLOCK;
size_t outbuf_size = CONV_WINO_AREA;
AutoBuffer<float> out_wbuf_, outbuf_;
out_wbuf_.allocate(out_wbuf_size + VEC_ALIGN);
float* out_wbuf = alignPtr(out_wbuf_.data(), VEC_ALIGN);
// For saving the accumulation output.
AutoBuffer<char> out_wbuf_;
out_wbuf_.allocate((out_wbuf_size + VEC_ALIGN) * esz);
char* out_wbuf = alignPtr(out_wbuf_.data(), VEC_ALIGN * esz);
memset(out_wbuf, 0, out_wbuf_size * esz);
// For saving the fuse_Add data.
AutoBuffer<float> outbuf_;
outbuf_.allocate(outbuf_size + VEC_ALIGN);
float* outbuf = alignPtr(outbuf_.data(), VEC_ALIGN);
memset(out_wbuf, 0, out_wbuf_size * sizeof(float));
memset(outbuf, 0, outbuf_size * sizeof(float));
memset(outbuf, 0, outbuf_size * sizeof(outbuf[0]));
int ngk0 = (int)(((int64_t)N*Kg_nblocks*ngroups)*task_id/ntasks);
int ngk1 = (int)(((int64_t)N*Kg_nblocks*ngroups)*(task_id+1)/ntasks);
@@ -214,30 +268,40 @@ int runWinograd63(InputArray _input, InputArray _fusedAddMat, OutputArray _outpu
size_t inwofs = ((n*ngroups + g)*blocks_per_plane_aligned + block_id0)*Cg*CONV_WINO_AREA;
size_t wofs = (g*Kg_nblocks*CONV_WINO_KBLOCK + k0)*Cg*CONV_WINO_AREA;
float* inwptr = wbuf_all + inwofs;
const float* wptr = conv->weightsWinoBufPtr + wofs;
char* inwptr = wbuf_all + inwofs * esz;
char* wptr = wptr0 + wofs * esz;
#if CV_TRY_AVX2
if (conv->useAVX2)
opt_AVX2::winofunc_accum_f32(inwptr, wptr, out_wbuf, Cg, block_id1 - block_id0, CONV_WINO_IBLOCK,
CONV_WINO_KBLOCK, CONV_WINO_ATOM_F32, CONV_WINO_NATOMS_F32);
opt_AVX2::winofunc_accum_F32((float *)inwptr, (float *)wptr, (float *)out_wbuf, Cg, block_id1 - block_id0, CONV_WINO_IBLOCK,
CONV_WINO_KBLOCK, CONV_WINO_ATOM, CONV_WINO_NATOMS);
else
#endif
#if CV_TRY_AVX
if (conv->useAVX)
opt_AVX::winofunc_accum_f32(inwptr, wptr, out_wbuf, Cg, block_id1 - block_id0, CONV_WINO_IBLOCK,
CONV_WINO_KBLOCK, CONV_WINO_ATOM_F32, CONV_WINO_NATOMS_F32);
opt_AVX::winofunc_accum_F32((float *)inwptr, (float *)wptr, (float *)out_wbuf, Cg, block_id1 - block_id0, CONV_WINO_IBLOCK,
CONV_WINO_KBLOCK, CONV_WINO_ATOM, CONV_WINO_NATOMS);
else
#endif
#if CV_NEON && CV_NEON_AARCH64
if (conv->useNEON)
opt_NEON::winofunc_accum_f32(inwptr, wptr, out_wbuf, Cg, block_id1 - block_id0, CONV_WINO_IBLOCK,
CONV_WINO_KBLOCK, CONV_WINO_ATOM_F32, CONV_WINO_NATOMS_F32);
{
#ifdef CONV_ARM_FP16
if (useFP16)
{
opt_NEON_FP16::winofunc_accum_F16(inwptr, wptr, out_wbuf, Cg, block_id1 - block_id0, CONV_WINO_IBLOCK,
CONV_WINO_KBLOCK, CONV_WINO_ATOM, CONV_WINO_NATOMS);
}
else
#endif
opt_NEON::winofunc_accum_F32((float *)inwptr, (float *)wptr, (float *)out_wbuf, Cg, block_id1 - block_id0, CONV_WINO_IBLOCK,
CONV_WINO_KBLOCK, CONV_WINO_ATOM, CONV_WINO_NATOMS);
}
else
#endif
winofunc_accum_F32((float *)inwptr, (float *)wptr, (float *)out_wbuf, Cg, block_id1 - block_id0, CONV_WINO_IBLOCK,
CONV_WINO_KBLOCK, CONV_WINO_ATOM, CONV_WINO_NATOMS);
winofunc_accum_f32(inwptr, wptr, out_wbuf, Cg, block_id1 - block_id0, CONV_WINO_IBLOCK,
CONV_WINO_KBLOCK, CONV_WINO_ATOM_F32, CONV_WINO_NATOMS_F32);
for (int k = k0; k < k1; k++)
{
float biasv = conv->biasBuf[g*Kg + k];
@@ -274,31 +338,42 @@ int runWinograd63(InputArray _input, InputArray _fusedAddMat, OutputArray _outpu
}
#if CV_TRY_AVX2
if (conv->useAVX2)
opt_AVX::winofunc_AtXA_8x8_f32(out_wbuf + ((k - k0)*CONV_WINO_IBLOCK + (block_id - block_id0))*CONV_WINO_AREA, CONV_WINO_SIZE,
opt_AVX::winofunc_AtXA_8x8_F32((float *)out_wbuf + ((k - k0)*CONV_WINO_IBLOCK + (block_id - block_id0))*CONV_WINO_AREA, CONV_WINO_SIZE,
bpptr, outstep, outptr, outstep, biasv, minval, maxval, ifMinMaxAct);
else
#endif
#if CV_TRY_AVX
if (conv->useAVX)
opt_AVX::winofunc_AtXA_8x8_f32(out_wbuf + ((k - k0)*CONV_WINO_IBLOCK + (block_id - block_id0))*CONV_WINO_AREA, CONV_WINO_SIZE,
opt_AVX::winofunc_AtXA_8x8_F32((float *)out_wbuf + ((k - k0)*CONV_WINO_IBLOCK + (block_id - block_id0))*CONV_WINO_AREA, CONV_WINO_SIZE,
bpptr, outstep, outptr, outstep, biasv, minval, maxval, ifMinMaxAct);
else
#endif
#if CV_NEON && CV_NEON_AARCH64
// NEON optimization is only for ARMv8 device, and for ARMv7 device, we use the Universal intrinsics.
if (conv->useNEON)
// NEON optimization is only for ARMv8 device, and for ARMv7 device, we use the Universal intrinsics.
opt_NEON::winofunc_AtXA_8x8_f32(out_wbuf + ((k - k0)*CONV_WINO_IBLOCK + (block_id - block_id0))*CONV_WINO_AREA, CONV_WINO_SIZE,
{
#ifdef CONV_ARM_FP16
if (useFP16)
{
opt_NEON_FP16::winofunc_AtXA_8x8_F16(out_wbuf + ((k - k0)*CONV_WINO_IBLOCK + (block_id - block_id0))*CONV_WINO_AREA * esz, CONV_WINO_SIZE,
bpptr, outstep, outptr, outstep, biasv, minval, maxval, ifMinMaxAct);
}
else
#endif
opt_NEON::winofunc_AtXA_8x8_F32((float *)out_wbuf + ((k - k0)*CONV_WINO_IBLOCK + (block_id - block_id0))*CONV_WINO_AREA, CONV_WINO_SIZE,
bpptr, outstep, outptr, outstep, biasv, minval, maxval, ifMinMaxAct);
}
else
#endif
winofunc_AtXA_8x8_f32(out_wbuf + ((k - k0)*CONV_WINO_IBLOCK + (block_id - block_id0))*CONV_WINO_AREA, CONV_WINO_SIZE,
winofunc_AtXA_8x8_F32((float *)out_wbuf + ((k - k0)*CONV_WINO_IBLOCK + (block_id - block_id0))*CONV_WINO_AREA, CONV_WINO_SIZE,
bpptr, outstep, outptr, outstep, biasv, minval, maxval, ifMinMaxAct);
if (partial)
{
if (activ)
activ->forwardSlice(outptr, outptr, CONV_WINO_SIZE*CONV_WINO_STEP, 0, g*Kg + k, g*Kg + k + 1);
for (int y = 0; y < dy1; y++)
memcpy(outptr0 + y*W0, outptr + y*CONV_WINO_SIZE,dx1*sizeof(outptr0[0]));
memcpy(outptr0 + y*W0, outptr + y*CONV_WINO_SIZE, dx1*sizeof(outptr0[0]));
}
}
}
@@ -314,7 +389,7 @@ int runWinograd63(InputArray _input, InputArray _fusedAddMat, OutputArray _outpu
#if CV_SIMD128
void winofunc_accum_f32(const float* inwptr, const float* wptr, float* outbuf, int Cg, int iblock,
void winofunc_accum_F32(const float* inwptr, const float* wptr, float* outbuf, int Cg, int iblock,
const int winoIblock, const int winoKblock, const int winoAtomF32, const int winoNatomF32)
{
#if 1
@@ -411,7 +486,7 @@ void winofunc_accum_f32(const float* inwptr, const float* wptr, float* outbuf, i
}
/*Input transform*/
void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep,
void winofunc_BtXB_8x8_F32(const float* inptr, int inpstep,
float* outptr, int Cg, const int winoIblock, const int winoAtomF32)
{
CV_Assert(winoIblock == 3 && winoAtomF32 == 4);
@@ -585,7 +660,7 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep,
the Winograd-transformed weights should also be transposed.
init_conv() (see OpConv.fx) takes care of that.
*/
void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep,
void winofunc_AtXA_8x8_F32(const float* inptr, int inpstep,
float* bpptr, int bpstep, float* outptr, int outstep,
float bias, float minval, float maxval, bool ifMinMaxAct)
{
modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.neon.cpp
+476
View File
@@ -0,0 +1,476 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#include "../../precomp.hpp"
#include "convolution.hpp"
#include "opencv2/core/hal/intrin.hpp"
namespace cv {
namespace dnn {
// NEON code work around.
namespace opt_NEON
{
#if CV_NEON && CV_NEON_AARCH64
/* Accumulate */
void winofunc_accum_F32(const float* inwptr, const float* wptr, float* outbuf, int Cg, int iblock,
const int winoIblock, const int winoKblock, const int winoAtomF32, const int winoNatomF32)
{
CV_Assert(winoIblock == 6 && winoKblock == 4 && winoAtomF32 == 4);
if (iblock > 3)
{
for (int atom_id = 0; atom_id < winoNatomF32; atom_id++,
outbuf += winoAtomF32)
{
float32x4_t s00 = vdupq_n_f32(0.f), s01 = s00, s02 = s00, s03 = s00, s04 = s00, s05 = s00;
float32x4_t s10 = vdupq_n_f32(0.f), s11 = s00, s12 = s00, s13 = s00, s14 = s00, s15 = s00;
float32x4_t s20 = vdupq_n_f32(0.f), s21 = s00, s22 = s00, s23 = s00, s24 = s00, s25 = s00;
float32x4_t s30 = vdupq_n_f32(0.f), s31 = s00, s32 = s00, s33 = s00, s34 = s00, s35 = s00;
for (int c = 0; c < Cg; c++, inwptr += winoIblock*winoAtomF32,
wptr += winoKblock*winoAtomF32) {
float32x4_t w0 = vld1q_f32(wptr), w1 = vld1q_f32(wptr + 4);
float32x4_t w2 = vld1q_f32(wptr + 8), w3 = vld1q_f32(wptr + 12);
float32x4_t x0, x1;
x0 = vld1q_f32(inwptr);
x1 = vld1q_f32(inwptr + 4);
s00 = vfmaq_f32(s00, w0, x0);
s01 = vfmaq_f32(s01, w0, x1);
s10 = vfmaq_f32(s10, w1, x0);
s11 = vfmaq_f32(s11, w1, x1);
s20 = vfmaq_f32(s20, w2, x0);
s21 = vfmaq_f32(s21, w2, x1);
s30 = vfmaq_f32(s30, w3, x0);
s31 = vfmaq_f32(s31, w3, x1);
x0 = vld1q_f32(inwptr + 8);
x1 = vld1q_f32(inwptr + 12);
s02 = vfmaq_f32(s02, w0, x0);
s03 = vfmaq_f32(s03, w0, x1);
s12 = vfmaq_f32(s12, w1, x0);
s13 = vfmaq_f32(s13, w1, x1);
s22 = vfmaq_f32(s22, w2, x0);
s23 = vfmaq_f32(s23, w2, x1);
s32 = vfmaq_f32(s32, w3, x0);
s33 = vfmaq_f32(s33, w3, x1);
x0 = vld1q_f32(inwptr + 16);
x1 = vld1q_f32(inwptr + 20);
s04 = vfmaq_f32(s04, w0, x0);
s05 = vfmaq_f32(s05, w0, x1);
s14 = vfmaq_f32(s14, w1, x0);
s15 = vfmaq_f32(s15, w1, x1);
s24 = vfmaq_f32(s24, w2, x0);
s25 = vfmaq_f32(s25, w2, x1);
s34 = vfmaq_f32(s34, w3, x0);
s35 = vfmaq_f32(s35, w3, x1);
}
vst1q_f32(outbuf, s00);
vst1q_f32(outbuf + 1*64, s01);
vst1q_f32(outbuf + 2*64, s02);
vst1q_f32(outbuf + 3*64, s03);
vst1q_f32(outbuf + 4*64, s04);
vst1q_f32(outbuf + 5*64, s05);
vst1q_f32(outbuf + 6*64, s10);
vst1q_f32(outbuf + 7*64, s11);
vst1q_f32(outbuf + 8*64, s12);
vst1q_f32(outbuf + 9*64, s13);
vst1q_f32(outbuf + 10*64, s14);
vst1q_f32(outbuf + 11*64, s15);
vst1q_f32(outbuf + 12*64, s20);
vst1q_f32(outbuf + 13*64, s21);
vst1q_f32(outbuf + 14*64, s22);
vst1q_f32(outbuf + 15*64, s23);
vst1q_f32(outbuf + 16*64, s24);
vst1q_f32(outbuf + 17*64, s25);
vst1q_f32(outbuf + 18*64, s30);
vst1q_f32(outbuf + 19*64, s31);
vst1q_f32(outbuf + 20*64, s32);
vst1q_f32(outbuf + 21*64, s33);
vst1q_f32(outbuf + 22*64, s34);
vst1q_f32(outbuf + 23*64, s35);
}
}
else
{
for (int atom_id = 0; atom_id < winoNatomF32; atom_id++,
outbuf += winoAtomF32)
{
float32x4_t s00 = vdupq_n_f32(0.f), s01 = s00, s02 = s00;
float32x4_t s10 = vdupq_n_f32(0.f), s11 = s00, s12 = s00;
float32x4_t s20 = vdupq_n_f32(0.f), s21 = s00, s22 = s00;
float32x4_t s30 = vdupq_n_f32(0.f), s31 = s00, s32 = s00;
for (int c = 0; c < Cg; c++, inwptr += winoIblock*winoAtomF32,
wptr += winoKblock*winoAtomF32) {
float32x4_t w0 = vld1q_f32(wptr), w1 = vld1q_f32(wptr + 4);
float32x4_t w2 = vld1q_f32(wptr + 8), w3 = vld1q_f32(wptr + 12);
float32x4_t x0, x1, x2;
x0 = vld1q_f32(inwptr);
x1 = vld1q_f32(inwptr + 4);
x2 = vld1q_f32(inwptr + 8);
s00 = vfmaq_f32(s00, w0, x0);
s01 = vfmaq_f32(s01, w0, x1);
s02 = vfmaq_f32(s02, w0, x2);
s10 = vfmaq_f32(s10, w1, x0);
s11 = vfmaq_f32(s11, w1, x1);
s12 = vfmaq_f32(s12, w1, x2);
s20 = vfmaq_f32(s20, w2, x0);
s21 = vfmaq_f32(s21, w2, x1);
s22 = vfmaq_f32(s22, w2, x2);
s30 = vfmaq_f32(s30, w3, x0);
s31 = vfmaq_f32(s31, w3, x1);
s32 = vfmaq_f32(s32, w3, x2);
}
vst1q_f32(outbuf, s00);
vst1q_f32(outbuf + 1*64, s01);
vst1q_f32(outbuf + 2*64, s02);
vst1q_f32(outbuf + 6*64, s10);
vst1q_f32(outbuf + 7*64, s11);
vst1q_f32(outbuf + 8*64, s12);
vst1q_f32(outbuf + 12*64, s20);
vst1q_f32(outbuf + 13*64, s21);
vst1q_f32(outbuf + 14*64, s22);
vst1q_f32(outbuf + 18*64, s30);
vst1q_f32(outbuf + 19*64, s31);
vst1q_f32(outbuf + 20*64, s32);
}
}
}
#undef T4x4
#define T4x4(a, b, c, d, tr0, tr1) \
tr0 = vtrnq_f32(a, b); \
tr1 = vtrnq_f32(c, d); \
a = vcombine_f32(vget_low_f32(tr0.val[0]), vget_low_f32(tr1.val[0])); \
b = vcombine_f32(vget_low_f32(tr0.val[1]), vget_low_f32(tr1.val[1])); \
c = vcombine_f32(vget_high_f32(tr0.val[0]), vget_high_f32(tr1.val[0])); \
d = vcombine_f32(vget_high_f32(tr0.val[1]), vget_high_f32(tr1.val[1]))
/*Input transform*/
void winofunc_BtXB_8x8_F32(const float* inptr, int inpstep,
float* outptr, int Cg, const int winoIblock, const int winoAtomF32)
{
float32x4_t x00 = vld1q_f32(inptr), x01 = vld1q_f32(inptr + 4);
float32x4_t x10 = vld1q_f32(inptr + inpstep), x11 = vld1q_f32(inptr + inpstep + 4);
float32x4_t x20 = vld1q_f32(inptr + inpstep*2), x21 = vld1q_f32(inptr + inpstep*2 + 4);
float32x4_t x30 = vld1q_f32(inptr + inpstep*3), x31 = vld1q_f32(inptr + inpstep*3 + 4);
float32x4_t x40 = vld1q_f32(inptr + inpstep*4), x41 = vld1q_f32(inptr + inpstep*4 + 4);
float32x4_t x50 = vld1q_f32(inptr + inpstep*5), x51 = vld1q_f32(inptr + inpstep*5 + 4);
float32x4_t x60 = vld1q_f32(inptr + inpstep*6), x61 = vld1q_f32(inptr + inpstep*6 + 4);
float32x4_t x70 = vld1q_f32(inptr + inpstep*7), x71 = vld1q_f32(inptr + inpstep*7 + 4);
float32x4_t z00, z01, z10, z11, z20, z21, z30, z31, z40, z41, z50, z51, z60, z61, z70, z71;
{
/* Y[0] = [1.f, 0.f, -5.25f, 0.f, 5.25f, 0.f, -1.f, 0.f]*X */
/* Y[7] = [0.f, -1.f, 0.f, 5.25f, 0.f, -5.25f, 0.f, 1.f]*X */
float32x4_t q5_25 = vdupq_n_f32(5.25f), t00, t01, t10, t11;
t00 = vsubq_f32(x40, x20);
t01 = vsubq_f32(x41, x21);
t10 = vsubq_f32(x30, x50);
t11 = vsubq_f32(x31, x51);
float32x4_t y00 = vfmaq_f32(vsubq_f32(x00, x60), t00, q5_25);
float32x4_t y01 = vfmaq_f32(vsubq_f32(x01, x61), t01, q5_25);
float32x4_t y70 = vfmaq_f32(vsubq_f32(x70, x10), t10, q5_25);
float32x4_t y71 = vfmaq_f32(vsubq_f32(x71, x11), t11, q5_25);
/* Y[1] = [0.f, 1.f, 1.f, -4.25f, -4.25f, 1.f, 1.f, 0.f]*X */
/* Y[2] = [0.f, -1.f, 1.f, 4.25f, -4.25f, -1.f, 1.f, 0.f]*X */
float32x4_t qm4_25 = vdupq_n_f32(-4.25f);
t00 = vfmaq_f32(vaddq_f32(x10, x50), x30, qm4_25);
t01 = vfmaq_f32(vaddq_f32(x11, x51), x31, qm4_25);
t10 = vfmaq_f32(vaddq_f32(x20, x60), x40, qm4_25);
t11 = vfmaq_f32(vaddq_f32(x21, x61), x41, qm4_25);
float32x4_t y10 = vaddq_f32(t00, t10), y11 = vaddq_f32(t01, t11);
float32x4_t y20 = vsubq_f32(t10, t00), y21 = vsubq_f32(t11, t01);
/* Y[3] = [0.f, 0.5f, 0.25f, -2.5f, -1.25f, 2.f, 1.f, 0.f]*X */
/* Y[4] = [0.f, -0.5f, 0.25f, 2.5f, -1.25f, -2.f, 1.f, 0.f]*X */
float32x4_t q0_5 = vdupq_n_f32(0.5f), q0_25 = vdupq_n_f32(0.25f);
float32x4_t qm2_5 = vdupq_n_f32(-2.5f), qm1_25 = vdupq_n_f32(-1.25f);
t00 = vfmaq_f32(vaddq_f32(x50, x50), x10, q0_5);
t01 = vfmaq_f32(vaddq_f32(x51, x51), x11, q0_5);
t10 = vfmaq_f32(x60, x20, q0_25);
t11 = vfmaq_f32(x61, x21, q0_25);
t00 = vfmaq_f32(t00, x30, qm2_5);
t01 = vfmaq_f32(t01, x31, qm2_5);
t10 = vfmaq_f32(t10, x40, qm1_25);
t11 = vfmaq_f32(t11, x41, qm1_25);
float32x4_t y30 = vaddq_f32(t00, t10), y31 = vaddq_f32(t01, t11);
float32x4_t y40 = vsubq_f32(t10, t00), y41 = vsubq_f32(t11, t01);
/* Y[5] = [0.f, 2.f, 4.f, -2.5f, -5.f, 0.5f, 1.f, 0.f]*X */
/* Y[6] = [0.f, -2.f, 4.f, 2.5f, -5.f, -0.5f, 1.f, 0.f]*X */
float32x4_t q4 = vdupq_n_f32(4.f), qm5 = vdupq_n_f32(-5.f);
t00 = vfmaq_f32(vaddq_f32(x10, x10), x50, q0_5);
t01 = vfmaq_f32(vaddq_f32(x11, x11), x51, q0_5);
t10 = vfmaq_f32(x60, x20, q4);
t11 = vfmaq_f32(x61, x21, q4);
t00 = vfmaq_f32(t00, x30, qm2_5);
t01 = vfmaq_f32(t01, x31, qm2_5);
t10 = vfmaq_f32(t10, x40, qm5);
t11 = vfmaq_f32(t11, x41, qm5);
float32x4_t y50 = vaddq_f32(t00, t10), y51 = vaddq_f32(t01, t11);
float32x4_t y60 = vsubq_f32(t10, t00), y61 = vsubq_f32(t11, t01);
/* transpose 8x8 matrix in-place with some renumeration of the elements: */
/* Y: */
/* y00 y01 */
/* y10 y11 */
/* ... */
/* y70 y71 */
/* Y': */
/* y00 y40 */
/* y10 y50 */
/* y20 y60 */
/* y30 y70 */
/* y01 y41 */
/* y11 y51 */
/* y21 y61 */
/* y31 y71 */
/* in other words, y40 <-> y01, y50 <-> y11, y60 <-> y21, y70 <-> y31 */
float32x4x2_t tr0, tr1;
T4x4(y00, y10, y20, y30, tr0, tr1);
T4x4(y01, y11, y21, y31, tr0, tr1);
T4x4(y40, y50, y60, y70, tr0, tr1);
T4x4(y41, y51, y61, y71, tr0, tr1);
/* Z[0] = [1.f, 0.f, -5.25f, 0.f, 5.25f, 0.f, -1.f, 0.f]*Y */
/* Z[7] = [0.f, -1.f, 0.f, 5.25f, 0.f, -5.25f, 0.f, 1.f]*Y */
t00 = vsubq_f32(y01, y20);
t01 = vsubq_f32(y41, y60);
t10 = vsubq_f32(y30, y11);
t11 = vsubq_f32(y70, y51);
z00 = vfmaq_f32(vsubq_f32(y00, y21), t00, q5_25);
z01 = vfmaq_f32(vsubq_f32(y40, y61), t01, q5_25);
z70 = vfmaq_f32(vsubq_f32(y31, y10), t10, q5_25);
z71 = vfmaq_f32(vsubq_f32(y71, y50), t11, q5_25);
/* Z[1] = [0.f, 1.f, 1.f, -4.25f, -4.25f, 1.f, 1.f, 0.f]*Y */
/* Z[2] = [0.f, -1.f, 1.f, 4.25f, -4.25f, -1.f, 1.f, 0.f]*Y */
t00 = vfmaq_f32(vaddq_f32(y10, y11), y30, qm4_25);
t01 = vfmaq_f32(vaddq_f32(y50, y51), y70, qm4_25);
t10 = vfmaq_f32(vaddq_f32(y20, y21), y01, qm4_25);
t11 = vfmaq_f32(vaddq_f32(y60, y61), y41, qm4_25);
z10 = vaddq_f32(t00, t10); z11 = vaddq_f32(t01, t11);
z20 = vsubq_f32(t10, t00); z21 = vsubq_f32(t11, t01);
/* Z[3] = [0.f, 0.5f, 0.25f, -2.5f, -1.25f, 2.f, 1.f, 0.f]*Y */
/* Z[4] = [0.f, -0.5f, 0.25f, 2.5f, -1.25f, -2.f, 1.f, 0.f]*Y */
t00 = vfmaq_f32(vaddq_f32(y11, y11), y10, q0_5);
t01 = vfmaq_f32(vaddq_f32(y51, y51), y50, q0_5);
t10 = vfmaq_f32(y21, y20, q0_25);
t11 = vfmaq_f32(y61, y60, q0_25);
t00 = vfmaq_f32(t00, y30, qm2_5);
t01 = vfmaq_f32(t01, y70, qm2_5);
t10 = vfmaq_f32(t10, y01, qm1_25);
t11 = vfmaq_f32(t11, y41, qm1_25);
z30 = vaddq_f32(t00, t10); z31 = vaddq_f32(t01, t11);
z40 = vsubq_f32(t10, t00); z41 = vsubq_f32(t11, t01);
/* Z[5] = [0.f, 2.f, 4.f, -2.5f, -5.f, 0.5f, 1.f, 0.f]*Y */
/* Z[6] = [0.f, -2.f, 4.f, 2.5f, -5.f, -0.5f, 1.f, 0.f]*Y */
t00 = vfmaq_f32(vaddq_f32(y10, y10), y11, q0_5);
t01 = vfmaq_f32(vaddq_f32(y50, y50), y51, q0_5);
t10 = vfmaq_f32(y21, y20, q4);
t11 = vfmaq_f32(y61, y60, q4);
t00 = vfmaq_f32(t00, y30, qm2_5);
t01 = vfmaq_f32(t01, y70, qm2_5);
t10 = vfmaq_f32(t10, y01, qm5);
t11 = vfmaq_f32(t11, y41, qm5);
z50 = vaddq_f32(t00, t10); z51 = vaddq_f32(t01, t11);
z60 = vsubq_f32(t10, t00); z61 = vsubq_f32(t11, t01);
}
const int outstep = winoIblock*winoAtomF32*Cg;
vst1q_f32(outptr, z00);
vst1q_f32(outptr + outstep, z01);
vst1q_f32(outptr + outstep*2, z10);
vst1q_f32(outptr + outstep*3, z11);
vst1q_f32(outptr + outstep*4, z20);
vst1q_f32(outptr + outstep*5, z21);
vst1q_f32(outptr + outstep*6, z30);
vst1q_f32(outptr + outstep*7, z31);
vst1q_f32(outptr + outstep*8, z40);
vst1q_f32(outptr + outstep*9, z41);
vst1q_f32(outptr + outstep*10, z50);
vst1q_f32(outptr + outstep*11, z51);
vst1q_f32(outptr + outstep*12, z60);
vst1q_f32(outptr + outstep*13, z61);
vst1q_f32(outptr + outstep*14, z70);
vst1q_f32(outptr + outstep*15, z71);
}
/*Output transform*/
void winofunc_AtXA_8x8_F32(const float* inptr, int inpstep,
float* bpptr, int bpstep, float* outptr, int outstep,
float bias, float minval, float maxval, bool ifMinMaxAct)
{
float32x4_t x00 = vld1q_f32(inptr), x01 = vld1q_f32(inptr + 4);
float32x4_t x10 = vld1q_f32(inptr + inpstep), x11 = vld1q_f32(inptr + inpstep + 4);
float32x4_t x20 = vld1q_f32(inptr + inpstep*2), x21 = vld1q_f32(inptr + inpstep*2 + 4);
float32x4_t x30 = vld1q_f32(inptr + inpstep*3), x31 = vld1q_f32(inptr + inpstep*3 + 4);
float32x4_t x40 = vld1q_f32(inptr + inpstep*4), x41 = vld1q_f32(inptr + inpstep*4 + 4);
float32x4_t x50 = vld1q_f32(inptr + inpstep*5), x51 = vld1q_f32(inptr + inpstep*5 + 4);
float32x4_t x60 = vld1q_f32(inptr + inpstep*6), x61 = vld1q_f32(inptr + inpstep*6 + 4);
float32x4_t x70 = vld1q_f32(inptr + inpstep*7), x71 = vld1q_f32(inptr + inpstep*7 + 4);
float32x4_t z00, z01, z10, z11, z20, z21, z30, z31, z40, z41, z50, z51;
{
float32x4_t s12_0, s12_1, s34_0, s34_1, s56_0, s56_1;
s12_0 = vaddq_f32(x10, x20); s12_1 = vaddq_f32(x11, x21);
s34_0 = vaddq_f32(x30, x40); s34_1 = vaddq_f32(x31, x41);
s56_0 = vaddq_f32(x50, x60); s56_1 = vaddq_f32(x51, x61);
float32x4_t y00 = vaddq_f32(vaddq_f32(vaddq_f32(x00, s12_0), s34_0), s56_0);
float32x4_t y01 = vaddq_f32(vaddq_f32(vaddq_f32(x01, s12_1), s34_1), s56_1);
float32x4_t y20 = vfmaq_n_f32(vfmaq_n_f32(s12_0, s34_0, 4.0f), s56_0, 0.25f);
float32x4_t y21 = vfmaq_n_f32(vfmaq_n_f32(s12_1, s34_1, 4.0f), s56_1, 0.25f);
float32x4_t y40 = vfmaq_n_f32(vfmaq_n_f32(s12_0, s34_0, 16.0f), s56_0, 1.f/16);
float32x4_t y41 = vfmaq_n_f32(vfmaq_n_f32(s12_1, s34_1, 16.0f), s56_1, 1.f/16);
s12_0 = vsubq_f32(x10, x20); s12_1 = vsubq_f32(x11, x21);
s34_0 = vsubq_f32(x30, x40); s34_1 = vsubq_f32(x31, x41);
s56_0 = vsubq_f32(x50, x60); s56_1 = vsubq_f32(x51, x61);
float32x4_t y50 = vfmaq_n_f32(vfmaq_n_f32(vaddq_f32(x70, s12_0),
s34_0, 32.f), s56_0, 1.f/32);
float32x4_t y51 = vfmaq_n_f32(vfmaq_n_f32(vaddq_f32(x71, s12_1),
s34_1, 32.f), s56_1, 1.f/32);
float32x4_t y10 = vfmaq_n_f32(vfmaq_n_f32(s12_0, s34_0, 2.0f), s56_0, 0.5f);
float32x4_t y11 = vfmaq_n_f32(vfmaq_n_f32(s12_1, s34_1, 2.0f), s56_1, 0.5f);
float32x4_t y30 = vfmaq_n_f32(vfmaq_n_f32(s12_0, s34_0, 8.0f), s56_0, 0.125f);
float32x4_t y31 = vfmaq_n_f32(vfmaq_n_f32(s12_1, s34_1, 8.0f), s56_1, 0.125f);
float32x4_t y60 = vdupq_n_f32(0.f), y61 = y60, y70 = y60, y71 = y60;
/* transpose 8x8 matrix in-place with some renumeration of the elements: */
/* Y: */
/* y00 y01 */
/* y10 y11 */
/* ... */
/* y50 y51 */
/* 0 0 */
/* 0 0 */
/* Y': */
/* y00 y40 */
/* y10 y50 */
/* y20 y60 */
/* y30 y70 */
/* y01 y41 */
/* y11 y51 */
/* y21 y61 */
/* y31 y71 */
/* in other words, y40 <-> y01, y50 <-> y11, y60 <-> y21, y70 <-> y31 */
float32x4x2_t tr0, tr1;
T4x4(y00, y10, y20, y30, tr0, tr1);
T4x4(y01, y11, y21, y31, tr0, tr1);
T4x4(y40, y50, y60, y70, tr0, tr1);
T4x4(y41, y51, y61, y71, tr0, tr1);
s12_0 = vaddq_f32(y10, y20); s12_1 = vaddq_f32(y50, y60);
s34_0 = vaddq_f32(y30, y01); s34_1 = vaddq_f32(y70, y41);
s56_0 = vaddq_f32(y11, y21); s56_1 = vaddq_f32(y51, y61);
z00 = vaddq_f32(vaddq_f32(vaddq_f32(y00, s12_0), s34_0), s56_0);
z01 = vaddq_f32(vaddq_f32(vaddq_f32(y40, s12_1), s34_1), s56_1);
z20 = vfmaq_n_f32(vfmaq_n_f32(s12_0, s34_0, 4.0f), s56_0, 0.25f);
z21 = vfmaq_n_f32(vfmaq_n_f32(s12_1, s34_1, 4.0f), s56_1, 0.25f);
z40 = vfmaq_n_f32(vfmaq_n_f32(s12_0, s34_0, 16.0f), s56_0, 1.f/16);
z41 = vfmaq_n_f32(vfmaq_n_f32(s12_1, s34_1, 16.0f), s56_1, 1.f/16);
s12_0 = vsubq_f32(y10, y20); s12_1 = vsubq_f32(y50, y60);
s34_0 = vsubq_f32(y30, y01); s34_1 = vsubq_f32(y70, y41);
s56_0 = vsubq_f32(y11, y21); s56_1 = vsubq_f32(y51, y61);
z50 = vfmaq_n_f32(vfmaq_n_f32(vaddq_f32(y31, s12_0),
s34_0, 32.f), s56_0, 1.f/32);
z51 = vfmaq_n_f32(vfmaq_n_f32(vaddq_f32(y71, s12_1),
s34_1, 32.f), s56_1, 1.f/32);
z10 = vfmaq_n_f32(vfmaq_n_f32(s12_0, s34_0, 2.0f), s56_0, 0.5f);
z11 = vfmaq_n_f32(vfmaq_n_f32(s12_1, s34_1, 2.0f), s56_1, 0.5f);
z30 = vfmaq_n_f32(vfmaq_n_f32(s12_0, s34_0, 8.0f), s56_0, 0.125f);
z31 = vfmaq_n_f32(vfmaq_n_f32(s12_1, s34_1, 8.0f), s56_1, 0.125f);
float32x4_t vbias = vdupq_n_f32(bias);
z00 = vaddq_f32(z00, vbias);
z01 = vaddq_f32(z01, vbias);
z10 = vaddq_f32(z10, vbias);
z11 = vaddq_f32(z11, vbias);
z20 = vaddq_f32(z20, vbias);
z21 = vaddq_f32(z21, vbias);
z30 = vaddq_f32(z30, vbias);
z31 = vaddq_f32(z31, vbias);
z40 = vaddq_f32(z40, vbias);
z41 = vaddq_f32(z41, vbias);
z50 = vaddq_f32(z50, vbias);
z51 = vaddq_f32(z51, vbias);
}
if (bpptr)
{
float32x2_t zhalf = vdup_n_f32(0.f);
z00 = vaddq_f32(z00, vld1q_f32(bpptr));
z01 = vaddq_f32(z01, vcombine_f32(vld1_f32(bpptr + 4), zhalf));
z10 = vaddq_f32(z10, vld1q_f32(bpptr + bpstep));
z11 = vaddq_f32(z11, vcombine_f32(vld1_f32(bpptr + bpstep + 4), zhalf));
z20 = vaddq_f32(z20, vld1q_f32(bpptr + bpstep*2));
z21 = vaddq_f32(z21, vcombine_f32(vld1_f32(bpptr + bpstep*2 + 4), zhalf));
z30 = vaddq_f32(z30, vld1q_f32(bpptr + bpstep*3));
z31 = vaddq_f32(z31, vcombine_f32(vld1_f32(bpptr + bpstep*3 + 4), zhalf));
z40 = vaddq_f32(z40, vld1q_f32(bpptr + bpstep*4));
z41 = vaddq_f32(z41, vcombine_f32(vld1_f32(bpptr + bpstep*4 + 4), zhalf));
z50 = vaddq_f32(z50, vld1q_f32(bpptr + bpstep*5));
z51 = vaddq_f32(z51, vcombine_f32(vld1_f32(bpptr + bpstep*5 + 4), zhalf));
}
if (ifMinMaxAct)
{
float32x4_t vmax = vdupq_n_f32(maxval);
float32x4_t vmin = vdupq_n_f32(minval);
z00 = vminq_f32(vmaxq_f32(z00, vmin), vmax);
z01 = vminq_f32(vmaxq_f32(z01, vmin), vmax);
z10 = vminq_f32(vmaxq_f32(z10, vmin), vmax);
z11 = vminq_f32(vmaxq_f32(z11, vmin), vmax);
z20 = vminq_f32(vmaxq_f32(z20, vmin), vmax);
z21 = vminq_f32(vmaxq_f32(z21, vmin), vmax);
z30 = vminq_f32(vmaxq_f32(z30, vmin), vmax);
z31 = vminq_f32(vmaxq_f32(z31, vmin), vmax);
z40 = vminq_f32(vmaxq_f32(z40, vmin), vmax);
z41 = vminq_f32(vmaxq_f32(z41, vmin), vmax);
z50 = vminq_f32(vmaxq_f32(z50, vmin), vmax);
z51 = vminq_f32(vmaxq_f32(z51, vmin), vmax);
}
vst1q_f32(outptr, z00);
vst1_f32(outptr + 4, vget_low_f32(z01));
vst1q_f32(outptr + outstep, z10);
vst1_f32(outptr + outstep + 4, vget_low_f32(z11));
vst1q_f32(outptr + outstep*2, z20);
vst1_f32(outptr + outstep*2 + 4, vget_low_f32(z21));
vst1q_f32(outptr + outstep*3, z30);
vst1_f32(outptr + outstep*3 + 4, vget_low_f32(z31));
vst1q_f32(outptr + outstep*4, z40);
vst1_f32(outptr + outstep*4 + 4, vget_low_f32(z41));
vst1q_f32(outptr + outstep*5, z50);
vst1_f32(outptr + outstep*5 + 4, vget_low_f32(z51));
}
#endif
}
}} // namespace
modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.simd.hpp
+271 -240
View File
@@ -9,26 +9,37 @@ namespace dnn {
CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
/* Accumulate */
void winofunc_accum_f32(const float* inwptr, const float* wptr, float* outbuf, int Cg, int iblock,
void winofunc_accum_F32(const float* inwptr, const float* wptr, float* outbuf, int Cg, int iblock,
const int winoIblock, const int winoKblock, const int winoAtomF32, const int winoNatomF32);
/*Input transform*/
void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep,
void winofunc_BtXB_8x8_F32(const float* inptr, int inpstep,
float* outptr, int Cg, const int winoIblock, const int winoAtomF32);
/*Output transform*/
void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep,
void winofunc_AtXA_8x8_F32(const float* inptr, int inpstep,
float* bpptr, int bpstep, float* outptr, int outstep,
float bias, float minval, float maxval, bool ifMinMaxAct);
#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) && CV_AVX
// FP 16 branch, only ARMv8 supports.
void winofunc_accum_F16(const char* _inwptr, const char* _wptr, char* _outbuf, int Cg, int iblock,
const int winoIblock, const int winoKblock, const int winoAtomF16, const int winoNatomF16);
void winofunc_BtXB_8x8_F16(const float * inptr, int inpstep,
char * _outptr, int Cg, const int winoIblock, const int winoAtomF16);
void winofunc_AtXA_8x8_F16(const char* inptr, int inpstep,
float * bpptr, int bpstep, float* outptr, int outstep,
float bias, float minval, float maxval, bool ifMinMaxAct);
#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY)
#if CV_AVX
#if !CV_FMA3 // AVX workaround
#undef _mm256_fmadd_ps
#define _mm256_fmadd_ps(a, b, c) _mm256_add_ps(c, _mm256_mul_ps(a, b))
#endif
void winofunc_accum_f32(const float* inwptr, const float* wptr, float* outbuf, int Cg, int iblock,
void winofunc_accum_F32(const float* inwptr, const float* wptr, float* outbuf, int Cg, int iblock,
const int winoIblock, const int winoKblock, const int winoAtomF32, const int winoNatomF32)
{
CV_Assert(winoIblock == 6 && winoKblock == 4 && winoAtomF32 == 8);
@@ -187,7 +198,7 @@ void transpose8_ps(__m256 &row0, __m256 &row1, __m256 &row2, __m256 &row3, __m25
}
/*Input transform*/
void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep,
void winofunc_BtXB_8x8_F32(const float* inptr, int inpstep,
float* outptr, int Cg, const int winoIblock, const int winoAtomF32)
{
__m256 x00 = _mm256_loadu_ps(inptr);
@@ -311,7 +322,7 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep,
0.f, 1.f, 1.f, 16.f, 16.f, 1.f/16, 1.f/16, 0.f,
0.f, 1.f, -1.f, 32.f, -32.f, 1.f/32, -1.f/32, 1.f]
*/
void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep,
void winofunc_AtXA_8x8_F32(const float* inptr, int inpstep,
float* bpptr, int bpstep, float* outptr, int outstep,
float bias, float minval, float maxval, bool ifMinMaxAct)
{
@@ -405,154 +416,13 @@ void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep,
STORE6_ELE_FROM_16(outptr + outstep * 5, z50, lowM, highM);
_mm256_zeroupper();
}
#endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
CV_CPU_OPTIMIZATION_NAMESPACE_END
#endif // CV_AVX
// NEON code work around.
namespace opt_NEON
{
#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) && CV_NEON && CV_NEON_AARCH64
/* Accumulate */
void winofunc_accum_f32(const float* inwptr, const float* wptr, float* outbuf, int Cg, int iblock,
const int winoIblock, const int winoKblock, const int winoAtomF32, const int winoNatomF32);
/*Input transform*/
void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep,
float* outptr, int Cg, const int winoIblock, const int winoAtomF32);
/*Output transform*/
void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep,
float* bpptr, int bpstep, float* outptr, int outstep,
float bias, float minval, float maxval, bool ifMinMaxAct);
void winofunc_accum_f32(const float* inwptr, const float* wptr, float* outbuf, int Cg, int iblock,
const int winoIblock, const int winoKblock, const int winoAtomF32, const int winoNatomF32)
{
CV_Assert(winoIblock == 6 && winoKblock == 4 && winoAtomF32 == 4);
if (iblock > 3)
{
for (int atom_id = 0; atom_id < winoNatomF32; atom_id++,
outbuf += winoAtomF32)
{
float32x4_t s00 = vdupq_n_f32(0.f), s01 = s00, s02 = s00, s03 = s00, s04 = s00, s05 = s00;
float32x4_t s10 = vdupq_n_f32(0.f), s11 = s00, s12 = s00, s13 = s00, s14 = s00, s15 = s00;
float32x4_t s20 = vdupq_n_f32(0.f), s21 = s00, s22 = s00, s23 = s00, s24 = s00, s25 = s00;
float32x4_t s30 = vdupq_n_f32(0.f), s31 = s00, s32 = s00, s33 = s00, s34 = s00, s35 = s00;
for (int c = 0; c < Cg; c++, inwptr += winoIblock*winoAtomF32,
wptr += winoKblock*winoAtomF32) {
float32x4_t w0 = vld1q_f32(wptr), w1 = vld1q_f32(wptr + 4);
float32x4_t w2 = vld1q_f32(wptr + 8), w3 = vld1q_f32(wptr + 12);
float32x4_t x0, x1;
x0 = vld1q_f32(inwptr);
x1 = vld1q_f32(inwptr + 4);
s00 = vfmaq_f32(s00, w0, x0);
s01 = vfmaq_f32(s01, w0, x1);
s10 = vfmaq_f32(s10, w1, x0);
s11 = vfmaq_f32(s11, w1, x1);
s20 = vfmaq_f32(s20, w2, x0);
s21 = vfmaq_f32(s21, w2, x1);
s30 = vfmaq_f32(s30, w3, x0);
s31 = vfmaq_f32(s31, w3, x1);
x0 = vld1q_f32(inwptr + 8);
x1 = vld1q_f32(inwptr + 12);
s02 = vfmaq_f32(s02, w0, x0);
s03 = vfmaq_f32(s03, w0, x1);
s12 = vfmaq_f32(s12, w1, x0);
s13 = vfmaq_f32(s13, w1, x1);
s22 = vfmaq_f32(s22, w2, x0);
s23 = vfmaq_f32(s23, w2, x1);
s32 = vfmaq_f32(s32, w3, x0);
s33 = vfmaq_f32(s33, w3, x1);
x0 = vld1q_f32(inwptr + 16);
x1 = vld1q_f32(inwptr + 20);
s04 = vfmaq_f32(s04, w0, x0);
s05 = vfmaq_f32(s05, w0, x1);
s14 = vfmaq_f32(s14, w1, x0);
s15 = vfmaq_f32(s15, w1, x1);
s24 = vfmaq_f32(s24, w2, x0);
s25 = vfmaq_f32(s25, w2, x1);
s34 = vfmaq_f32(s34, w3, x0);
s35 = vfmaq_f32(s35, w3, x1);
}
vst1q_f32(outbuf, s00);
vst1q_f32(outbuf + 1*64, s01);
vst1q_f32(outbuf + 2*64, s02);
vst1q_f32(outbuf + 3*64, s03);
vst1q_f32(outbuf + 4*64, s04);
vst1q_f32(outbuf + 5*64, s05);
vst1q_f32(outbuf + 6*64, s10);
vst1q_f32(outbuf + 7*64, s11);
vst1q_f32(outbuf + 8*64, s12);
vst1q_f32(outbuf + 9*64, s13);
vst1q_f32(outbuf + 10*64, s14);
vst1q_f32(outbuf + 11*64, s15);
vst1q_f32(outbuf + 12*64, s20);
vst1q_f32(outbuf + 13*64, s21);
vst1q_f32(outbuf + 14*64, s22);
vst1q_f32(outbuf + 15*64, s23);
vst1q_f32(outbuf + 16*64, s24);
vst1q_f32(outbuf + 17*64, s25);
vst1q_f32(outbuf + 18*64, s30);
vst1q_f32(outbuf + 19*64, s31);
vst1q_f32(outbuf + 20*64, s32);
vst1q_f32(outbuf + 21*64, s33);
vst1q_f32(outbuf + 22*64, s34);
vst1q_f32(outbuf + 23*64, s35);
}
}
else
{
for (int atom_id = 0; atom_id < winoNatomF32; atom_id++,
outbuf += winoAtomF32)
{
float32x4_t s00 = vdupq_n_f32(0.f), s01 = s00, s02 = s00;
float32x4_t s10 = vdupq_n_f32(0.f), s11 = s00, s12 = s00;
float32x4_t s20 = vdupq_n_f32(0.f), s21 = s00, s22 = s00;
float32x4_t s30 = vdupq_n_f32(0.f), s31 = s00, s32 = s00;
for (int c = 0; c < Cg; c++, inwptr += winoIblock*winoAtomF32,
wptr += winoKblock*winoAtomF32) {
float32x4_t w0 = vld1q_f32(wptr), w1 = vld1q_f32(wptr + 4);
float32x4_t w2 = vld1q_f32(wptr + 8), w3 = vld1q_f32(wptr + 12);
float32x4_t x0, x1, x2;
x0 = vld1q_f32(inwptr);
x1 = vld1q_f32(inwptr + 4);
x2 = vld1q_f32(inwptr + 8);
s00 = vfmaq_f32(s00, w0, x0);
s01 = vfmaq_f32(s01, w0, x1);
s02 = vfmaq_f32(s02, w0, x2);
s10 = vfmaq_f32(s10, w1, x0);
s11 = vfmaq_f32(s11, w1, x1);
s12 = vfmaq_f32(s12, w1, x2);
s20 = vfmaq_f32(s20, w2, x0);
s21 = vfmaq_f32(s21, w2, x1);
s22 = vfmaq_f32(s22, w2, x2);
s30 = vfmaq_f32(s30, w3, x0);
s31 = vfmaq_f32(s31, w3, x1);
s32 = vfmaq_f32(s32, w3, x2);
}
vst1q_f32(outbuf, s00);
vst1q_f32(outbuf + 1*64, s01);
vst1q_f32(outbuf + 2*64, s02);
vst1q_f32(outbuf + 6*64, s10);
vst1q_f32(outbuf + 7*64, s11);
vst1q_f32(outbuf + 8*64, s12);
vst1q_f32(outbuf + 12*64, s20);
vst1q_f32(outbuf + 13*64, s21);
vst1q_f32(outbuf + 14*64, s22);
vst1q_f32(outbuf + 18*64, s30);
vst1q_f32(outbuf + 19*64, s31);
vst1q_f32(outbuf + 20*64, s32);
}
}
}
// FP16, currently, only ARMv8 may support it
#if defined(CV_NEON_AARCH64) && CV_NEON_AARCH64 && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
#undef T4x4
#define T4x4(a, b, c, d, tr0, tr1) \
tr0 = vtrnq_f32(a, b); \
tr1 = vtrnq_f32(c, d); \
@@ -561,10 +431,168 @@ void winofunc_accum_f32(const float* inwptr, const float* wptr, float* outbuf, i
c = vcombine_f32(vget_high_f32(tr0.val[0]), vget_high_f32(tr1.val[0])); \
d = vcombine_f32(vget_high_f32(tr0.val[1]), vget_high_f32(tr1.val[1]))
/*Input transform*/
void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep,
float* outptr, int Cg, const int winoIblock, const int winoAtomF32)
/* Accumulate */
void winofunc_accum_F16(const char* _inwptr, const char* _wptr, char* _outbuf, int Cg, int iblock,
const int winoIblock, const int winoKblock, const int winoAtomF16, const int winoNatomF16)
{
typedef __fp16 float16_t;
const float16_t* inwptr = (const float16_t*)_inwptr;
const float16_t* wptr = (const float16_t*)_wptr;
float16_t* outbuf = (float16_t*)_outbuf;
CV_Assert(winoIblock == 6 && winoKblock == 4 && winoAtomF16 == 8);
if (iblock > 3)
{
for (int atom_id = 0; atom_id < winoNatomF16; atom_id++, outbuf += winoAtomF16)
{
float16x8_t s00 = vdupq_n_f16(0.f), s01 = s00, s02 = s00, s03 = s00, s04 = s00, s05 = s00;
float16x8_t s10 = vdupq_n_f16(0.f), s11 = s00, s12 = s00, s13 = s00, s14 = s00, s15 = s00;
float16x8_t s20 = vdupq_n_f16(0.f), s21 = s00, s22 = s00, s23 = s00, s24 = s00, s25 = s00;
float16x8_t s30 = vdupq_n_f16(0.f), s31 = s00, s32 = s00, s33 = s00, s34 = s00, s35 = s00;
for (int c = 0; c < Cg; c++, inwptr += winoIblock*winoAtomF16,
wptr += winoKblock*winoAtomF16)
{
float16x8_t w0 = vld1q_f16(wptr), w1 = vld1q_f16(wptr + 8);
float16x8_t w2 = vld1q_f16(wptr + 16), w3 = vld1q_f16(wptr + 24);
float16x8_t x0, x1, x2;
x0 = vld1q_f16(inwptr);
x1 = vld1q_f16(inwptr + 8);
x2 = vld1q_f16(inwptr + 16);
s00 = vfmaq_f16(s00, w0, x0);
s01 = vfmaq_f16(s01, w0, x1);
s02 = vfmaq_f16(s02, w0, x2);
s10 = vfmaq_f16(s10, w1, x0);
s11 = vfmaq_f16(s11, w1, x1);
s12 = vfmaq_f16(s12, w1, x2);
s20 = vfmaq_f16(s20, w2, x0);
s21 = vfmaq_f16(s21, w2, x1);
s22 = vfmaq_f16(s22, w2, x2);
s30 = vfmaq_f16(s30, w3, x0);
s31 = vfmaq_f16(s31, w3, x1);
s32 = vfmaq_f16(s32, w3, x2);
x0 = vld1q_f16(inwptr + 24);
x1 = vld1q_f16(inwptr + 32);
x2 = vld1q_f16(inwptr + 40);
s03 = vfmaq_f16(s03, w0, x0);
s04 = vfmaq_f16(s04, w0, x1);
s05 = vfmaq_f16(s05, w0, x2);
s13 = vfmaq_f16(s13, w1, x0);
s14 = vfmaq_f16(s14, w1, x1);
s15 = vfmaq_f16(s15, w1, x2);
s23 = vfmaq_f16(s23, w2, x0);
s24 = vfmaq_f16(s24, w2, x1);
s25 = vfmaq_f16(s25, w2, x2);
s33 = vfmaq_f16(s33, w3, x0);
s34 = vfmaq_f16(s34, w3, x1);
s35 = vfmaq_f16(s35, w3, x2);
}
vst1q_f16(outbuf, s00);
vst1q_f16(outbuf + 1*64, s01);
vst1q_f16(outbuf + 2*64, s02);
vst1q_f16(outbuf + 3*64, s03);
vst1q_f16(outbuf + 4*64, s04);
vst1q_f16(outbuf + 5*64, s05);
vst1q_f16(outbuf + 6*64, s10);
vst1q_f16(outbuf + 7*64, s11);
vst1q_f16(outbuf + 8*64, s12);
vst1q_f16(outbuf + 9*64, s13);
vst1q_f16(outbuf + 10*64, s14);
vst1q_f16(outbuf + 11*64, s15);
vst1q_f16(outbuf + 12*64, s20);
vst1q_f16(outbuf + 13*64, s21);
vst1q_f16(outbuf + 14*64, s22);
vst1q_f16(outbuf + 15*64, s23);
vst1q_f16(outbuf + 16*64, s24);
vst1q_f16(outbuf + 17*64, s25);
vst1q_f16(outbuf + 18*64, s30);
vst1q_f16(outbuf + 19*64, s31);
vst1q_f16(outbuf + 20*64, s32);
vst1q_f16(outbuf + 21*64, s33);
vst1q_f16(outbuf + 22*64, s34);
vst1q_f16(outbuf + 23*64, s35);
}
}
else
{
for (int atom_id = 0; atom_id < winoNatomF16; atom_id++,
outbuf += winoAtomF16)
{
float16x8_t s00 = vdupq_n_f16(0.f), s01 = s00, s02 = s00;
float16x8_t s10 = vdupq_n_f16(0.f), s11 = s00, s12 = s00;
float16x8_t s20 = vdupq_n_f16(0.f), s21 = s00, s22 = s00;
float16x8_t s30 = vdupq_n_f16(0.f), s31 = s00, s32 = s00;
for (int c = 0; c < Cg; c++, inwptr += winoIblock*winoAtomF16,
wptr += winoKblock*winoAtomF16)
{
float16x8_t w0 = vld1q_f16(wptr), w1 = vld1q_f16(wptr + 8);
float16x8_t w2 = vld1q_f16(wptr + 16), w3 = vld1q_f16(wptr + 24);
float16x8_t x0, x1, x2;
x0 = vld1q_f16(inwptr);
x1 = vld1q_f16(inwptr + 8);
x2 = vld1q_f16(inwptr + 16);
s00 = vfmaq_f16(s00, w0, x0);
s01 = vfmaq_f16(s01, w0, x1);
s02 = vfmaq_f16(s02, w0, x2);
s10 = vfmaq_f16(s10, w1, x0);
s11 = vfmaq_f16(s11, w1, x1);
s12 = vfmaq_f16(s12, w1, x2);
s20 = vfmaq_f16(s20, w2, x0);
s21 = vfmaq_f16(s21, w2, x1);
s22 = vfmaq_f16(s22, w2, x2);
s30 = vfmaq_f16(s30, w3, x0);
s31 = vfmaq_f16(s31, w3, x1);
s32 = vfmaq_f16(s32, w3, x2);
}
vst1q_f16(outbuf, s00);
vst1q_f16(outbuf + 1*64, s01);
vst1q_f16(outbuf + 2*64, s02);
vst1q_f16(outbuf + 6*64, s10);
vst1q_f16(outbuf + 7*64, s11);
vst1q_f16(outbuf + 8*64, s12);
vst1q_f16(outbuf + 12*64, s20);
vst1q_f16(outbuf + 13*64, s21);
vst1q_f16(outbuf + 14*64, s22);
vst1q_f16(outbuf + 18*64, s30);
vst1q_f16(outbuf + 19*64, s31);
vst1q_f16(outbuf + 20*64, s32);
}
}
}
/*Input transform*/
//NOTE: Since we don't have the fully fp16 support. Current work around is that we need packing the data and
// convert it to FP16 in input transform stage. And at output transform stage we will convert it back to FP32.
void winofunc_BtXB_8x8_F16(const float * inptr, int inpstep,
char * _outptr, int Cg, const int winoIblock, const int winoAtomF16)
{
typedef __fp16 float16_t;
float16_t* outptr = (float16_t*)_outptr;
float32x4_t x00 = vld1q_f32(inptr), x01 = vld1q_f32(inptr + 4);
float32x4_t x10 = vld1q_f32(inptr + inpstep), x11 = vld1q_f32(inptr + inpstep + 4);
float32x4_t x20 = vld1q_f32(inptr + inpstep*2), x21 = vld1q_f32(inptr + inpstep*2 + 4);
@@ -577,8 +605,8 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep,
float32x4_t z00, z01, z10, z11, z20, z21, z30, z31, z40, z41, z50, z51, z60, z61, z70, z71;
{
/* Y[0] = [1.f, 0.f, -5.25f, 0.f, 5.25f, 0.f, -1.f, 0.f]*X */
/* Y[7] = [0.f, -1.f, 0.f, 5.25f, 0.f, -5.25f, 0.f, 1.f]*X */
// Y[0] = [1.f, 0.f, -5.25f, 0.f, 5.25f, 0.f, -1.f, 0.f]*X
// Y[7] = [0.f, -1.f, 0.f, 5.25f, 0.f, -5.25f, 0.f, 1.f]*X
float32x4_t q5_25 = vdupq_n_f32(5.25f), t00, t01, t10, t11;
t00 = vsubq_f32(x40, x20);
t01 = vsubq_f32(x41, x21);
@@ -589,8 +617,8 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep,
float32x4_t y70 = vfmaq_f32(vsubq_f32(x70, x10), t10, q5_25);
float32x4_t y71 = vfmaq_f32(vsubq_f32(x71, x11), t11, q5_25);
/* Y[1] = [0.f, 1.f, 1.f, -4.25f, -4.25f, 1.f, 1.f, 0.f]*X */
/* Y[2] = [0.f, -1.f, 1.f, 4.25f, -4.25f, -1.f, 1.f, 0.f]*X */
// Y[1] = [0.f, 1.f, 1.f, -4.25f, -4.25f, 1.f, 1.f, 0.f]*X
// Y[2] = [0.f, -1.f, 1.f, 4.25f, -4.25f, -1.f, 1.f, 0.f]*X
float32x4_t qm4_25 = vdupq_n_f32(-4.25f);
t00 = vfmaq_f32(vaddq_f32(x10, x50), x30, qm4_25);
t01 = vfmaq_f32(vaddq_f32(x11, x51), x31, qm4_25);
@@ -600,8 +628,8 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep,
float32x4_t y10 = vaddq_f32(t00, t10), y11 = vaddq_f32(t01, t11);
float32x4_t y20 = vsubq_f32(t10, t00), y21 = vsubq_f32(t11, t01);
/* Y[3] = [0.f, 0.5f, 0.25f, -2.5f, -1.25f, 2.f, 1.f, 0.f]*X */
/* Y[4] = [0.f, -0.5f, 0.25f, 2.5f, -1.25f, -2.f, 1.f, 0.f]*X */
// Y[3] = [0.f, 0.5f, 0.25f, -2.5f, -1.25f, 2.f, 1.f, 0.f]*X
// Y[4] = [0.f, -0.5f, 0.25f, 2.5f, -1.25f, -2.f, 1.f, 0.f]*X
float32x4_t q0_5 = vdupq_n_f32(0.5f), q0_25 = vdupq_n_f32(0.25f);
float32x4_t qm2_5 = vdupq_n_f32(-2.5f), qm1_25 = vdupq_n_f32(-1.25f);
t00 = vfmaq_f32(vaddq_f32(x50, x50), x10, q0_5);
@@ -616,8 +644,8 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep,
float32x4_t y30 = vaddq_f32(t00, t10), y31 = vaddq_f32(t01, t11);
float32x4_t y40 = vsubq_f32(t10, t00), y41 = vsubq_f32(t11, t01);
/* Y[5] = [0.f, 2.f, 4.f, -2.5f, -5.f, 0.5f, 1.f, 0.f]*X */
/* Y[6] = [0.f, -2.f, 4.f, 2.5f, -5.f, -0.5f, 1.f, 0.f]*X */
// Y[5] = [0.f, 2.f, 4.f, -2.5f, -5.f, 0.5f, 1.f, 0.f]*X
// Y[6] = [0.f, -2.f, 4.f, 2.5f, -5.f, -0.5f, 1.f, 0.f]*X
float32x4_t q4 = vdupq_n_f32(4.f), qm5 = vdupq_n_f32(-5.f);
t00 = vfmaq_f32(vaddq_f32(x10, x10), x50, q0_5);
t01 = vfmaq_f32(vaddq_f32(x11, x11), x51, q0_5);
@@ -631,22 +659,22 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep,
float32x4_t y50 = vaddq_f32(t00, t10), y51 = vaddq_f32(t01, t11);
float32x4_t y60 = vsubq_f32(t10, t00), y61 = vsubq_f32(t11, t01);
/* transpose 8x8 matrix in-place with some renumeration of the elements: */
/* Y: */
/* y00 y01 */
/* y10 y11 */
/* ... */
/* y70 y71 */
/* Y': */
/* y00 y40 */
/* y10 y50 */
/* y20 y60 */
/* y30 y70 */
/* y01 y41 */
/* y11 y51 */
/* y21 y61 */
/* y31 y71 */
/* in other words, y40 <-> y01, y50 <-> y11, y60 <-> y21, y70 <-> y31 */
// transpose 8x8 matrix in-place with some renumeration of the elements:
// Y:
// y00 y01
// y10 y11
// ...
// y70 y71
// Y':
// y00 y40
// y10 y50
// y20 y60
// y30 y70
// y01 y41
// y11 y51
// y21 y61
// y31 y71
// in other words, y40 <-> y01, y50 <-> y11, y60 <-> y21, y70 <-> y31
float32x4x2_t tr0, tr1;
T4x4(y00, y10, y20, y30, tr0, tr1);
@@ -654,8 +682,8 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep,
T4x4(y40, y50, y60, y70, tr0, tr1);
T4x4(y41, y51, y61, y71, tr0, tr1);
/* Z[0] = [1.f, 0.f, -5.25f, 0.f, 5.25f, 0.f, -1.f, 0.f]*Y */
/* Z[7] = [0.f, -1.f, 0.f, 5.25f, 0.f, -5.25f, 0.f, 1.f]*Y */
// Z[0] = [1.f, 0.f, -5.25f, 0.f, 5.25f, 0.f, -1.f, 0.f]*Y
// Z[7] = [0.f, -1.f, 0.f, 5.25f, 0.f, -5.25f, 0.f, 1.f]*Y
t00 = vsubq_f32(y01, y20);
t01 = vsubq_f32(y41, y60);
t10 = vsubq_f32(y30, y11);
@@ -665,8 +693,8 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep,
z70 = vfmaq_f32(vsubq_f32(y31, y10), t10, q5_25);
z71 = vfmaq_f32(vsubq_f32(y71, y50), t11, q5_25);
/* Z[1] = [0.f, 1.f, 1.f, -4.25f, -4.25f, 1.f, 1.f, 0.f]*Y */
/* Z[2] = [0.f, -1.f, 1.f, 4.25f, -4.25f, -1.f, 1.f, 0.f]*Y */
// Z[1] = [0.f, 1.f, 1.f, -4.25f, -4.25f, 1.f, 1.f, 0.f]*Y
// Z[2] = [0.f, -1.f, 1.f, 4.25f, -4.25f, -1.f, 1.f, 0.f]*Y
t00 = vfmaq_f32(vaddq_f32(y10, y11), y30, qm4_25);
t01 = vfmaq_f32(vaddq_f32(y50, y51), y70, qm4_25);
t10 = vfmaq_f32(vaddq_f32(y20, y21), y01, qm4_25);
@@ -675,8 +703,8 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep,
z10 = vaddq_f32(t00, t10); z11 = vaddq_f32(t01, t11);
z20 = vsubq_f32(t10, t00); z21 = vsubq_f32(t11, t01);
/* Z[3] = [0.f, 0.5f, 0.25f, -2.5f, -1.25f, 2.f, 1.f, 0.f]*Y */
/* Z[4] = [0.f, -0.5f, 0.25f, 2.5f, -1.25f, -2.f, 1.f, 0.f]*Y */
// Z[3] = [0.f, 0.5f, 0.25f, -2.5f, -1.25f, 2.f, 1.f, 0.f]*Y
// Z[4] = [0.f, -0.5f, 0.25f, 2.5f, -1.25f, -2.f, 1.f, 0.f]*Y
t00 = vfmaq_f32(vaddq_f32(y11, y11), y10, q0_5);
t01 = vfmaq_f32(vaddq_f32(y51, y51), y50, q0_5);
t10 = vfmaq_f32(y21, y20, q0_25);
@@ -689,8 +717,8 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep,
z30 = vaddq_f32(t00, t10); z31 = vaddq_f32(t01, t11);
z40 = vsubq_f32(t10, t00); z41 = vsubq_f32(t11, t01);
/* Z[5] = [0.f, 2.f, 4.f, -2.5f, -5.f, 0.5f, 1.f, 0.f]*Y */
/* Z[6] = [0.f, -2.f, 4.f, 2.5f, -5.f, -0.5f, 1.f, 0.f]*Y */
// Z[5] = [0.f, 2.f, 4.f, -2.5f, -5.f, 0.5f, 1.f, 0.f]*Y
// Z[6] = [0.f, -2.f, 4.f, 2.5f, -5.f, -0.5f, 1.f, 0.f]*Y
t00 = vfmaq_f32(vaddq_f32(y10, y10), y11, q0_5);
t01 = vfmaq_f32(vaddq_f32(y50, y50), y51, q0_5);
t10 = vfmaq_f32(y21, y20, q4);
@@ -704,39 +732,42 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep,
z60 = vsubq_f32(t10, t00); z61 = vsubq_f32(t11, t01);
}
const int outstep = winoIblock*winoAtomF32*Cg;
const int outstep = winoIblock*winoAtomF16*Cg;
vst1q_f32(outptr, z00);
vst1q_f32(outptr + outstep, z01);
vst1q_f32(outptr + outstep*2, z10);
vst1q_f32(outptr + outstep*3, z11);
vst1q_f32(outptr + outstep*4, z20);
vst1q_f32(outptr + outstep*5, z21);
vst1q_f32(outptr + outstep*6, z30);
vst1q_f32(outptr + outstep*7, z31);
vst1q_f32(outptr + outstep*8, z40);
vst1q_f32(outptr + outstep*9, z41);
vst1q_f32(outptr + outstep*10, z50);
vst1q_f32(outptr + outstep*11, z51);
vst1q_f32(outptr + outstep*12, z60);
vst1q_f32(outptr + outstep*13, z61);
vst1q_f32(outptr + outstep*14, z70);
vst1q_f32(outptr + outstep*15, z71);
vst1_f16(outptr, vcvt_f16_f32(z00));
vst1_f16(outptr + 4, vcvt_f16_f32(z01));
vst1_f16(outptr + outstep, vcvt_f16_f32(z10));
vst1_f16(outptr + outstep + 4, vcvt_f16_f32(z11));
vst1_f16(outptr + outstep*2, vcvt_f16_f32(z20));
vst1_f16(outptr + outstep*2 + 4, vcvt_f16_f32(z21));
vst1_f16(outptr + outstep*3, vcvt_f16_f32(z30));
vst1_f16(outptr + outstep*3 + 4, vcvt_f16_f32(z31));
vst1_f16(outptr + outstep*4, vcvt_f16_f32(z40));
vst1_f16(outptr + outstep*4 + 4, vcvt_f16_f32(z41));
vst1_f16(outptr + outstep*5, vcvt_f16_f32(z50));
vst1_f16(outptr + outstep*5 + 4, vcvt_f16_f32(z51));
vst1_f16(outptr + outstep*6, vcvt_f16_f32(z60));
vst1_f16(outptr + outstep*6 + 4, vcvt_f16_f32(z61));
vst1_f16(outptr + outstep*7, vcvt_f16_f32(z70));
vst1_f16(outptr + outstep*7 + 4, vcvt_f16_f32(z71));
}
/*Output transform*/
void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep,
float* bpptr, int bpstep, float* outptr, int outstep,
float bias, float minval, float maxval, bool ifMinMaxAct)
// Output transform
void winofunc_AtXA_8x8_F16(const char* _inptr, int inpstep,
float * bpptr, int bpstep, float* outptr, int outstep,
float bias, float minval, float maxval, bool ifMinMaxAct)
{
float32x4_t x00 = vld1q_f32(inptr), x01 = vld1q_f32(inptr + 4);
float32x4_t x10 = vld1q_f32(inptr + inpstep), x11 = vld1q_f32(inptr + inpstep + 4);
float32x4_t x20 = vld1q_f32(inptr + inpstep*2), x21 = vld1q_f32(inptr + inpstep*2 + 4);
float32x4_t x30 = vld1q_f32(inptr + inpstep*3), x31 = vld1q_f32(inptr + inpstep*3 + 4);
float32x4_t x40 = vld1q_f32(inptr + inpstep*4), x41 = vld1q_f32(inptr + inpstep*4 + 4);
float32x4_t x50 = vld1q_f32(inptr + inpstep*5), x51 = vld1q_f32(inptr + inpstep*5 + 4);
float32x4_t x60 = vld1q_f32(inptr + inpstep*6), x61 = vld1q_f32(inptr + inpstep*6 + 4);
float32x4_t x70 = vld1q_f32(inptr + inpstep*7), x71 = vld1q_f32(inptr + inpstep*7 + 4);
typedef __fp16 float16_t;
const float16_t* inptr = (const float16_t*)_inptr;
float32x4_t x00 = vcvt_f32_f16(vld1_f16(inptr)), x01 = vcvt_f32_f16(vld1_f16(inptr + 4));
float32x4_t x10 = vcvt_f32_f16(vld1_f16(inptr + inpstep)), x11 = vcvt_f32_f16(vld1_f16(inptr + inpstep + 4));
float32x4_t x20 = vcvt_f32_f16(vld1_f16(inptr + inpstep*2)), x21 = vcvt_f32_f16(vld1_f16(inptr + inpstep*2 + 4));
float32x4_t x30 = vcvt_f32_f16(vld1_f16(inptr + inpstep*3)), x31 = vcvt_f32_f16(vld1_f16(inptr + inpstep*3 + 4));
float32x4_t x40 = vcvt_f32_f16(vld1_f16(inptr + inpstep*4)), x41 = vcvt_f32_f16(vld1_f16(inptr + inpstep*4 + 4));
float32x4_t x50 = vcvt_f32_f16(vld1_f16(inptr + inpstep*5)), x51 = vcvt_f32_f16(vld1_f16(inptr + inpstep*5 + 4));
float32x4_t x60 = vcvt_f32_f16(vld1_f16(inptr + inpstep*6)), x61 = vcvt_f32_f16(vld1_f16(inptr + inpstep*6 + 4));
float32x4_t x70 = vcvt_f32_f16(vld1_f16(inptr + inpstep*7)), x71 = vcvt_f32_f16(vld1_f16(inptr + inpstep*7 + 4));
float32x4_t z00, z01, z10, z11, z20, z21, z30, z31, z40, z41, z50, z51;
{
@@ -757,33 +788,33 @@ void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep,
s56_0 = vsubq_f32(x50, x60); s56_1 = vsubq_f32(x51, x61);
float32x4_t y50 = vfmaq_n_f32(vfmaq_n_f32(vaddq_f32(x70, s12_0),
s34_0, 32.f), s56_0, 1.f/32);
s34_0, 32.f), s56_0, 1.f/32);
float32x4_t y51 = vfmaq_n_f32(vfmaq_n_f32(vaddq_f32(x71, s12_1),
s34_1, 32.f), s56_1, 1.f/32);
s34_1, 32.f), s56_1, 1.f/32);
float32x4_t y10 = vfmaq_n_f32(vfmaq_n_f32(s12_0, s34_0, 2.0f), s56_0, 0.5f);
float32x4_t y11 = vfmaq_n_f32(vfmaq_n_f32(s12_1, s34_1, 2.0f), s56_1, 0.5f);
float32x4_t y30 = vfmaq_n_f32(vfmaq_n_f32(s12_0, s34_0, 8.0f), s56_0, 0.125f);
float32x4_t y31 = vfmaq_n_f32(vfmaq_n_f32(s12_1, s34_1, 8.0f), s56_1, 0.125f);
float32x4_t y60 = vdupq_n_f32(0.f), y61 = y60, y70 = y60, y71 = y60;
/* transpose 8x8 matrix in-place with some renumeration of the elements: */
/* Y: */
/* y00 y01 */
/* y10 y11 */
/* ... */
/* y50 y51 */
/* 0 0 */
/* 0 0 */
/* Y': */
/* y00 y40 */
/* y10 y50 */
/* y20 y60 */
/* y30 y70 */
/* y01 y41 */
/* y11 y51 */
/* y21 y61 */
/* y31 y71 */
/* in other words, y40 <-> y01, y50 <-> y11, y60 <-> y21, y70 <-> y31 */
// transpose 8x8 matrix in-place with some renumeration of the elements:
// Y:
// y00 y01
// y10 y11
// ...
// y50 y51
// 0 0
// 0 0
// Y':
// y00 y40
// y10 y50
// y20 y60
// y30 y70
// y01 y41
// y11 y51
// y21 y61
// y31 y71
// in other words, y40 <-> y01, y50 <-> y11, y60 <-> y21, y70 <-> y31
float32x4x2_t tr0, tr1;
T4x4(y00, y10, y20, y30, tr0, tr1);
@@ -807,9 +838,9 @@ void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep,
s56_0 = vsubq_f32(y11, y21); s56_1 = vsubq_f32(y51, y61);
z50 = vfmaq_n_f32(vfmaq_n_f32(vaddq_f32(y31, s12_0),
s34_0, 32.f), s56_0, 1.f/32);
s34_0, 32.f), s56_0, 1.f/32);
z51 = vfmaq_n_f32(vfmaq_n_f32(vaddq_f32(y71, s12_1),
s34_1, 32.f), s56_1, 1.f/32);
s34_1, 32.f), s56_1, 1.f/32);
z10 = vfmaq_n_f32(vfmaq_n_f32(s12_0, s34_0, 2.0f), s56_0, 0.5f);
z11 = vfmaq_n_f32(vfmaq_n_f32(s12_1, s34_1, 2.0f), s56_1, 0.5f);
z30 = vfmaq_n_f32(vfmaq_n_f32(s12_0, s34_0, 8.0f), s56_0, 0.125f);
@@ -879,8 +910,8 @@ void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep,
vst1q_f32(outptr + outstep*5, z50);
vst1_f32(outptr + outstep*5 + 4, vget_low_f32(z51));
}
#endif
}
#endif
CV_CPU_OPTIMIZATION_NAMESPACE_END
}} // namespace
modules/dnn/src/layers/cpu_kernels/convolution.cpp
+118 -119
View File
@@ -14,15 +14,76 @@
#include "conv_block.simd.hpp"
#include "layers/cpu_kernels/conv_block.simd_declarations.hpp" // defines CV_CPU_DISPATCH_MODES_ALL=AVX2,...,BASELINE based on CMakeLists.txt content
#include <opencv2/core/utils/logger.hpp>
namespace cv { namespace dnn {
enum { VEC_ALIGN = 32, DFT_TYPE = CV_32F }; // Memory alignment.
enum { VEC_ALIGN = 32}; // Memory alignment.
void convBlock(int np, const float* a, const float* b, float* c, int ldc, bool init_c, const int outLen,
void convBlock_F32(int np, const float* a, const float* b, float* c, int ldc, bool init_c, const int outLen,
const int convMR, const int convNR);
void convBlockMR1(int np, const float* a, const float* b, float *c, const float bias, bool init_c,
void convBlockMR1_F32(int np, const float* a, const float* b, float *c, const float bias, bool init_c,
const float minval, const float maxval, bool ifMinMaxAct, const int outLen, const int convNR);
#ifdef CONV_ARM_FP16
// Fast convert float 32 to float16
static inline void _cvt32f16f(const float* src, float16_t* dst, int len)
{
int j = 0;
const int VECSZ = 4;
__fp16* dst_FP16 = (__fp16 *)dst;
if (len > VECSZ * 4)
{
const int VECSZ4 = 4 * VECSZ;
for( ; j + VECSZ4 < len; j += VECSZ4)
{
float32x4_t v0 = vld1q_f32(src + j);
float32x4_t v1 = vld1q_f32(src + j + 4);
float32x4_t v2 = vld1q_f32(src + j + 8);
float32x4_t v3 = vld1q_f32(src + j + 12);
vst1q_f16(dst_FP16 + j, vcombine_f16(vcvt_f16_f32(v0), vcvt_f16_f32(v1)));
vst1q_f16(dst_FP16 + j + 8, vcombine_f16(vcvt_f16_f32(v2), vcvt_f16_f32(v3)));
}
}
for( ; j < len; j += VECSZ )
{
if( j > len - VECSZ )
{
if( j == 0 )
break;
j = len - VECSZ;
}
float16x4_t hv = vcvt_f16_f32(vld1q_f32(src + j));
vst1_f16(dst_FP16 + j, hv);
}
for( ; j < len; j++ )
dst[j] = float16_t(src[j]);
}
#endif
float* FastConv::getWeights()
{
return alignPtr(weightsBuf.data(), VEC_ALIGN);
}
float* FastConv::getWeightsWino()
{
return alignPtr(weightsWinoBuf.data(), VEC_ALIGN);
}
float16_t* FastConv::getWeightsFP16()
{
return alignPtr(weightsBuf_FP16.data(), VEC_ALIGN);
}
float16_t* FastConv::getWeightsWinoFP16()
{
return alignPtr(weightsWinoBuf_FP16.data(), VEC_ALIGN);
}
Ptr<FastConv> initFastConv(
InputArray _weightsMat,
float* srcBias,
@@ -119,9 +180,16 @@ Ptr<FastConv> initFastConv(
conv->useFP16 = false;
#ifdef CONV_ARM_FP16
// TODO: add FP16 support for Winograd.
if (_useFP16 && (conv->conv_type == CONV_TYPE_GENERIC || conv->conv_type == CONV_TYPE_DEPTHWISE_REMAIN))
if (_useFP16 && (conv->conv_type == CONV_TYPE_GENERIC || conv->conv_type == CONV_TYPE_DEPTHWISE_REMAIN
|| conv->conv_type == CONV_TYPE_WINOGRAD3X3))
conv->useFP16 = true;
// Runtime FP16 check.
if (conv->useFP16 && !checkHardwareSupport(CPU_NEON_FP16))
{
conv->useFP16 = false;
CV_LOG_ONCE_WARNING(NULL, "DNN: the CPU does not support the instruction set required by FP16, fallback to FP32.");
}
#endif
float *srcWeights = (float *)weightsMat.data;
@@ -141,31 +209,25 @@ Ptr<FastConv> initFastConv(
if (conv->useFP16)
{
conv->weightsBuf_FP16.resize(nweights + VEC_ALIGN);
conv->weightsBufPtr_FP16 = alignPtr(conv->weightsBuf_FP16.data(), VEC_ALIGN * sizeof(float16_t ));
memset(conv->weightsBufPtr_FP16, 0, nweights * sizeof(float16_t ));
auto weightsBufPtr_FP16 = conv->weightsBufPtr_FP16;
auto weightsPtr_FP16 = conv->getWeightsFP16();
memset(reinterpret_cast<short*>(weightsPtr_FP16), 0, nweights * sizeof(weightsPtr_FP16[0]));
parallel_for_(Range(0, C), [&](const Range& r0){
for(int c = r0.start; c < r0.end; c++)
{
for (int k = 0; k < ksize; k++)
weightsBufPtr_FP16[c*padded_ksize + k] = (float16_t)srcWeights[c*wstep + k];
}});
for(int c = r0.start; c < r0.end; c++)
_cvt32f16f(srcWeights + c*wstep, weightsPtr_FP16 + c*padded_ksize, ksize);
});
}
else
#endif
{
conv->weightsBuf.resize(nweights + VEC_ALIGN);
conv->weightsBufPtr = alignPtr(conv->weightsBuf.data(), VEC_ALIGN * sizeof(float ));
memset(conv->weightsBufPtr, 0, nweights*sizeof(float ));
auto weightsBufPtr = conv->weightsBufPtr;
auto weightsPtr = conv->getWeights();
memset(weightsPtr, 0, nweights*sizeof(weightsPtr[0]));
parallel_for_(Range(0, C), [&](const Range& r0){
for(int c = r0.start; c < r0.end; c++)
{
for (int k = 0; k < ksize; k++)
weightsBufPtr[c*padded_ksize + k] = srcWeights[c*wstep + k];
}});
parallel_for_(Range(0, C), [&](const Range& r0) {
for(int c = r0.start; c < r0.end; c++)
memcpy(weightsPtr + c*padded_ksize, srcWeights + c*wstep, ksize*sizeof(weightsPtr[0]));
});
}
}
else if(conv->conv_type == CONV_TYPE_WINOGRAD3X3) // winograd
@@ -213,16 +275,14 @@ Ptr<FastConv> initFastConv(
if (conv->useFP16)
{
conv->weightsWinoBuf_FP16.resize(nweights + VEC_ALIGN);
conv->weightsWinoBufPtr_FP16 = alignPtr(conv->weightsWinoBuf_FP16.data(), VEC_ALIGN);
wptrWino_FP16 = conv->weightsWinoBufPtr_FP16;
memset(wptrWino_FP16, 0, nweights * sizeof(wptrWino_FP16[0]));
wptrWino_FP16 = conv->getWeightsWinoFP16();
memset(reinterpret_cast<short*>(wptrWino_FP16), 0, nweights * sizeof(wptrWino_FP16[0]));
}
else
#endif
{
conv->weightsWinoBuf.resize(nweights + VEC_ALIGN);
conv->weightsWinoBufPtr = alignPtr(conv->weightsWinoBuf.data(), VEC_ALIGN);
wptrWino = conv->weightsWinoBufPtr;
wptrWino = conv->getWeightsWino();
memset(wptrWino, 0, nweights * sizeof(wptrWino[0]));
}
@@ -272,7 +332,7 @@ Ptr<FastConv> initFastConv(
for (int i = 0; i < CONV_WINO_NATOMS_F16; i++,
wptr += Cg * CONV_WINO_KBLOCK * CONV_WINO_ATOM_F16)
{
CV_Assert(conv->weightsWinoBufPtr_FP16 <= wptr && wptr + CONV_WINO_ATOM_F16 <= conv->weightsWinoBufPtr_FP16 + nweights);
CV_Assert(wptrWino_FP16 <= wptr && wptr + CONV_WINO_ATOM_F16 <= wptrWino_FP16 + nweights);
for (int j = 0; j < CONV_WINO_ATOM_F16; j++)
{
wptr[j] = (float16_t)kernelTm[i * CONV_WINO_ATOM_F16 + j];
@@ -287,7 +347,7 @@ Ptr<FastConv> initFastConv(
for (int i = 0; i < CONV_WINO_NATOMS_F32; i++,
wptr += Cg * CONV_WINO_KBLOCK * CONV_WINO_ATOM_F32)
{
CV_Assert(conv->weightsWinoBufPtr <= wptr && wptr + CONV_WINO_ATOM_F32 <= conv->weightsWinoBufPtr + nweights);
CV_Assert(wptrWino <= wptr && wptr + CONV_WINO_ATOM_F32 <= wptrWino + nweights);
memcpy(wptr, kernelTm + i * CONV_WINO_ATOM_F32, CONV_WINO_ATOM_F32*sizeof (wptr[0]));
}
}
@@ -305,29 +365,26 @@ Ptr<FastConv> initFastConv(
int numStripsMR = (Kg + CONV_MR_FP32 - 1) / CONV_MR_FP32;
int Kg_aligned = numStripsMR * CONV_MR_FP32;
size_t nweights = ngroups*Kg_aligned*DkHkWkCg;
float* weightsBufPtr = nullptr;
float* weightsPtr = nullptr;
#ifdef CONV_ARM_FP16
int numStripsMR_FP16 = (Kg + CONV_MR_FP16 - 1) / CONV_MR_FP16;
int Kg_aligned_FP16 = numStripsMR_FP16 * CONV_MR_FP16;
size_t nweights_FP16 = ngroups * Kg_aligned_FP16 * DkHkWkCg;
float16_t* weightsPtr_FP16 = nullptr;
float16_t* weightsBufPtr_FP16 = nullptr;
if (conv->useFP16)
{
conv->weightsBuf_FP16.resize(nweights_FP16 + VEC_ALIGN);
conv->weightsBufPtr_FP16 = alignPtr(conv->weightsBuf_FP16.data(), VEC_ALIGN);
weightsBufPtr_FP16 = conv->weightsBufPtr_FP16;
memset(weightsBufPtr_FP16, 0, nweights_FP16*sizeof(weightsBufPtr_FP16[0]));
weightsPtr_FP16 = conv->getWeightsFP16();
memset(reinterpret_cast<short*>(weightsPtr_FP16), 0, nweights_FP16*sizeof(weightsPtr_FP16[0]));
}
else
#endif
{
conv->weightsBuf.resize(nweights + VEC_ALIGN);
conv->weightsBufPtr = alignPtr(conv->weightsBuf.data(), VEC_ALIGN);
weightsBufPtr = conv->weightsBufPtr;
memset(weightsBufPtr, 0, nweights*sizeof(weightsBufPtr[0]));
weightsPtr = conv->getWeights();
memset(weightsPtr, 0, nweights*sizeof(weightsPtr[0]));
}
// Pack the weight.
@@ -343,7 +400,7 @@ Ptr<FastConv> initFastConv(
int startK = si * CONV_MR_FP16;
CV_Assert(startK < Kg_aligned_FP16);
float16_t* packed_wptr = weightsBufPtr_FP16 + DkHkWkCg * (startK + g * Kg_aligned_FP16);
float16_t* packed_wptr = weightsPtr_FP16 + DkHkWkCg * (startK + g * Kg_aligned_FP16);
int dk = Kg - startK < CONV_MR_FP16 ? Kg - startK : CONV_MR_FP16; // check if we need zero padding.
int k_idx = g*Kg + startK;
@@ -373,7 +430,7 @@ Ptr<FastConv> initFastConv(
int startK = si * CONV_MR_FP32;
CV_Assert(startK < Kg_aligned);
float* packed_wptr = weightsBufPtr + DkHkWkCg * (startK + g * Kg_aligned);
float* packed_wptr = weightsPtr + DkHkWkCg * (startK + g * Kg_aligned);
int dk = Kg - startK < CONV_MR_FP32 ? Kg - startK : CONV_MR_FP32; // check if we need zero padding.
int k_idx = g*Kg + startK;
@@ -410,7 +467,7 @@ Ptr<FastConv> initFastConv(
}
static inline void packData8(char*& inpbuf, float*& inptrIn, int& in_w, int& x0, int& s0, const int* ofstab,
const int stride_w, const int ksize, const int esz)
const int stride_w, const int ksize, const int esz)
{
char * inpbufC = inpbuf + s0 * esz;
float* inptrInC = (float* )inptrIn;
@@ -435,16 +492,8 @@ static inline void packData8(char*& inpbuf, float*& inptrIn, int& in_w, int& x0,
for (int k = 0; k < ksize; k++)
{
int k1 = ofstab[k];
float32x4_t v0, v1;
v0[0] = inptrInC[k1];
v0[1] = inptrInC[k1 + stride_w];
v0[2] = inptrInC[k1 + 2*stride_w];
v0[3] = inptrInC[k1 + 3*stride_w];
v1[0] = inptrInC[k1 + 4*stride_w];
v1[1] = inptrInC[k1 + 5*stride_w];
v1[2] = inptrInC[k1 + 6*stride_w];
v1[3] = inptrInC[k1 + 7*stride_w];
float32x4_t v0 = {inptrInC[k1], inptrInC[k1 + stride_w], inptrInC[k1 + 2*stride_w], inptrInC[k1 + 3*stride_w]};
float32x4_t v1 = {inptrInC[k1 + 4*stride_w], inptrInC[k1 + 5*stride_w], inptrInC[k1 + 6*stride_w], inptrInC[k1 + 7*stride_w]};
vst1q_f16((__fp16*)inpbufC_FP16 + k * CONV_NR_FP16, vcombine_f16(vcvt_f16_f32(v0), vcvt_f16_f32(v1)));
}
@@ -516,7 +565,7 @@ static inline void packData8(char*& inpbuf, float*& inptrIn, int& in_w, int& x0,
}
static inline void packData2(char *& inpbuf, float*& inptrIn, int& in_w, int& x0, int& s0, const int* ofstab,
const int stride_w, const int ksize, const int esz)
const int stride_w, const int ksize, const int esz)
{
char* inpbufC = inpbuf + s0 * esz;
float* inptrInC = inptrIn;
@@ -553,46 +602,6 @@ static inline void packData2(char *& inpbuf, float*& inptrIn, int& in_w, int& x0
in_w += stride_w;
}
#ifdef CONV_ARM_FP16
// Fast convert float 32 to float16
static inline void _cvt32f16f( const float* src, float16_t* dst, int len)
{
int j = 0;
const int VECSZ = 4;
__fp16* dst_FP16 = (__fp16 *)dst;
if (len > VECSZ * 4)
{
const int VECSZ4 = 4 * VECSZ;
for( ; j + VECSZ4 < len; j += VECSZ4)
{
float32x4_t v0 = vld1q_f32(src + j);
float32x4_t v1 = vld1q_f32(src + j + 4);
float32x4_t v2 = vld1q_f32(src + j + 8);
float32x4_t v3 = vld1q_f32(src + j + 12);
vst1q_f16(dst_FP16 + j, vcombine_f16(vcvt_f16_f32(v0), vcvt_f16_f32(v1)));
vst1q_f16(dst_FP16 + j + 8, vcombine_f16(vcvt_f16_f32(v2), vcvt_f16_f32(v3)));
}
}
for( ; j < len; j += VECSZ )
{
if( j > len - VECSZ )
{
if( j == 0 )
break;
j = len - VECSZ;
}
float16x4_t hv = vcvt_f16_f32(vld1q_f32(src + j));
vst1_f16(dst_FP16 + j, hv);
}
for( ; j < len; j++ )
dst[j] = float16_t(src[j]);
}
#endif
static inline void packInputData(char* inpbuf_task, float* inp, const int* ofstab, const int* dhwTab, int zyx0, int zyx_limit,
int ksize, int stride_d, int stride_h, int stride_w, int pad_front, int pad_top, int pad_left,
int Dk, int Hk, int Wk, int dilation_d, int dilation_h, int dilation_w, int Di, int Hi, int Wi,
@@ -939,11 +948,8 @@ static inline void packInputData(char* inpbuf_task, float* inp, const int* ofsta
{
for (int c = 0; c < Cg; c++, inpbuf_ki_FP16 += CONV_NR, inptr_ki += inp_planesize)
{
float32x4_t v0, v1;
v0[0] = inptr_ki[0], v0[1] = inptr_ki[2];
v0[2] = inptr_ki[4], v0[3] = inptr_ki[6];
v1[0] = inptr_ki[8], v1[1] = inptr_ki[10];
v1[2] = inptr_ki[12], v1[3] = inptr_ki[14];
float32x4_t v0 = {inptr_ki[0], inptr_ki[2], inptr_ki[4], inptr_ki[6]};
float32x4_t v1 = {inptr_ki[8], inptr_ki[10], inptr_ki[12], inptr_ki[14]};
vst1q_f16((__fp16* )inpbuf_ki_FP16, vcombine_f16(vcvt_f16_f32(v0), vcvt_f16_f32(v1)));
}
}
@@ -972,12 +978,8 @@ static inline void packInputData(char* inpbuf_task, float* inp, const int* ofsta
{
for (int c = 0; c < Cg; c++, inpbuf_ki_FP16 += CONV_NR, inptr_ki += inp_planesize)
{
float32x4_t v0, v1;
v0[0] = inptr_ki[0], v0[1] = inptr_ki[stride_w];
v0[2] = inptr_ki[stride_w * 2], v0[3] = inptr_ki[stride_w * 3];
v1[0] = inptr_ki[stride_w * 4], v1[1] = inptr_ki[stride_w * 5];
v1[2] = inptr_ki[stride_w * 6], v1[3] = inptr_ki[stride_w * 7];
float32x4_t v0 = {inptr_ki[0], inptr_ki[stride_w], inptr_ki[stride_w * 2], inptr_ki[stride_w * 3]};
float32x4_t v1 = {inptr_ki[stride_w * 4], inptr_ki[stride_w * 5], inptr_ki[stride_w * 6], inptr_ki[stride_w * 7]};
vst1q_f16((__fp16* )inpbuf_ki_FP16, vcombine_f16(vcvt_f16_f32(v0), vcvt_f16_f32(v1)));
}
}
@@ -1034,9 +1036,7 @@ static inline void packInputData(char* inpbuf_task, float* inp, const int* ofsta
{
for (int c = 0; c < Cg; c++, inpbuf_ki_FP16 += CONV_NR, inptr_ki += inp_planesize)
{
float32x4_t v0;
v0[0] = inptr_ki[0], v0[1] = inptr_ki[stride_w];
v0[2] = inptr_ki[stride_w * 2], v0[3] = inptr_ki[stride_w * 3];
float32x4_t v0 = {inptr_ki[0], inptr_ki[stride_w], inptr_ki[stride_w * 2], inptr_ki[stride_w * 3]};
vst1_f16((__fp16* )inpbuf_ki_FP16, vcvt_f16_f32(v0));
}
}
@@ -1174,10 +1174,9 @@ void runFastConv(InputArray _input, OutputArray _output, const Ptr<FastConv>& co
else
activ = nullptr;
// TODO: support FP16 for winograd.
if (conv->conv_type == CONV_TYPE_WINOGRAD3X3) // winograd
{
CV_Assert(conv->weightsWinoBufPtr && input.dims == 4 && conv_dim == CONV_2D && !useFP16);
CV_Assert((!conv->weightsWinoBuf.empty() || !conv->weightsWinoBuf_FP16.empty()) && input.dims == 4 && conv_dim == CONV_2D);
if (runWinograd63(input, fusedAddMat, output, conv, ntasks, minval, maxval, activ, ifMinMaxAct))
return;
}
@@ -1437,13 +1436,13 @@ void runFastConv(InputArray _input, OutputArray _output, const Ptr<FastConv>& co
if (useFP16)
{
CV_Assert(!conv->weightsBuf_FP16.empty());
weights = (char *)conv->weightsBufPtr_FP16;
weights = (char *)conv->getWeightsFP16();
}
else
#endif
{
CV_Assert(!conv->weightsBuf.empty());
weights = (char *)conv->weightsBufPtr;
weights = (char *)conv->getWeights();
}
// optional branch, only for depth-wise convolution which was implemented by generic convolution.
// In this case, CONV_MR is 1, and CONV_NR remains the same.
@@ -1477,7 +1476,7 @@ void runFastConv(InputArray _input, OutputArray _output, const Ptr<FastConv>& co
#ifdef CONV_ARM_FP16
if (useFP16)
{
opt_NEON::convBlockMR1_FP16(DkHkWkCg, weights, inptr, cptr, biasVal, fusedAdd, minval, maxval, ifMinMaxAct, outLen, CONV_NR);
opt_NEON_FP16::convBlockMR1_F16(DkHkWkCg, weights, inptr, cptr, biasVal, fusedAdd, minval, maxval, ifMinMaxAct, outLen, CONV_NR);
}
else
#endif
@@ -1485,7 +1484,7 @@ void runFastConv(InputArray _input, OutputArray _output, const Ptr<FastConv>& co
}
else
#endif
convBlockMR1(DkHkWkCg, (const float *)weights, (const float *)inptr, cptr, biasVal, fusedAdd, minval, maxval, ifMinMaxAct, outLen, CONV_NR);
convBlockMR1_F32(DkHkWkCg, (const float *)weights, (const float *)inptr, cptr, biasVal, fusedAdd, minval, maxval, ifMinMaxAct, outLen, CONV_NR);
if (ifBuffer)
{
@@ -1526,12 +1525,12 @@ void runFastConv(InputArray _input, OutputArray _output, const Ptr<FastConv>& co
{
#if CV_TRY_AVX2
if (conv->useAVX2)
opt_AVX2::convBlock(c1 - c0, (const float *)wptr, (const float *)inptr, cptr, ldc, c0 == 0, outLen, CONV_MR, CONV_NR);
opt_AVX2::convBlock_F32(c1 - c0, (const float *)wptr, (const float *)inptr, cptr, ldc, c0 == 0, outLen, CONV_MR, CONV_NR);
else
#endif
#if CV_TRY_AVX
if (conv->useAVX)
opt_AVX::convBlock(c1 - c0, (const float *)wptr, (const float *)inptr, cptr, ldc, c0 == 0, outLen, CONV_MR, CONV_NR);
opt_AVX::convBlock_F32(c1 - c0, (const float *)wptr, (const float *)inptr, cptr, ldc, c0 == 0, outLen, CONV_MR, CONV_NR);
else
#endif
#if CV_NEON
@@ -1540,16 +1539,16 @@ void runFastConv(InputArray _input, OutputArray _output, const Ptr<FastConv>& co
#ifdef CONV_ARM_FP16
if (useFP16)
{
opt_NEON::convBlock_FP16(c1 - c0, wptr, inptr, (char *)cptr_f16, ldc, c0 == 0, outLen, CONV_MR, CONV_NR);
opt_NEON_FP16::convBlock_F16(c1 - c0, wptr, inptr, (char *)cptr_f16, ldc, c0 == 0, outLen, CONV_MR, CONV_NR);
}
else
#endif
opt_NEON::convBlock(c1 - c0, (const float *)wptr, (const float *)inptr, cptr, ldc, c0 == 0, outLen, CONV_MR, CONV_NR);
opt_NEON::convBlock_F32(c1 - c0, (const float *)wptr, (const float *)inptr, cptr, ldc, c0 == 0, outLen, CONV_MR, CONV_NR);
}
else
#endif
// The possible outLen range is 24 or 8~1.
convBlock(c1 - c0, (const float *)wptr, (const float *)inptr, cptr, ldc, c0 == 0, outLen, CONV_MR, CONV_NR);
convBlock_F32(c1 - c0, (const float *)wptr, (const float *)inptr, cptr, ldc, c0 == 0, outLen, CONV_MR, CONV_NR);
}
}
}
@@ -1838,7 +1837,7 @@ static inline void convBlockMR1x12(int np, const float* a, const float* b, float
}
#endif
void convBlockMR1(int np, const float* a, const float* b, float *c, const float bias, bool init_c,
void convBlockMR1_F32(int np, const float* a, const float* b, float *c, const float bias, bool init_c,
const float minval, const float maxval, bool ifMinMaxAct, const int outLen, const int convNR)
{
#if CV_SIMD128
@@ -2088,7 +2087,7 @@ static inline void convBlockNoSIMD(int np, const float* a, const float* b, float
}
}
void convBlock(int np, const float* a, const float* b, float* c, int ldc, bool init_c, const int outLen,
void convBlock_F32(int np, const float* a, const float* b, float* c, int ldc, bool init_c, const int outLen,
const int convMR, const int convNR)
{
// The possible outLen range is [24, 8~1].
modules/dnn/src/layers/cpu_kernels/convolution.hpp
+31 -8
View File
@@ -14,7 +14,7 @@
#define CONV_NR_FP32 28
// The FP16 can only be supported by ARM64 and with FP16 FMA supported.
#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && CV_FP16 // check FP16 FMA.
#if CV_FP16 && CV_TRY_NEON_FP16 // check FP16 FMA.
#define CONV_ARM_FP16 1
#endif
@@ -22,7 +22,6 @@
// Currently, only ARM 64 support FP16.
#define CONV_MR_FP16 8
#define CONV_NR_FP16 24
typedef __fp16 float16_t; // Fix conflict between float16_t in arm_neon.h and float16_t in cvdef.h.
#endif
#elif CV_NEON // 16 registers.
@@ -58,17 +57,15 @@ struct FastConv
int pad_top, pad_bottom, pad_left, pad_right, pad_front, pad_behind;
std::vector<float> weightsBuf; // For generic Conv 2D
float* weightsBufPtr;
std::vector<float> weightsWinoBuf; // For Winograd F(6x6, 3x3).
float* weightsWinoBufPtr;
std::vector<float> biasBuf;
float* getWeights();
float* getWeightsWino();
#if CV_NEON && CV_NEON_AARCH64 && CV_FP16
std::vector<float16_t> weightsBuf_FP16;
float16_t* weightsBufPtr_FP16;
std::vector<float16_t> weightsWinoBuf_FP16;
float16_t* weightsWinoBufPtr_FP16;
#endif
float16_t* getWeightsFP16();
float16_t* getWeightsWinoFP16();
int conv_type;
int conv_dim; // Flag for conv1d, conv2d, or conv3d.
@@ -115,6 +112,32 @@ void runDepthwise(InputArray _input, OutputArray _output, const Ptr<FastConv>& c
int runWinograd63(InputArray _input, InputArray _fusedAddMat, OutputArray _output, const Ptr<FastConv>& conv, int ntasks,
float minval, float maxval, ActivationLayer* activ, bool ifMinMaxAct);
// Work around of NEON, the following functions are only used internally.
namespace opt_NEON {
#if CV_NEON
void convBlock_F32(int np, const float* a, const float* b, float* c, int ldc, bool init_c, int width, const int convMR, const int convNR);
void convBlockMR1_F32(int np, const float* a, const float* b, float* c, const float bias, bool init_c,
const float minval, const float maxval, bool ifMinMaxAct, const int width, const int convNR);
#if CV_NEON_AARCH64
/* Accumulate */
void winofunc_accum_F32(const float* inwptr, const float* wptr, float* outbuf, int Cg, int iblock,
const int winoIblock, const int winoKblock, const int winoAtom, const int winoNatom);
/*Input transform*/
void winofunc_BtXB_8x8_F32(const float* inptr, int inpstep,
float* outptr, int Cg, const int winoIblock, const int winoAtom);
/*Output transform*/
void winofunc_AtXA_8x8_F32(const float* inptr, int inpstep,
float* bpptr, int bpstep, float* outptr, int outstep,
float bias, float minval, float maxval, bool ifMinMaxAct);
#endif // CV_NEON_AARCH64
#endif // CV_NEON
} // namespace opt_NEON.
} // namespace dnn
} // namespace cv
modules/dnn/src/layers/cpu_kernels/fast_gemm.cpp
+220 -80
View File
@@ -20,56 +20,145 @@
namespace cv { namespace dnn {
size_t fastGemmPackBSize(size_t N, size_t K, const FastGemmOpt &opt) {
#if CV_TRY_NEON
if (opt.use_neon) {
return static_cast<size_t>(opt_NEON::fastGemmPackBSize(N, K));
} else
#endif
#if CV_TRY_AVX2
if (opt.use_avx2) {
return static_cast<size_t>(opt_AVX2::fastGemmPackBSize(N, K));
} else
#endif
#if CV_TRY_AVX
if (opt.use_avx) {
return static_cast<size_t>(opt_AVX::fastGemmPackBSize(N, K));
} else
#endif
#if CV_TRY_LASX
if (opt.use_lasx) {
return static_cast<size_t>(opt_LASX::fastGemmPackBSize(N, K));
} else
#endif
{
return static_cast<size_t>(cpu_baseline::fastGemmPackBSize(N, K));
}
}
void fastGemmPackB(const Mat &B, std::vector<float> &packed_B, bool trans, FastGemmOpt &opt) {
CV_CheckEQ(B.dims, 2, "fastGemmPackB: input mat should be two-dimensional");
CV_CheckTypeEQ(B.type(), CV_32F, "fastGemmPackB: only float32 is supported for now");
auto B_shape = shape(B);
int K = B_shape[0], N = B_shape[1], ldb0 = N, ldb1 = 1;
int batch = total(B_shape, 0, B_shape.size() - 2),
K = B_shape[B_shape.size() - 2], N = B_shape.back(), ldb0 = N, ldb1 = 1;
if (trans) {
std::swap(K, N);
std::swap(ldb0, ldb1);
}
const auto *b = B.ptr<const char>();
int esz = B.elemSize();
#if CV_TRY_NEON
if (opt.use_neon) {
int size_packed_B = opt_NEON::fastGemmPackBSize(N, K);
packed_B.resize(size_packed_B);
opt_NEON::fastGemmPackBKernel(B.ptr<const char>(), (char *)packed_B.data(), N, K, ldb0, ldb1, B.elemSize());
packed_B.resize(size_packed_B * batch);
auto *packed_b = (char*)packed_B.data();
for (int i = 0; i < batch; i++) {
opt_NEON::fastGemmPackBKernel(b, packed_b, N, K, ldb0, ldb1, esz);
b += N * K * esz;
packed_b += size_packed_B * esz;
}
} else
#endif
#if CV_TRY_AVX2
if (opt.use_avx2) {
int size_packed_B = opt_AVX2::fastGemmPackBSize(N, K);
packed_B.resize(size_packed_B);
opt_AVX2::fastGemmPackBKernel(B.ptr<const char>(), (char *)packed_B.data(), N, K, ldb0, ldb1, B.elemSize());
packed_B.resize(size_packed_B * batch);
auto *packed_b = (char*)packed_B.data();
for (int i = 0; i < batch; i++) {
opt_AVX2::fastGemmPackBKernel(b, packed_b, N, K, ldb0, ldb1, esz);
b += N * K * esz;
packed_b += size_packed_B * esz;
}
} else
#endif
#if CV_TRY_AVX
if (opt.use_avx) {
int size_packed_B = opt_AVX::fastGemmPackBSize(N, K);
packed_B.resize(size_packed_B);
opt_AVX::fastGemmPackBKernel(B.ptr<const char>(), (char *)packed_B.data(), N, K, ldb0, ldb1, B.elemSize());
packed_B.resize(size_packed_B * batch);
auto *packed_b = (char*)packed_B.data();
for (int i = 0; i < batch; i++) {
opt_AVX::fastGemmPackBKernel(b, packed_b, N, K, ldb0, ldb1, esz);
b += N * K * esz;
packed_b += size_packed_B * esz;
}
} else
#endif
#if CV_TRY_LASX
if (opt.use_lasx) {
int size_packed_B = opt_LASX::fastGemmPackBSize(N, K);
packed_B.resize(size_packed_B);
opt_LASX::fastGemmPackBKernel(B.ptr<const char>(), (char *)packed_B.data(), N, K, ldb0, ldb1, B.elemSize());
packed_B.resize(size_packed_B * batch);
auto *packed_b = (char*)packed_B.data();
for (int i = 0; i < batch; i++) {
opt_LASX::fastGemmPackBKernel(b, packed_b, N, K, ldb0, ldb1, esz);
b += N * K * esz;
packed_b += size_packed_B * esz;
}
} else
#endif
{
int size_packed_B = cpu_baseline::fastGemmPackBSize(N, K);
packed_B.resize(size_packed_B);
cpu_baseline::fastGemmPackBKernel(B.ptr<const char>(), (char *)packed_B.data(), N, K, ldb0, ldb1, B.elemSize());
packed_B.resize(size_packed_B * batch);
auto *packed_b = (char*)packed_B.data();
for (int i = 0; i < batch; i++) {
cpu_baseline::fastGemmPackBKernel(b, packed_b, N, K, ldb0, ldb1, esz);
b += N * K * esz;
packed_b += size_packed_B * esz;
}
}
}
void fastGemmPackB(bool trans, size_t N, size_t K, const float *B, size_t ldb, float *packed_B, const FastGemmOpt &opt) {
size_t ldb0 = ldb, ldb1 = 1;
if (trans) {
std::swap(K, N);
std::swap(ldb0, ldb1);
}
const auto &b = (const char *)B;
auto *packed_b = (char *)packed_B;
#if CV_TRY_NEON
if (opt.use_neon) {
opt_NEON::fastGemmPackBKernel(b, packed_b, N, K, ldb0, ldb1, sizeof(float));
} else
#endif
#if CV_TRY_AVX2
if (opt.use_avx2) {
opt_AVX2::fastGemmPackBKernel(b, packed_b, N, K, ldb0, ldb1, sizeof(float));
} else
#endif
#if CV_TRY_AVX
if (opt.use_avx) {
opt_AVX::fastGemmPackBKernel(b, packed_b, N, K, ldb0, ldb1, sizeof(float));
} else
#endif
#if CV_TRY_LASX
if (opt.use_lasx) {
opt_LASX::fastGemmPackBKernel(b, packed_b, N, K, ldb0, ldb1, sizeof(float));
} else
#endif
{
cpu_baseline::fastGemmPackBKernel(b, packed_b, N, K, ldb0, ldb1, sizeof(float));
}
}
static void fast_gemm_thin(float alpha, float beta, int M, int N, int K,
const char *a_, int lda0, int lda1,
const char *b_, int ldb,
char *c_, int ldc) {
char *c_, int ldc, bool multi_thread) {
const float* a = (const float*)a_;
auto fn = [&](const Range &r) {
@@ -88,16 +177,24 @@ static void fast_gemm_thin(float alpha, float beta, int M, int N, int K,
}
};
int total = M; // outer loops
int cost_per_thread = static_cast<int>(K * N); // inner loops
double nstripes = (size_t)total * cost_per_thread * (1 / 1024.0);
parallel_for_(Range(0, total), fn, nstripes);
if (multi_thread) {
int total = M; // outer loops
int cost_per_thread = static_cast<int>(K * N); // inner loops
double nstripes = (size_t)total * cost_per_thread * (1 / 1024.0);
parallel_for_(Range(0, total), fn, nstripes);
} else {
fn(Range(0, M));
}
}
void fastGemm(bool trans_a, int M, int N, int K,
float alpha, const float *A, int lda,
const float *packed_B, float beta,
float *C, int ldc, FastGemmOpt &opt) {
const char *a = (const char *)A;
const char *packed_b = (const char *)packed_B;
char *c = (char *)C;
int lda0 = lda, lda1 = 1;
if (trans_a) {
std::swap(lda0, lda1);
@@ -105,33 +202,32 @@ void fastGemm(bool trans_a, int M, int N, int K,
#if CV_TRY_NEON
if (opt.use_neon) {
opt_NEON::fastGemmKernel(M, N, K, alpha, (const char *)A, lda0, lda1, (const char *)packed_B, beta, (char *)C, ldc, sizeof(float));
opt_NEON::fastGemmKernel(M, N, K, alpha, a, lda0, lda1, packed_b, beta, c, ldc, sizeof(float), opt.multi_thread);
} else
#endif
#if CV_TRY_AVX2
if (opt.use_avx2) {
opt_AVX2::fastGemmKernel(M, N, K, alpha, (const char *)A, lda0, lda1, (const char *)packed_B, beta, (char *)C, ldc, sizeof(float));
opt_AVX2::fastGemmKernel(M, N, K, alpha, a, lda0, lda1, packed_b, beta, c, ldc, sizeof(float), opt.multi_thread);
} else
#endif
#if CV_TRY_AVX
if (opt.use_avx) {
opt_AVX::fastGemmKernel(M, N, K, alpha, (const char *)A, lda0, lda1, (const char *)packed_B, beta, (char *)C, ldc, sizeof(float));
opt_AVX::fastGemmKernel(M, N, K, alpha, a, lda0, lda1, packed_b, beta, c, ldc, sizeof(float), opt.multi_thread);
} else
#endif
#if CV_TRY_LASX
if (opt.use_lasx) {
opt_LASX::fastGemmKernel(M, N, K, alpha, (const char *)A, lda0, lda1, (const char *)packed_B, beta, (char *)C, ldc, sizeof(float));
opt_LASX::fastGemmKernel(M, N, K, alpha, a, lda0, lda1, packed_b, beta, c, ldc, sizeof(float), opt.multi_thread);
} else
#endif
{
cpu_baseline::fastGemmKernel(M, N, K, alpha, (const char *)A, lda0, lda1, (const char *)packed_B, beta, (char *)C, ldc, sizeof(float));
cpu_baseline::fastGemmKernel(M, N, K, alpha, a, lda0, lda1, packed_b, beta, c, ldc, sizeof(float), opt.multi_thread);
}
}
void fastGemm(bool trans_a, bool trans_b, int ma, int na, int mb, int nb,
float alpha, const float *A, int lda0, int lda1, const float *B, int ldb0, int ldb1,
float beta, float *C, int ldc, FastGemmOpt &opt) {
const char *a = (const char *)A;
const char *b = (const char *)B;
char *c = (char *)C;
@@ -148,36 +244,41 @@ void fastGemm(bool trans_a, bool trans_b, int ma, int na, int mb, int nb,
}
if (!trans_b && ldb1 == 1 && (M <= 4 || (uint64_t)M * N * K <= 10000)) {
return fast_gemm_thin(alpha, beta, M, N, K, a, lda0, lda1, b, ldb0, c, ldc);
return fast_gemm_thin(alpha, beta, M, N, K, a, lda0, lda1, b, ldb0, c, ldc, opt.multi_thread);
}
#if CV_TRY_NEON
if (opt.use_neon) {
opt_NEON::fastGemmKernel(M, N, K, alpha, (const char *)A, lda0, lda1,
(const char *)B, ldb0, ldb1, beta, (char *)C, ldc, sizeof(float));
opt_NEON::fastGemmKernel(M, N, K, alpha, a, lda0, lda1,
b, ldb0, ldb1, beta,
c, ldc, sizeof(float), opt.multi_thread);
} else
#endif
#if CV_TRY_AVX2
if (opt.use_avx2) {
opt_AVX2::fastGemmKernel(M, N, K, alpha, (const char *)A, lda0, lda1,
(const char *)B, ldb0, ldb1, beta, (char *)C, ldc, sizeof(float));
opt_AVX2::fastGemmKernel(M, N, K, alpha, a, lda0, lda1,
b, ldb0, ldb1, beta,
c, ldc, sizeof(float), opt.multi_thread);
} else
#endif
#if CV_TRY_AVX
if (opt.use_avx) {
opt_AVX::fastGemmKernel(M, N, K, alpha, (const char *)A, lda0, lda1,
(const char *)B, ldb0, ldb1, beta, (char *)C, ldc, sizeof(float));
opt_AVX::fastGemmKernel(M, N, K, alpha, a, lda0, lda1,
b, ldb0, ldb1, beta,
c, ldc, sizeof(float), opt.multi_thread);
} else
#endif
#if CV_TRY_LASX
if (opt.use_lasx) {
opt_LASX::fastGemmKernel(M, N, K, alpha, (const char *)A, lda0, lda1,
(const char *)B, ldb0, ldb1, beta, (char *)C, ldc, sizeof(float));
opt_LASX::fastGemmKernel(M, N, K, alpha, a, lda0, lda1,
b, ldb0, ldb1, beta,
c, ldc, sizeof(float), opt.multi_thread);
} else
#endif
{
cpu_baseline::fastGemmKernel(M, N, K, alpha, (const char *)A, lda0, lda1,
(const char *)B, ldb0, ldb1, beta, (char *)C, ldc, sizeof(float));
cpu_baseline::fastGemmKernel(M, N, K, alpha, a, lda0, lda1,
b, ldb0, ldb1, beta,
c, ldc, sizeof(float), opt.multi_thread);
}
}
@@ -209,54 +310,93 @@ void fastGemm(bool trans_a, bool trans_b,
beta, c, ldc, opt);
}
void fastGemmBatched(bool trans_a, bool trans_b,
float alpha, const Mat &A, const Mat &B,
float beta, Mat &C, FastGemmOpt &opt) {
CV_CheckTypeEQ(A.type(), B.type(), "DNN/fastGemmBatched: A and B should have the same type");
CV_CheckTypeEQ(B.type(), C.type(), "DNN/fastGemmBatched: B and C should have the same type");
CV_CheckTypeEQ(A.type(), CV_32F, "DNN/fastGemmBatched: only support float32 for now");
void fastGemmBatch(size_t batch, const size_t *A_offsets, const size_t *B_offsets, const size_t *C_offsets,
int M, int N, int K, float alpha, const float *A, int lda0, int lda1,
const float *B, int ldb0, int ldb1, float beta, float *C, int ldc, FastGemmOpt &opt) {
const char *a = (const char *)A;
const char *b = (const char *)B;
char *c = (char *)C;
const auto shape_a = shape(A);
size_t dims_A = shape_a.size();
CV_CheckGE(dims_A, static_cast<size_t>(2), "DNN/fastGemmBatched: A must be n-dimensional (n >= 2)");
const auto shape_b = shape(B);
CV_CheckEQ(shape_b.size(), static_cast<size_t>(2), "DNN/fastGemmBatched: B must be 2-dimensional");
const auto shape_c = shape(C);
size_t dims_C = shape_c.size();
CV_CheckGE(dims_C, static_cast<size_t>(2), "DNN/fastGemmBatched: C must be n-dimensional (n >= 2)");
if (trans_a) {
int ma = shape_a[dims_A - 2], na = shape_a[dims_A - 1];
int mb = shape_b[0], nb = shape_b[1];
int lda0 = na, lda1 = 1, ldb0 = nb, ldb1 = 1, ldc = shape_c[1];
const float *a = A.ptr<const float>();
const float *b = B.ptr<const float>();
float *c = C.ptr<float>();
int batches = std::accumulate(shape_a.begin(), shape_a.end() - 2, 1, std::multiplies<int>());
int step_a = ma * na, step_c = na * nb;
for (int i = 0; i < batches; i++) {
fastGemm(true, trans_b, ma, na, mb, nb,
alpha, a + i * step_a, lda0, lda1, b, ldb0, ldb1,
beta, c + i * step_c, ldc, opt);
}
} else {
int ma = std::accumulate(shape_a.begin(), shape_a.end() - 1, 1, std::multiplies<int>()),
na = shape_a[dims_A - 1];
int mb = shape_b[0], nb = shape_b[1];
int lda0 = na, lda1 = 1, ldb0 = nb, ldb1 = 1, ldc = shape_c[1];
const float *a = A.ptr<const float>();
const float *b = B.ptr<const float>();
float *c = C.ptr<float>();
fastGemm(false, trans_b, ma, na, mb, nb,
alpha, a, lda0, lda1, b, ldb0, ldb1,
beta, c, ldc, opt);
#if CV_TRY_NEON
if (opt.use_neon) {
opt_NEON::fastGemmBatchKernel(batch, A_offsets, B_offsets, C_offsets, M, N, K, alpha, a, lda0, lda1, b, ldb0, ldb1, beta, c, ldc, sizeof(float));
} else
#endif
#if CV_TRY_AVX2
if (opt.use_avx2) {
opt_AVX2::fastGemmBatchKernel(batch, A_offsets, B_offsets, C_offsets, M, N, K, alpha, a, lda0, lda1, b, ldb0, ldb1, beta, c, ldc, sizeof(float));
} else
#endif
#if CV_TRY_AVX
if (opt.use_avx) {
opt_AVX::fastGemmBatchKernel(batch, A_offsets, B_offsets, C_offsets, M, N, K, alpha, a, lda0, lda1, b, ldb0, ldb1, beta, c, ldc, sizeof(float));
} else
#endif
#if CV_TRY_LASX
if (opt.use_lasx) {
opt_LASX::fastGemmBatchKernel(batch, A_offsets, B_offsets, C_offsets, M, N, K, alpha, a, lda0, lda1, b, ldb0, ldb1, beta, c, ldc, sizeof(float));
} else
#endif
{
cpu_baseline::fastGemmBatchKernel(batch, A_offsets, B_offsets, C_offsets, M, N, K, alpha, a, lda0, lda1, b, ldb0, ldb1, beta, c, ldc, sizeof(float));
}
}
void fastGemmBatch(size_t batch, const size_t *A_offsets, const size_t *packed_B_offsets, const size_t *C_offsets,
int M, int N, int K, float alpha, const float *A, int lda0, int lda1,
const float *packed_B, float beta, float *C, int ldc, FastGemmOpt &opt) {
const char *a = (const char *)A;
const char *b = (const char *)packed_B;
char *c = (char *)C;
#if CV_TRY_NEON
if (opt.use_neon) {
opt_NEON::fastGemmBatchKernel(batch, A_offsets, packed_B_offsets, C_offsets, M, N, K, alpha, a, lda0, lda1, b, beta, c, ldc, sizeof(float));
} else
#endif
#if CV_TRY_AVX2
if (opt.use_avx2) {
opt_AVX2::fastGemmBatchKernel(batch, A_offsets, packed_B_offsets, C_offsets, M, N, K, alpha, a, lda0, lda1, b, beta, c, ldc, sizeof(float));
} else
#endif
#if CV_TRY_AVX
if (opt.use_avx) {
opt_AVX::fastGemmBatchKernel(batch, A_offsets, packed_B_offsets, C_offsets, M, N, K, alpha, a, lda0, lda1, b, beta, c, ldc, sizeof(float));
} else
#endif
#if CV_TRY_LASX
if (opt.use_lasx) {
opt_LASX::fastGemmBatchKernel(batch, A_offsets, packed_B_offsets, C_offsets, M, N, K, alpha, a, lda0, lda1, b, beta, c, ldc, sizeof(float));
} else
#endif
{
cpu_baseline::fastGemmBatchKernel(batch, A_offsets, packed_B_offsets, C_offsets, M, N, K, alpha, a, lda0, lda1, b, beta, c, ldc, sizeof(float));
}
}
void fastGemmBatch(bool trans_a, bool trans_b,
float alpha, const Mat &A, const Mat &B,
float beta, Mat &C, FastGemmOpt &opt) {
CV_CheckTypeEQ(A.type(), B.type(), "DNN/fastGemmBatch: A and B should have the same type");
CV_CheckTypeEQ(B.type(), C.type(), "DNN/fastGemmBatch: B and C should have the same type");
CV_CheckTypeEQ(A.type(), CV_32F, "DNN/fastGemmBatch: only support float32 for now");
const auto shape_a = shape(A);
const auto shape_b = shape(B);
const auto shape_c = shape(C);
CV_CheckGE(shape_a.size(), static_cast<size_t>(2), "DNN/fastGemmBatch: A must be n-dimensional (n >= 2)");
CV_CheckEQ(shape_b.size(), static_cast<size_t>(2), "DNN/fastGemmBatch: B must be n-dimensional (n >= 2)");
const float *a = A.ptr<const float>();
const float *b = B.ptr<const float>();
float *c = C.ptr<float>();
MatMulHelper helper;
helper.compute(trans_a, trans_b, shape_a, shape_b, shape_c);
fastGemmBatch(helper.batch, helper.A_offsets.data(), helper.B_offsets.data(), helper.C_offsets.data(),
helper.M, helper.N, helper.K, alpha, a, helper.lda0, helper.lda1, b, helper.ldb0,
helper.ldb1, beta, c, helper.ldc, opt);
}
}} // cv::dnn
modules/dnn/src/layers/cpu_kernels/fast_gemm.hpp
+120 -4
View File
@@ -22,12 +22,14 @@ struct FastGemmOpt {
bool use_avx2;
bool use_neon;
bool use_lasx;
bool multi_thread;
FastGemmOpt() {
use_avx = false;
use_avx2 = false;
use_neon = false;
use_lasx = false;
multi_thread = false;
}
void init() {
@@ -35,6 +37,7 @@ struct FastGemmOpt {
use_avx2 = checkHardwareSupport(CPU_AVX2);
use_neon = checkHardwareSupport(CPU_NEON);
use_lasx = checkHardwareSupport(CPU_LASX);
multi_thread = true;
}
bool all() {
@@ -42,7 +45,116 @@ struct FastGemmOpt {
}
};
struct MatMulHelper {
std::vector<size_t> A_offsets;
std::vector<size_t> B_offsets;
std::vector<size_t> packed_B_offsets;
std::vector<size_t> C_offsets;
std::vector<size_t> A_rows;
std::vector<size_t> B_rows;
std::vector<size_t> C_rows;
size_t batch;
int lda0, lda1;
int ldb0, ldb1;
int ldc;
int M, N, K;
MatMulHelper() {
A_offsets = {0};
B_offsets = {0};
packed_B_offsets = {0};
C_offsets = {0};
A_rows = {0};
B_rows = {0};
C_rows = {0};
batch = 0;
}
bool empty() const {
return batch == 0;
}
void compute(bool trans_a, bool trans_b, MatShape A_shape, MatShape B_shape, MatShape C_shape) {
auto A_ndims = A_shape.size(), B_ndims = B_shape.size(), C_ndims = C_shape.size();
int ma = A_shape[A_ndims - 2], na = A_shape.back();
int mb = B_shape[B_ndims - 2], nb = B_shape.back();
lda0 = na, lda1 = 1;
ldb0 = nb, ldb1 = 1;
ldc = C_shape.back();
M = trans_a ? na : ma;
N = trans_b ? mb : nb;
K = trans_a ? ma : na;
if (trans_a) {
std::swap(lda0, lda1);
}
if (trans_b) {
std::swap(ldb0, ldb1);
}
// compute offsets
auto batch_ndims = C_ndims - 2;
batch = total(C_shape, 0, batch_ndims);
A_offsets.resize(batch, 0);
B_offsets.resize(batch, 0);
C_offsets.resize(batch, 0);
A_rows.resize(batch, 0);
B_rows.resize(batch, 0);
C_rows.resize(batch, 0);
// build C_offsets
size_t C_step = total(C_shape, C_ndims - 2);
MatShape A_broadcast_shape(C_ndims, 1);
std::memcpy(A_broadcast_shape.data() + (C_ndims - A_ndims), A_shape.data(), A_ndims * sizeof(int));
MatShape B_broadcast_shape(C_shape.size(), 1);
std::memcpy(B_broadcast_shape.data() + (C_ndims - B_ndims), B_shape.data(), B_shape.size() * sizeof(int));
std::vector<size_t> A_steps(C_ndims, 1), B_steps(C_ndims, 1);
for (int i = C_ndims - 2; i >= 0; i--) {
A_steps[i] = A_steps[i + 1] * A_broadcast_shape[i + 1];
B_steps[i] = B_steps[i + 1] * B_broadcast_shape[i + 1];
}
size_t t, idx;
for (size_t i = 0; i < batch; i++) {
C_offsets[i] = i * C_step;
C_rows[i] = i;
size_t A_offset = 0, B_offset = 0;
t = i;
for (int j = batch_ndims - 1; j >= 0; j--) {
idx = t / C_shape[j];
int idx_offset = (int)(t - idx * C_shape[j]);
A_offset += A_broadcast_shape[j] == 1 ? 0 : idx_offset * A_steps[j];
B_offset += B_broadcast_shape[j] == 1 ? 0 : idx_offset * B_steps[j];
t = idx;
}
A_offsets[i] = A_offset;
B_offsets[i] = B_offset;
A_rows[i] = A_offset / (M * K);
B_rows[i] = B_offset / (N * K);
}
}
// only run after compute
void updatePackedBOffsets(size_t packed_B_size) {
size_t packed_B_inner_size = packed_B_size / batch;
packed_B_offsets.resize(B_offsets.size());
for (size_t i = 0; i < packed_B_offsets.size(); i++) {
packed_B_offsets[i] = (B_offsets[i] / (N * K)) * packed_B_inner_size;
}
}
};
size_t fastGemmPackBSize(size_t N, size_t K, const FastGemmOpt &opt);
void fastGemmPackB(const Mat &m, std::vector<float> &packed_B, bool trans, FastGemmOpt &opt);
void fastGemmPackB(bool trans, size_t N, size_t K, const float *B, size_t ldb, float *packed_B, const FastGemmOpt &opt);
void fastGemm(bool trans_a, int M, int N, int K,
float alpha, const float *A, int lda,
@@ -55,10 +167,14 @@ void fastGemm(bool trans_a, bool trans_b,
float alpha, const Mat &A, const Mat &B,
float beta, Mat &C, FastGemmOpt &opt);
// FIXME: B needs to 2d for now. Support nd (n>=2) B in the future.
void fastGemmBatched(bool trans_a, bool trans_b,
float alpha, const Mat &A, const Mat &B,
float beta, Mat &C, FastGemmOpt &opt);
void fastGemmBatch(size_t batch, const size_t *A_offsets, const size_t *B_offsets, const size_t *C_offsets,
int M, int N, int K, float alpha, const float *A, int lda0, int lda1,
const float *B, int ldb0, int ldb1, float beta, float *C, int ldc, FastGemmOpt &opt);
void fastGemmBatch(size_t batch, const size_t *A_offsets, const size_t *B_offsets, const size_t *C_offsets,
int M, int N, int K, float alpha, const float *A, int lda0, int lda1,
const float *packed_B, float beta, float *C, int ldc, FastGemmOpt &opt);
void fastGemmBatch(bool trans_a, bool trans_b, float alpha, const Mat &A,
const Mat &B, float beta, Mat &C, FastGemmOpt &opt);
}} // cv::dnn
modules/dnn/src/layers/cpu_kernels/fast_gemm_kernels.default.hpp
+169 -9
View File
@@ -83,10 +83,17 @@ void fastGemmPackBKernel(const char *B, char *packed_B, int N, int K, int ldb0,
void fastGemmKernel(int M, int N, int K,
float alpha, const char *A, int lda0, int lda1,
const char *B, int ldb0, int ldb1,
float beta, char *C, int ldc, int esz);
float beta, char *C, int ldc, int esz, bool multi_thread);
void fastGemmKernel(int M, int N, int K,
float alpha, const char *A, int lda0, int lda1,
const char *packed_B, float beta, char *C, int ldc, int esz);
const char *packed_B, float beta, char *C, int ldc, int esz, bool multi_thread);
void fastGemmBatchKernel(size_t batch, const size_t *A_offsets, const size_t *B_offsets, const size_t *C_offsets,
int M, int N, int K, float alpha, const char *A, int lda0, int lda1,
const char *B, int ldb0, int ldb1, float beta, char *C, int ldc, int esz);
void fastGemmBatchKernel(size_t batch, const size_t *A_offsets, const size_t *B_offsets, const size_t *C_offsets,
int M, int N, int K, float alpha, const char *A, int lda0, int lda1,
const char *packed_B, float beta, char *C, int ldc, int esz);
FAST_GEMM_IMPLEMENT_PACK(8, _f32, float, float)
FAST_GEMM_IMPLEMENT_PACK(12, _f32, float, float)
@@ -172,7 +179,7 @@ static void fast_gemm_macro_kernel(int m, int n, int k,
void fastGemmKernel(int M, int N, int K,
float alpha, const char *A, int lda0, int lda1,
const char *B, int ldb0, int ldb1,
float beta, char *C, int ldc, int esz) {
float beta, char *C, int ldc, int esz, bool multi_thread) {
int GEMM_MC = FAST_GEMM_F32_MC,
GEMM_NC = FAST_GEMM_F32_NC,
GEMM_MR = FAST_GEMM_F32_MR,
@@ -229,15 +236,18 @@ void fastGemmKernel(int M, int N, int K,
}
};
int total = total_tiles;
int cost_per_thread = static_cast<int>((K / KC) * (MC / GEMM_MR) * (NC / GEMM_NR));
double nstripes = (size_t)total * cost_per_thread * (1 / 1024.0);
parallel_for_(Range(0, total), fn, nstripes);
if (multi_thread) {
int cost_per_thread = static_cast<int>((K / KC) * (MC / GEMM_MR) * (NC / GEMM_NR));
double nstripes = (size_t)total_tiles * cost_per_thread * (1 / 1024.0);
parallel_for_(Range(0, total_tiles), fn, nstripes);
} else {
fn(Range(0, total_tiles));
}
}
void fastGemmKernel(int M, int N, int K,
float alpha, const char *A, int lda0, int lda1,
const char *packed_B, float beta, char *C, int ldc, int esz) {
const char *packed_B, float beta, char *C, int ldc, int esz, bool multi_thread) {
int GEMM_MC = FAST_GEMM_F32_MC,
GEMM_NC = FAST_GEMM_F32_NC,
GEMM_MR = FAST_GEMM_F32_MR,
@@ -294,7 +304,157 @@ void fastGemmKernel(int M, int N, int K,
}
};
int total = total_tiles;
if (multi_thread) {
int cost_per_thread = static_cast<int>((K / KC) * (MC / GEMM_MR) * (NC / GEMM_NR));
double nstripes = (size_t)total_tiles * cost_per_thread * (1 / 1024.0);
parallel_for_(Range(0, total_tiles), fn, nstripes);
} else {
fn(Range(0, total_tiles));
}
}
void fastGemmBatchKernel(size_t batch, const size_t *A_offsets, const size_t *B_offsets, const size_t *C_offsets,
int M, int N, int K, float alpha, const char *A, int lda0, int lda1,
const char *B, int ldb0, int ldb1, float beta, char *C, int ldc, int esz) {
int GEMM_MC = FAST_GEMM_F32_MC,
GEMM_NC = FAST_GEMM_F32_NC,
GEMM_MR = FAST_GEMM_F32_MR,
GEMM_NR = FAST_GEMM_F32_NR;
int MC = (((GEMM_MC < M ? GEMM_MC : M) + GEMM_MR - 1) / GEMM_MR) * GEMM_MR;
int NC = (((GEMM_NC < N ? GEMM_NC : N) + GEMM_NR - 1) / GEMM_NR) * GEMM_NR;
int KC = std::min(FAST_GEMM_F32_PACKED_STRIDE_K, K);
size_t buff_size = KC * (MC + NC) * esz;
bool use_stackbuff = buff_size <= FAST_GEMM_MAX_STACKBUF;
int m_tiles = (M + MC - 1) / MC;
int n_tiles = (N + NC - 1) / NC;
int total_tiles = m_tiles * n_tiles;
auto fn = [&](const Range &r) {
char* packed_a = (char*)(use_stackbuff ? alloca(buff_size) : malloc(buff_size));
char* packed_b = packed_a + KC * MC * esz;
int start = r.start;
int end = r.end;
for (int tile_idx = start; tile_idx < end; tile_idx++) {
const int batch_index = static_cast<int>(tile_idx / total_tiles);
const int m_tiles_index = static_cast<int>((tile_idx - batch_index * total_tiles) / n_tiles);
const int n_tiles_index = static_cast<int>(tile_idx % n_tiles);
int i0 = m_tiles_index * MC;
int j0 = n_tiles_index * NC;
int mc = M - i0 < MC ? M - i0 : MC;
int nc = N - j0 < NC ? N - j0 : NC;
int ldc_block = ldc;
const char *a_block = A + A_offsets[batch_index] * esz;
const char *b_block = B + B_offsets[batch_index] * esz;
char* c_block = C + C_offsets[batch_index] * esz + (i0 * ldc + j0) * esz;
if (beta == 0.f) {
for(int i = 0; i < mc; i++)
memset(c_block + i * ldc_block * esz, 0, nc * esz);
} else if (beta != 1.f) {
for(int i = 0; i < mc; i++) {
float* c_i = (float*)c_block + i * ldc_block;
for(int j = 0; j < nc; j++)
c_i[j] *= beta;
}
}
for(int k0 = 0; k0 < K; k0 += KC)
{
int kc = K - k0 < KC ? K - k0 : KC;
// pack a
fast_gemm_pack8_f32(mc, kc, a_block + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a);
// pack b
fast_gemm_pack12_f32(nc, kc, b_block + (k0 * ldb0 + j0 * ldb1) * esz, ldb1, ldb0, packed_b);
// run kernel
fast_gemm_macro_kernel(mc, nc, kc, packed_a, packed_b, alpha, c_block, ldc_block, esz);
}
}
if (!use_stackbuff) {
free(packed_a);
}
};
int total = batch * total_tiles;
int cost_per_thread = static_cast<int>((K / KC) * (MC / GEMM_MR) * (NC / GEMM_NR));
double nstripes = (size_t)total * cost_per_thread * (1 / 1024.0);
parallel_for_(Range(0, total), fn, nstripes);
}
void fastGemmBatchKernel(size_t batch, const size_t *A_offsets, const size_t *B_offsets, const size_t *C_offsets,
int M, int N, int K, float alpha, const char *A, int lda0, int lda1,
const char *packed_B, float beta, char *C, int ldc, int esz) {
int GEMM_MC = FAST_GEMM_F32_MC,
GEMM_NC = FAST_GEMM_F32_NC,
GEMM_MR = FAST_GEMM_F32_MR,
GEMM_NR = FAST_GEMM_F32_NR;
int MC = (((GEMM_MC < M ? GEMM_MC : M) + GEMM_MR - 1) / GEMM_MR) * GEMM_MR;
int NC = (((GEMM_NC < N ? GEMM_NC : N) + GEMM_NR - 1) / GEMM_NR) * GEMM_NR;
int KC = std::min(FAST_GEMM_F32_PACKED_STRIDE_K, K);
size_t buff_size = KC * MC * esz;
bool use_stackbuff = buff_size <= FAST_GEMM_MAX_STACKBUF;
int m_tiles = (M + MC - 1) / MC;
int n_tiles = (N + NC - 1) / NC;
int total_tiles = m_tiles * n_tiles;
auto fn = [&](const Range &r) {
char* packed_a = (char*)(use_stackbuff ? alloca(buff_size) : malloc(buff_size));
const char *packed_b = packed_B;
int start = r.start;
int end = r.end;
for (int tile_idx = start; tile_idx < end; tile_idx++) {
const int batch_index = static_cast<int>(tile_idx / total_tiles);
const int m_tiles_index = static_cast<int>((tile_idx - batch_index * total_tiles) / n_tiles);
const int n_tiles_index = static_cast<int>(tile_idx % n_tiles);
int i0 = m_tiles_index * MC;
int j0 = n_tiles_index * NC;
int mc = M - i0 < MC ? M - i0 : MC;
int nc = N - j0 < NC ? N - j0 : NC;
int ldc_block = ldc;
const char *a_block = A + A_offsets[batch_index] * esz;
packed_b = packed_B + B_offsets[batch_index] * esz + j0 * K * esz;
char* c_block = C + C_offsets[batch_index] * esz + (i0 * ldc + j0) * esz;
if (beta == 0.f) {
for(int i = 0; i < mc; i++)
memset(c_block + i * ldc_block * esz, 0, nc * esz);
} else if (beta != 1.f) {
for(int i = 0; i < mc; i++) {
float* c_i = (float*)c_block + i * ldc_block;
for(int j = 0; j < nc; j++)
c_i[j] *= beta;
}
}
int _nc = static_cast<int>((nc + GEMM_NR - 1) / GEMM_NR) * GEMM_NR * esz;
for(int k0 = 0; k0 < K; k0 += KC)
{
int kc = K - k0 < KC ? K - k0 : KC;
// pack a
fast_gemm_pack8_f32(mc, kc, a_block + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a);
// run kernel
fast_gemm_macro_kernel(mc, nc, kc, packed_a, packed_b, alpha, c_block, ldc_block, esz);
packed_b += _nc * kc;
}
}
if (!use_stackbuff) {
free(packed_a);
}
};
int total = batch * total_tiles;
int cost_per_thread = static_cast<int>((K / KC) * (MC / GEMM_MR) * (NC / GEMM_NR));
double nstripes = (size_t)total * cost_per_thread * (1 / 1024.0);
parallel_for_(Range(0, total), fn, nstripes);
modules/dnn/src/layers/cpu_kernels/fast_gemm_kernels.simd.hpp
+196 -11
View File
@@ -22,8 +22,8 @@
#define FAST_GEMM_F32_MC 48
#define FAST_GEMM_F32_NC 128
#else // CV_NEON_AARCH64, SIMD128
#define FAST_GEMM_F32_MC 64
#define FAST_GEMM_F32_NC 240
#define FAST_GEMM_F32_MC 144
#define FAST_GEMM_F32_NC 72
#endif
#if CV_AVX
@@ -122,10 +122,17 @@ void fastGemmPackBKernel(const char *B, char *packed_B, int N, int K, int ldb0,
void fastGemmKernel(int M, int N, int K,
float alpha, const char *A, int lda0, int lda1,
const char *B, int ldb0, int ldb1,
float beta, char *C, int ldc, int esz);
float beta, char *C, int ldc, int esz, bool multi_thread);
void fastGemmKernel(int M, int N, int K,
float alpha, const char *A, int lda0, int lda1,
const char *packed_B, float beta, char *C, int ldc, int esz);
const char *packed_B, float beta, char *C, int ldc, int esz, bool multi_thread);
void fastGemmBatchKernel(size_t batch, const size_t *A_offsets, const size_t *B_offsets, const size_t *C_offsets,
int M, int N, int K, float alpha, const char *A, int lda0, int lda1,
const char *B, int ldb0, int ldb1, float beta, char *C, int ldc, int esz);
void fastGemmBatchKernel(size_t batch, const size_t *A_offsets, const size_t *B_offsets, const size_t *C_offsets,
int M, int N, int K, float alpha, const char *A, int lda0, int lda1,
const char *packed_B, float beta, char *C, int ldc, int esz);
#ifndef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
@@ -561,7 +568,7 @@ void fastGemmPackBKernel(const char *B, char *packed_B, int N, int K, int ldb0,
void fastGemmKernel(int M, int N, int K,
float alpha, const char *A, int lda0, int lda1,
const char *B, int ldb0, int ldb1,
float beta, char *C, int ldc, int esz) {
float beta, char *C, int ldc, int esz, bool multi_thread) {
int GEMM_MC = FAST_GEMM_F32_MC,
GEMM_NC = FAST_GEMM_F32_NC,
GEMM_MR = FAST_GEMM_F32_MR,
@@ -639,15 +646,19 @@ void fastGemmKernel(int M, int N, int K,
}
};
int total = total_tiles;
int cost_per_thread = static_cast<int>((K / KC) * (MC / GEMM_MR) * (NC / GEMM_NR));
double nstripes = (size_t)total * cost_per_thread * (1 / 1024.0);
parallel_for_(Range(0, total), fn, nstripes);
if (multi_thread) {
int cost_per_thread = static_cast<int>((K / KC) * (MC / GEMM_MR) * (NC / GEMM_NR));
double nstripes = (size_t)total_tiles * cost_per_thread * (1 / 1024.0);
parallel_for_(Range(0, total_tiles), fn, nstripes);
} else {
fn(Range(0, total_tiles));
}
}
void fastGemmKernel(int M, int N, int K,
float alpha, const char *A, int lda0, int lda1,
const char *packed_B, float beta, char *C, int ldc, int esz) {
const char *packed_B, float beta, char *C, int ldc, int esz, bool multi_thread) {
int GEMM_MC = FAST_GEMM_F32_MC,
GEMM_NC = FAST_GEMM_F32_NC,
GEMM_MR = FAST_GEMM_F32_MR,
@@ -715,7 +726,181 @@ void fastGemmKernel(int M, int N, int K,
}
};
int total = total_tiles;
if (multi_thread) {
int cost_per_thread = static_cast<int>((K / KC) * (MC / GEMM_MR) * (NC / GEMM_NR));
double nstripes = (size_t)total_tiles * cost_per_thread * (1 / 1024.0);
parallel_for_(Range(0, total_tiles), fn, nstripes);
} else {
fn(Range(0, total_tiles));
}
}
void fastGemmBatchKernel(size_t batch, const size_t *A_offsets, const size_t *B_offsets, const size_t *C_offsets,
int M, int N, int K, float alpha, const char *A, int lda0, int lda1,
const char *B, int ldb0, int ldb1, float beta, char *C, int ldc, int esz) {
int GEMM_MC = FAST_GEMM_F32_MC,
GEMM_NC = FAST_GEMM_F32_NC,
GEMM_MR = FAST_GEMM_F32_MR,
GEMM_NR = FAST_GEMM_F32_NR;
int MC = (((GEMM_MC < M ? GEMM_MC : M) + GEMM_MR - 1) / GEMM_MR) * GEMM_MR;
int NC = (((GEMM_NC < N ? GEMM_NC : N) + GEMM_NR - 1) / GEMM_NR) * GEMM_NR;
int KC = std::min(FAST_GEMM_F32_PACKED_STRIDE_K, K);
size_t buff_size = KC * (MC + NC) * esz;
bool use_stackbuff = buff_size <= FAST_GEMM_MAX_STACKBUF;
int m_tiles = (M + MC - 1) / MC;
int n_tiles = (N + NC - 1) / NC;
int total_tiles = m_tiles * n_tiles;
auto fn = [&](const Range &r) {
char* packed_a = (char*)(use_stackbuff ? alloca(buff_size) : malloc(buff_size));
char* packed_b = packed_a + KC * MC * esz;
int start = r.start;
int end = r.end;
for (int tile_idx = start; tile_idx < end; tile_idx++) {
const int batch_index = static_cast<int>(tile_idx / total_tiles);
const int m_tiles_index = static_cast<int>((tile_idx - batch_index * total_tiles) / n_tiles);
const int n_tiles_index = static_cast<int>(tile_idx % n_tiles);
int i0 = m_tiles_index * MC;
int j0 = n_tiles_index * NC;
int mc = M - i0 < MC ? M - i0 : MC;
int nc = N - j0 < NC ? N - j0 : NC;
int ldc_block = ldc;
const char *a_block = A + A_offsets[batch_index] * esz;
const char *b_block = B + B_offsets[batch_index] * esz;
char* c_block = C + C_offsets[batch_index] * esz + (i0 * ldc + j0) * esz;
if (beta == 0.f) {
for(int i = 0; i < mc; i++)
memset(c_block + i * ldc_block * esz, 0, nc * esz);
} else if (beta != 1.f) {
for(int i = 0; i < mc; i++) {
float* c_i = (float*)c_block + i * ldc_block;
for(int j = 0; j < nc; j++)
c_i[j] *= beta;
}
}
for(int k0 = 0; k0 < K; k0 += KC)
{
int kc = K - k0 < KC ? K - k0 : KC;
// pack a
#if CV_NEON && CV_NEON_AARCH64
fast_gemm_pack8_f32(mc, kc, a_block + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a);
#elif CV_AVX
fast_gemm_pack12_f32(mc, kc, a_block + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a);
#elif CV_LASX
fast_gemm_pack12_f32(mc, kc, a_block + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a);
#elif CV_SIMD128
fast_gemm_pack8_f32(mc, kc, a_block + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a);
#endif
// pack b
#if CV_NEON && CV_NEON_AARCH64
fast_gemm_pack12_f32(nc, kc, b_block + (k0 * ldb0 + j0 * ldb1) * esz, ldb1, ldb0, packed_b);
#elif CV_AVX
fast_gemm_pack8_f32(nc, kc, b_block + (k0 * ldb0 + j0 * ldb1) * esz, ldb1, ldb0, packed_b);
#elif CV_LASX
fast_gemm_pack16_f32(nc, kc, b_block + (k0 * ldb0 + j0 * ldb1) * esz, ldb1, ldb0, packed_b);
#elif CV_SIMD128
fast_gemm_pack12_f32(nc, kc, b_block + (k0 * ldb0 + j0 * ldb1) * esz, ldb1, ldb0, packed_b);
#endif
// run kernel
fast_gemm_macro_kernel(mc, nc, kc, packed_a, packed_b, alpha, c_block, ldc_block, esz);
}
}
if (!use_stackbuff) {
free(packed_a);
}
};
int total = batch * total_tiles;
int cost_per_thread = static_cast<int>((K / KC) * (MC / GEMM_MR) * (NC / GEMM_NR));
double nstripes = (size_t)total * cost_per_thread * (1 / 1024.0);
parallel_for_(Range(0, total), fn, nstripes);
}
void fastGemmBatchKernel(size_t batch, const size_t *A_offsets, const size_t *B_offsets, const size_t *C_offsets,
int M, int N, int K, float alpha, const char *A, int lda0, int lda1,
const char *packed_B, float beta, char *C, int ldc, int esz) {
int GEMM_MC = FAST_GEMM_F32_MC,
GEMM_NC = FAST_GEMM_F32_NC,
GEMM_MR = FAST_GEMM_F32_MR,
GEMM_NR = FAST_GEMM_F32_NR;
int MC = (((GEMM_MC < M ? GEMM_MC : M) + GEMM_MR - 1) / GEMM_MR) * GEMM_MR;
int NC = (((GEMM_NC < N ? GEMM_NC : N) + GEMM_NR - 1) / GEMM_NR) * GEMM_NR;
int KC = std::min(FAST_GEMM_F32_PACKED_STRIDE_K, K);
size_t buff_size = KC * MC * esz;
bool use_stackbuff = buff_size <= FAST_GEMM_MAX_STACKBUF;
int m_tiles = (M + MC - 1) / MC;
int n_tiles = (N + NC - 1) / NC;
int total_tiles = m_tiles * n_tiles;
auto fn = [&](const Range &r) {
char* packed_a = (char*)(use_stackbuff ? alloca(buff_size) : malloc(buff_size));
const char *packed_b = packed_B;
int start = r.start;
int end = r.end;
for (int tile_idx = start; tile_idx < end; tile_idx++) {
const int batch_index = static_cast<int>(tile_idx / total_tiles);
const int m_tiles_index = static_cast<int>((tile_idx - batch_index * total_tiles) / n_tiles);
const int n_tiles_index = static_cast<int>(tile_idx % n_tiles);
int i0 = m_tiles_index * MC;
int j0 = n_tiles_index * NC;
int mc = M - i0 < MC ? M - i0 : MC;
int nc = N - j0 < NC ? N - j0 : NC;
int ldc_block = ldc;
const char *a_block = A + A_offsets[batch_index] * esz;
packed_b = packed_B + B_offsets[batch_index] * esz + j0 * K * esz;
char* c_block = C + C_offsets[batch_index] * esz + (i0 * ldc + j0) * esz;
if (beta == 0.f) {
for(int i = 0; i < mc; i++)
memset(c_block + i * ldc_block * esz, 0, nc * esz);
} else if (beta != 1.f) {
for(int i = 0; i < mc; i++) {
float* c_i = (float*)c_block + i * ldc_block;
for(int j = 0; j < nc; j++)
c_i[j] *= beta;
}
}
int _nc = static_cast<int>((nc + GEMM_NR - 1) / GEMM_NR) * GEMM_NR * esz;
for(int k0 = 0; k0 < K; k0 += KC)
{
int kc = K - k0 < KC ? K - k0 : KC;
// pack a
#if CV_NEON && CV_NEON_AARCH64
fast_gemm_pack8_f32(mc, kc, a_block + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a);
#elif CV_AVX
fast_gemm_pack12_f32(mc, kc, a_block + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a);
#elif CV_LASX
fast_gemm_pack12_f32(mc, kc, a_block + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a);
#elif CV_SIMD128
fast_gemm_pack8_f32(mc, kc, a_block + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a);
#endif
// run kernel
fast_gemm_macro_kernel(mc, nc, kc, packed_a, packed_b, alpha, c_block, ldc_block, esz);
packed_b += _nc * kc;
}
}
if (!use_stackbuff) {
free(packed_a);
}
};
int total = batch * total_tiles;
int cost_per_thread = static_cast<int>((K / KC) * (MC / GEMM_MR) * (NC / GEMM_NR));
double nstripes = (size_t)total * cost_per_thread * (1 / 1024.0);
parallel_for_(Range(0, total), fn, nstripes);
modules/dnn/src/layers/cpu_kernels/fast_norm.cpp
+4 -3
View File
@@ -118,10 +118,11 @@ void fastNorm(const Mat &input, const Mat &scale, const Mat &bias, Mat &output,
void fastNormChannel(const Mat &input, const Mat &scale, const Mat &bias, Mat &output, float epsilon) {
const auto input_shape = shape(input);
size_t N = input_shape[0], C = input_shape[1];
CV_CheckEQ(scale.total(), bias.total(), "fastNormChannel: scale and bias should have the same shape");
CV_CheckEQ(scale.total(), C, "fastNormChannel: scale should be a 1d tensor and match the channel of input");
CV_CheckGE(input.dims, 3, "fastNormChannel: input dimension >= 3");
size_t N = input_shape[0], C = input_shape[1];
size_t loops = N * C,
norm_size = static_cast<size_t>(total(input_shape, 2));
float inv_norm_size = 1.0 / norm_size;
@@ -147,9 +148,9 @@ void fastNormChannel(const Mat &input, const Mat &scale, const Mat &bias, Mat &o
float inv_stdev = 1.f / mean_square;
size_t c = i % C;
float s = scale_data[c], b = bias_data[c];
float s = scale_data[c] * inv_stdev, b = bias_data[c];
for (size_t j = 0; j < norm_size; j++) {
y[j] = s * (x[j] - mean) * inv_stdev + b;
y[j] = s * (x[j] - mean) + b;
}
}
};
modules/dnn/src/layers/cpu_kernels/softmax.cpp
+157
View File
@@ -0,0 +1,157 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// This file is modified from the ficus (https://github.com/vpisarev/ficus/blob/master/lib/NN/OpNN.fx).
// Here is the original license:
/*
This file is a part of ficus language project.
See ficus/LICENSE for the licensing terms
*/
#include "../../precomp.hpp"
#include "softmax.hpp"
namespace cv { namespace dnn {
void softmax(Mat &dst, const Mat &src, int axis, int axisBias, int axisStep){
CV_Assert(src.type() == CV_32F);
CV_Assert(src.isContinuous() && dst.isContinuous());
CV_Assert(src.size == dst.size);
axis = normalize_axis(axis, src.dims);
size_t outerSize = src.total(0, axis),
innerSize = src.total(axis + 1);
const float *srcPtr = src.ptr<float>();
float *dstPtr = dst.ptr<float>();
size_t outerStep = src.total(axis);
size_t cnStep = src.total(axis + 1);
// multi-threads
size_t totalTasks = outerSize * innerSize;
double nstripes = (double) totalTasks / 1024.0;
// make the channel axis to be multiple of 8
size_t channelAxis = (axisStep + 7) & -8;
#if (CV_SIMD || CV_SIMD_SCALABLE)
const int nlanes = VTraits<v_float32>::vlanes();
// the number of redundant dimension
size_t redundantDim = nlanes - axisStep % nlanes;
#endif
parallel_for_(Range(0, (int) totalTasks), [&](const Range &range) {
AutoBuffer<float> axisBuf_(channelAxis);
float *axisBuf = axisBuf_.data();
for (size_t i = range.start; i < range.end; i++) {
size_t outerDim = i / innerSize;
size_t innerDim = i % innerSize;
size_t srcOffset = outerDim * outerStep + innerDim;
// copy data from src to buf along axis, since the data may not be continuous
for (size_t cnDim = 0; cnDim < axisStep; cnDim++)
axisBuf[cnDim] = srcPtr[srcOffset + (cnDim + axisBias) * cnStep];
float s = 0.f;
#if (CV_SIMD || CV_SIMD_SCALABLE)
// make the value of the redundant dimension to be -FLT_MAX
if (redundantDim != nlanes) {
for (size_t j = axisStep; j < axisStep + redundantDim; j++)
axisBuf[j] = -FLT_MAX;
}
// calculate the max value along the axis
v_float32 vmax = vx_load(axisBuf);
for (size_t cnDim = nlanes; cnDim < axisStep; cnDim += nlanes) {
v_float32 val = vx_load(axisBuf + cnDim);
vmax = v_max(vmax, val);
}
float maxVal = v_reduce_max(vmax);
// calculate the exp value along the axis
v_float32 vs = vx_setzero_f32();
vmax = vx_setall_f32(maxVal);
// initialize vexp constant
v_float32 _vexp_lo = vx_setall_f32(-88.3762626647949f);
v_float32 _vexp_hi = vx_setall_f32(88.3762626647949f);
v_float32 _vexp_half = vx_setall_f32(0.5f);
v_float32 _vexp_one = vx_setall_f32(1.f);
v_float32 _vexp_LOG2EF = vx_setall_f32(1.44269504088896341f);
v_float32 _vexp_C1 = vx_setall_f32(-0.693359375f);
v_float32 _vexp_C2 = vx_setall_f32(2.12194440e-4f);
v_float32 _vexp_p0 = vx_setall_f32(1.9875691500E-4f);
v_float32 _vexp_p1 = vx_setall_f32(1.3981999507E-3f);
v_float32 _vexp_p2 = vx_setall_f32(8.3334519073E-3f);
v_float32 _vexp_p3 = vx_setall_f32(4.1665795894E-2f);
v_float32 _vexp_p4 = vx_setall_f32(1.6666665459E-1f);
v_float32 _vexp_p5 = vx_setall_f32(5.0000001201E-1f);
// initialize temp vectors for vexp
v_float32 val, _vexp_, _vexp_x, _vexp_y, _vexp_z;
v_int32 _vexp_mm;
// calculate and sum all data along axis
for (size_t cnDim = 0; cnDim < axisStep; cnDim += nlanes) {
val = vx_load(axisBuf + cnDim);
val = v_sub(val, vmax);
// compute vexp of val
_vexp_x = v_min(val, _vexp_hi);
_vexp_x = v_max(_vexp_x, _vexp_lo);
_vexp_ = v_fma(_vexp_x, _vexp_LOG2EF, _vexp_half);
_vexp_mm = v_floor(_vexp_);
_vexp_ = v_cvt_f32(_vexp_mm);
_vexp_mm = v_add(_vexp_mm, vx_setall_s32(0x7f));
_vexp_mm = v_shl(_vexp_mm, 23);
_vexp_x = v_fma(_vexp_, _vexp_C1, _vexp_x);
_vexp_x = v_fma(_vexp_, _vexp_C2, _vexp_x);
_vexp_z = v_mul(_vexp_x, _vexp_x);
_vexp_y = v_fma(_vexp_x, _vexp_p0, _vexp_p1);
_vexp_y = v_fma(_vexp_y, _vexp_x, _vexp_p2);
_vexp_y = v_fma(_vexp_y, _vexp_x, _vexp_p3);
_vexp_y = v_fma(_vexp_y, _vexp_x, _vexp_p4);
_vexp_y = v_fma(_vexp_y, _vexp_x, _vexp_p5);
_vexp_y = v_fma(_vexp_y, _vexp_z, _vexp_x);
_vexp_y = v_add(_vexp_y, _vexp_one);
val = v_mul(_vexp_y, v_reinterpret_as_f32(_vexp_mm));
vs = v_add(vs, val);
v_store(axisBuf + cnDim, val);
}
s = v_reduce_sum(vs);
// subtract the value of the redundant dimension
if (redundantDim != nlanes) {
float _val[VTraits<v_float32>::max_nlanes];
v_store(_val, val);
for (size_t j = nlanes - redundantDim; j < nlanes; j++)
s -= _val[j];
}
#else
float maxVal = axisBuf[0];
for (size_t cnDim = 1; cnDim < axisStep; cnDim++) {
maxVal = std::max(maxVal, axisBuf[cnDim]);
}
for (size_t j = 0; j < axisStep; j++) {
axisBuf[j] = expf(axisBuf[j] - maxVal);
s += axisBuf[j];
}
#endif
s = 1.f / s;
// copy back the result to src
for (size_t cnDim = 0; cnDim < axisStep; cnDim++)
dstPtr[srcOffset + (cnDim + axisBias) * cnStep] = axisBuf[cnDim] * s;
}
}, nstripes);
}
void softmax(Mat &dst, const Mat &src, int axis) {
softmax(dst, src, axis, 0, src.size[axis]);
}
void logSoftmax(Mat &dst, const Mat &src, int axis) {
softmax(dst, src, axis);
log(dst, dst);
}
}} // cv::dnn
modules/dnn/src/layers/cpu_kernels/softmax.hpp
+28
View File
@@ -0,0 +1,28 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// This file is modified from the ficus (https://github.com/vpisarev/ficus/blob/master/lib/NN/OpNN.fx).
// Here is the original license:
/*
This file is a part of ficus language project.
See ficus/LICENSE for the licensing terms
*/
#ifndef OPENCV_DNN_SOFTMAX_HPP
#define OPENCV_DNN_SOFTMAX_HPP
#include "opencv2/core/hal/intrin.hpp"
#include <opencv2/dnn/shape_utils.hpp>
namespace cv { namespace dnn {
void softmax(Mat &dst, const Mat &src, int axis, int axisBias, int axisStep);
void softmax(Mat &dst, const Mat &src, int axis);
void logSoftmax(Mat &dst, const Mat &src, int axis);
}} // cv::dnn
#endif // OPENCV_DNN_SOFTMAX_HPP
modules/dnn/src/layers/einsum_layer.cpp
+116 -106
View File
@@ -6,6 +6,7 @@
#include <opencv2/dnn/shape_utils.hpp>
#include "../precomp.hpp"
#include "layers_common.hpp"
#include "cpu_kernels/fast_gemm.hpp"
namespace cv
{
@@ -32,111 +33,6 @@ static bool IsTransposeReshapeForEinsum(const std::vector<size_t>& perm,
return true;
}
static Mat batchwiseMatMul(
const Mat& input1,
const MatShape& input1ShapeOverride,
const Mat& input2,
const MatShape& input2ShapeOverride)
{
// Sanity checks before the actual MatMul
CV_CheckType(input1.type(), input2.type(), "Data types of the inputs must match for MatMul");
CV_CheckEQ(input1ShapeOverride.size(), (size_t) 3, "Only 1 batch dimension is allowed for MatMul");
CV_CheckEQ(input2ShapeOverride.size(), (size_t) 3, "Only 1 batch dimension is allowed for MatMul");
CV_CheckEQ((size_t) input1ShapeOverride[0], (size_t) input2ShapeOverride[0], "Batch dimension should match for MatMul;");
CV_CheckEQ((size_t) input1ShapeOverride[2], (size_t) input2ShapeOverride[1], "Incompatible matrix dimensions for matMul");
size_t batches = input1ShapeOverride[0];
size_t M = input1ShapeOverride[1];
size_t K = input1ShapeOverride[2];
size_t N = input2ShapeOverride[2];
std::vector<Mat> output;
if (batches > 1)
{
Mat reshapedInput1 = input1;
Mat reshapedInput2 = input2;
// input1 should of size MxK
// check if input1 needs reshape, if need reshape
if (input1.size[0] != M || input1.size[1] != K)
{
int shape[] = {static_cast<int>(batches), static_cast<int>(M), static_cast<int>(K)};
reshapedInput1 = input1.reshape(1, 3, shape);
}
// input2 should be of size KxN
// check if input2 needs reshape, if needs reshape
if (input2.size[0] != K || input2.size[1] != N)
{
int shape[] = {static_cast<int>(batches), static_cast<int>(K), static_cast<int>(N)};
reshapedInput2 = input2.reshape(1, 3, shape);
}
for (size_t i=0; i < batches; i++)
{
std::vector<Range> ranges1 = {cv::Range(i, i+1)};
for (int j = 1; j < reshapedInput1.dims; j++)
ranges1.emplace_back(cv::Range::all());
Mat part1 = reshapedInput1(ranges1);
int shape[] = {static_cast<int>(M), static_cast<int>(K)};
part1 = part1.reshape(1, sizeof(shape)/sizeof(shape[0]), shape);
std::vector<Range> ranges2 = {cv::Range(i, i+1)};
for (int j = 1; j < reshapedInput2.dims; j++)
ranges2.emplace_back(cv::Range::all());
Mat part2 = reshapedInput2(ranges2);
int shape2[] = {static_cast<int>(K), static_cast<int>(N)};
part2 = part2.reshape(1, sizeof(shape2)/sizeof(shape2[0]), shape2);
Mat tmp_output;
cv::gemm(part1, part2, 1.0, cv::Mat(), 1.0, tmp_output);
int newShape[] = {1, static_cast<int>(M), static_cast<int>(N)};
tmp_output = tmp_output.reshape(1, sizeof(newShape)/sizeof(newShape[0]), newShape);
output.emplace_back(tmp_output);
}
} else {
Mat reshapedInput1 = input1;
Mat reshapedInput2 = input2;
// input1 should of size MxK
// check if input1 needs reshape, if need reshape
if (input1.dims > 2 || input1.size[0] != M || (input1.dims > 1 && input1.size[1] != K) || input1.dims == 1)
{
int shape[] = {static_cast<int>(M), static_cast<int>(K)};
reshapedInput1 = input1.reshape(1, 2, shape);
}
// input2 should be of size KxN
// check if input2 needs reshape, if needs reshape
if (input2.dims > 2 || input2.size[0] != K || (input2.dims > 1 && input2.size[1] != N) || input2.dims == 1)
{
int shape2[] = {static_cast<int>(K), static_cast<int>(N)};
reshapedInput2 = input2.reshape(1, 2, shape2);
}
Mat tmp_output;
cv::gemm(reshapedInput1, reshapedInput2, 1.0, cv::Mat(), 1.0, tmp_output);
int newShape[] = {1, static_cast<int>(M), static_cast<int>(N)};
tmp_output = tmp_output.reshape(1, sizeof(newShape)/sizeof(newShape[0]), newShape);
output.emplace_back(tmp_output);
}
int outputDim[] = {static_cast<int>(output.size()), static_cast<int>(M), static_cast<int>(N)};
Mat output_buffer = Mat::zeros(3, outputDim, CV_32F);
for (size_t i = 0; i < output.size(); i++) {
Mat output_slice = output_buffer.row(i);
output[i].copyTo(output_slice);
}
return output_buffer;
};
static Mat Transpose(
const Mat& input,
@@ -452,6 +348,8 @@ public:
// The number of dimensions that are encompassed by an "ellipsis" - "...".
size_t numOfEllipsisDims = 0;
// Backend for fastgemm
FastGemmOpt opt;
void parseEquation(String equation);
void processEquation(const std::vector<MatShape>& inputs);
@@ -469,7 +367,12 @@ public:
const MatShape& reduceDims,
bool isFinalPair
);
Mat batchwiseMatMul(
const Mat& input1,
const MatShape& input1ShapeOverride,
const Mat& input2,
const MatShape& input2ShapeOverride
);
// constructor
LayerEinsumImpl(const LayerParams& params)
@@ -491,6 +394,7 @@ public:
einsumInpShapes.emplace_back(shape);
}
opt.init();
// Maintains a mapping between input indices and their corresponding subscript labels for each input
inputSubscriptIndices.reserve(numInputs);
@@ -1389,6 +1293,112 @@ Mat LayerEinsumImpl::pairwiseOperandProcess(
return output;
};
Mat LayerEinsumImpl::batchwiseMatMul(
const Mat& input1,
const MatShape& input1ShapeOverride,
const Mat& input2,
const MatShape& input2ShapeOverride)
{
// Sanity checks before the actual MatMul
CV_CheckType(input1.type(), input2.type(), "Data types of the inputs must match for MatMul");
CV_CheckEQ(input1ShapeOverride.size(), (size_t) 3, "Only 1 batch dimension is allowed for MatMul");
CV_CheckEQ(input2ShapeOverride.size(), (size_t) 3, "Only 1 batch dimension is allowed for MatMul");
CV_CheckEQ((size_t) input1ShapeOverride[0], (size_t) input2ShapeOverride[0], "Batch dimension should match for MatMul;");
CV_CheckEQ((size_t) input1ShapeOverride[2], (size_t) input2ShapeOverride[1], "Incompatible matrix dimensions for matMul");
int batches = input1ShapeOverride[0];
int M = input1ShapeOverride[1];
int K = input1ShapeOverride[2];
int N = input2ShapeOverride[2];
std::vector<Mat> output;
if (batches > 1)
{
Mat reshapedInput1 = input1;
Mat reshapedInput2 = input2;
// input1 should of size MxK
// check if input1 needs reshape, if need reshape
if (input1.size[0] != M || input1.size[1] != K)
{
int shape[] = {batches, M, K};
reshapedInput1 = input1.reshape(1, 3, shape);
}
// input2 should be of size KxN
// check if input2 needs reshape, if needs reshape
if (input2.size[0] != K || input2.size[1] != N)
{
int shape[] = {batches, K, N};
reshapedInput2 = input2.reshape(1, 3, shape);
}
for (size_t i=0; i < batches; i++)
{
std::vector<Range> ranges1 = {cv::Range(i, i+1)};
for (int j = 1; j < reshapedInput1.dims; j++)
ranges1.emplace_back(cv::Range::all());
Mat part1 = reshapedInput1(ranges1);
int shape[] = {M, K};
part1 = part1.reshape(1, sizeof(shape)/sizeof(shape[0]), shape);
std::vector<Range> ranges2 = {cv::Range(i, i+1)};
for (int j = 1; j < reshapedInput2.dims; j++)
ranges2.emplace_back(cv::Range::all());
Mat part2 = reshapedInput2(ranges2);
int shape2[] = {K, N};
part2 = part2.reshape(1, sizeof(shape2)/sizeof(shape2[0]), shape2);
Mat tmp_output(M, N, part1.type());
fastGemm(false, false, 1.0, part1, part2, 0.0, tmp_output, opt);
int newShape[] = {1, M, N};
tmp_output = tmp_output.reshape(1, sizeof(newShape)/sizeof(newShape[0]), newShape);
output.emplace_back(tmp_output);
}
} else {
Mat reshapedInput1 = input1;
Mat reshapedInput2 = input2;
// input1 should of size MxK
// check if input1 needs reshape, if need reshape
if (input1.dims > 2 || input1.size[0] != M || input1.size[1] != K)
{
int shape[] = {M, K};
reshapedInput1 = input1.reshape(1, 2, shape);
}
// input2 should be of size KxN
// check if input2 needs reshape, if needs reshape
if (input2.dims > 2 || input2.size[0] != K || input2.size[1] != N)
{
int shape2[] = {K, N};
reshapedInput2 = input2.reshape(1, 2, shape2);
}
Mat tmp_output(M, N, reshapedInput1.type());
fastGemm(false, false, 1.0, reshapedInput1, reshapedInput2, 0.0, tmp_output, opt);
int newShape[] = {1, M, N};
tmp_output = tmp_output.reshape(1, sizeof(newShape)/sizeof(newShape[0]), newShape);
output.emplace_back(tmp_output);
}
int outputDim[] = {static_cast<int>(output.size()), M, N};
Mat output_buffer = Mat::zeros(3, outputDim, CV_32F);
for (size_t i = 0; i < output.size(); i++) {
Mat output_slice = output_buffer.row(i);
output[i].copyTo(output_slice);
}
return output_buffer;
};
Ptr<EinsumLayer> EinsumLayer::create(const LayerParams& params)
{
return makePtr<LayerEinsumImpl>(params);
modules/dnn/src/layers/elementwise_layers.cpp
+24 -1
View File
@@ -1712,7 +1712,9 @@ struct HardSwishFunctor : public BaseDefaultFunctor<HardSwishFunctor>
bool supportBackend(int backendId, int)
{
return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA;
return backendId == DNN_BACKEND_OPENCV ||
backendId == DNN_BACKEND_CUDA ||
backendId == DNN_BACKEND_CANN;
}
inline float calculate(float x) const
@@ -1727,6 +1729,27 @@ struct HardSwishFunctor : public BaseDefaultFunctor<HardSwishFunctor>
}
#endif
#ifdef HAVE_CANN
Ptr<BackendNode> initCannOp(const std::string& name,
const std::vector<Ptr<BackendWrapper> > &inputs,
const std::vector<Ptr<BackendNode> >& nodes)
{
auto x = inputs[0].dynamicCast<CannBackendWrapper>();
auto op = std::make_shared<ge::op::HardSwish>(name);
auto op_x = nodes[0].dynamicCast<CannBackendNode>()->getOp();
op->set_input_x_by_name(*op_x, x->name.c_str());
auto x_desc = x->getTensorDesc();
op->update_input_desc_x(*x_desc);
auto output_desc = std::make_shared<ge::TensorDesc>(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT);
op->update_output_desc_y(*output_desc);
return Ptr<BackendNode>(new CannBackendNode(op));
}
#endif
int64 getFLOPSPerElement() const { return 1; }
};
modules/dnn/src/layers/gemm_layer.cpp
+2 -1
View File
@@ -211,7 +211,7 @@ public:
CV_CheckGT(packed_B.size(), static_cast<size_t>(0), "DNN/Gemm: constant B is not pre-packed");
fastGemm(trans_a, M, N, K, alpha, A.ptr<const float>(), na, packed_B.data(), 1.f, Y.ptr<float>(), N, opt);
} else {
fastGemmBatched(trans_a, trans_b, alpha, A, inputs[1], 1.f, Y, opt);
fastGemmBatch(trans_a, trans_b, alpha, A, inputs[1], 1.f, Y, opt);
}
}
@@ -274,6 +274,7 @@ public:
op->update_input_desc_bias(*(op_const_C->getTensorDesc()));
// set outputs
auto output_desc = std::make_shared<ge::TensorDesc>(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT);
op->update_output_desc_y(*output_desc);
return Ptr<BackendNode>(new CannBackendNode(op));
}
modules/dnn/src/layers/instance_norm_layer.cpp
+280
View File
@@ -0,0 +1,280 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#include "../precomp.hpp"
#include <opencv2/dnn/shape_utils.hpp>
#include "./cpu_kernels/fast_norm.hpp"
// CANN backend
#include "../op_cann.hpp"
// OpenVINO backend
#include "../op_inf_engine.hpp"
#include "../ie_ngraph.hpp"
// CUDA backend
#include "../op_cuda.hpp"
#ifdef HAVE_CUDA
#include "../cuda4dnn/primitives/instance_norm.hpp"
using namespace cv::dnn::cuda4dnn;
#endif
// OpenCL backend
#ifdef HAVE_OPENCL
#include "../ocl4dnn/include/math_functions.hpp"
#include "opencl_kernels_dnn.hpp"
#endif
namespace cv { namespace dnn {
// https://github.com/onnx/onnx/blob/main/docs/Operators.md#InstanceNormalization
class InstanceNormLayerImpl CV_FINAL : public InstanceNormLayer {
public:
InstanceNormLayerImpl(const LayerParams &params) {
setParamsFrom(params);
epsilon = params.get<float>("epsilon", 1e-5);
}
virtual bool supportBackend(int backendId) CV_OVERRIDE {
#ifdef HAVE_INF_ENGINE
if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)
return true;
#endif
return backendId == DNN_BACKEND_OPENCV ||
backendId == DNN_BACKEND_CUDA;
// backendId == DNN_BACKEND_CANN; // not supported due to 1d mat shape issue
}
bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int requiredOutputs,
std::vector<MatShape> &outputs,
std::vector<MatShape> &internals) const CV_OVERRIDE {
const auto &input = inputs[0];
const auto &scale = inputs[1];
const auto &bias = inputs[2];
CV_CheckGE(input.size(), static_cast<size_t>(3), "DNN/InstanceNorm: input dimension >= 3 is required");
int C = input[1];
int scale_dim = std::accumulate(scale.begin(), scale.end(), 1, std::multiplies<int>());
CV_CheckEQ(scale_dim, C, "DNN/InstanceNorm: scale must be a 1d tensor and match the channel of input");
int bias_dim = std::accumulate(bias.begin(), bias.end(), 1, std::multiplies<int>());
CV_CheckEQ(bias_dim, C, "DNN/InstanceNorm: bias must be a 1d tensor and match the channel of input");
outputs.assign(1, inputs[0]);
return false;
}
void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE {
CV_TRACE_FUNCTION();
CV_TRACE_ARG_VALUE(name, "name", name.c_str());
CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget),
forward_ocl(inputs_arr, outputs_arr, internals_arr))
if (inputs_arr.depth() == CV_16S)
{
forward_fallback(inputs_arr, outputs_arr, internals_arr);
return;
}
std::vector<Mat> inputs, outputs;
inputs_arr.getMatVector(inputs);
outputs_arr.getMatVector(outputs);
const auto &input = inputs[0];
const auto &scale = inputs[1];
const auto &bias = inputs[2];
fastNormChannel(input, scale, bias, outputs[0], epsilon);
}
#ifdef HAVE_OPENCL
bool forward_ocl(InputArrayOfArrays inputs_, OutputArrayOfArrays outputs_, OutputArrayOfArrays internals_) {
std::vector<UMat> inputs;
std::vector<UMat> outputs;
inputs_.getUMatVector(inputs);
outputs_.getUMatVector(outputs);
const auto &input = inputs[0], &scale = inputs[1], &bias = inputs[2];
auto &output = outputs[0];
const auto input_shape = shape(input);
size_t N = input_shape[0], C = input_shape[1],
loops = N * C, norm_size = static_cast<size_t>(total(input_shape, 2));
float inv_norm_size = 1.f / norm_size;
// no fp16 support
if (input.depth() == CV_16S) {
return false;
}
String base_opts = format(" -DT=float -DT4=float4 -Dconvert_T=convert_float4");
// Calculate mean
UMat one = UMat::ones(norm_size, 1, CV_32F);
UMat mean = UMat(loops, 1, CV_32F);
UMat mean_square = UMat(loops, 1, CV_32F);
UMat tmp = UMat(loops, norm_size, CV_32F);
bool ret = ocl4dnn::ocl4dnnGEMV<float>(ocl4dnn::CblasNoTrans, loops, norm_size, inv_norm_size,
input, 0, one, 0, 0.f, mean, 0);
if (!ret) {
return false;
}
// Calculate mean_square
int num_vector = (norm_size % 8 == 0) ? 8 : ((norm_size % 4 == 0) ? 4 : 1);
size_t global[] = {loops, static_cast<size_t>(norm_size / num_vector)};
String build_opt = format(" -DNUM=%d", num_vector) + base_opts;
String mean_square_kernel_name = format("calc_mean%d", num_vector);
ocl::Kernel mean_square_kernel(mean_square_kernel_name.c_str(), ocl::dnn::mvn_oclsrc, build_opt + " -DKERNEL_MEAN");
if (mean_square_kernel.empty()) {
return false;
}
mean_square_kernel.set(0, ocl::KernelArg::PtrReadOnly(input));
mean_square_kernel.set(1, (int)loops);
mean_square_kernel.set(2, (int)norm_size);
mean_square_kernel.set(3, ocl::KernelArg::PtrReadOnly(mean));
mean_square_kernel.set(4, ocl::KernelArg::PtrWriteOnly(tmp));
ret = mean_square_kernel.run(2, global, NULL, false);
if (!ret) {
return false;
}
ret = ocl4dnn::ocl4dnnGEMV<float>(ocl4dnn::CblasNoTrans, loops, norm_size, inv_norm_size,
tmp, 0, one, 0, 0.f, mean_square, 0);
if (!ret) {
return false;
}
// Calculate instance norm: output = scale * (x - mean) / sqrt(var + eps) + bias
String mvn_kernel_name = format("mvn%d", num_vector);
build_opt += " -DNORM_VARIANCE -DFUSE_BATCH_NORM -DKERNEL_MVN";
ocl::Kernel mvn_kernel(mvn_kernel_name.c_str(), ocl::dnn::mvn_oclsrc, build_opt);
if (mvn_kernel.empty()) {
return false;
}
mvn_kernel.set(0, ocl::KernelArg::PtrReadOnly(input));
mvn_kernel.set(1, (int)loops);
mvn_kernel.set(2, (int)norm_size);
mvn_kernel.set(3, (float)epsilon);
mvn_kernel.set(4, ocl::KernelArg::PtrReadOnly(mean));
mvn_kernel.set(5, ocl::KernelArg::PtrReadOnly(mean_square));
mvn_kernel.set(6, ocl::KernelArg::PtrReadOnly(scale));
mvn_kernel.set(7, ocl::KernelArg::PtrReadOnly(bias));
mvn_kernel.set(8, (int)C);
mvn_kernel.set(9, (float)0.f);
mvn_kernel.set(10, ocl::KernelArg::PtrWriteOnly(output));
ret = mvn_kernel.run(2, global, NULL, false);
if (!ret) {
return false;
}
return true;
}
#endif
#ifdef HAVE_CANN
virtual Ptr<BackendNode> initCann(const std::vector<Ptr<BackendWrapper> > &inputs,
const std::vector<Ptr<BackendWrapper> > &outputs,
const std::vector<Ptr<BackendNode> >& nodes) CV_OVERRIDE {
auto input_tensor_wrapper = inputs[0].dynamicCast<CannBackendWrapper>();
auto input_tensor_desc = input_tensor_wrapper->getTensorDesc();
auto scale_tensor_wrapper = inputs[1].dynamicCast<CannBackendWrapper>();
auto scale_tensor_desc = scale_tensor_wrapper->getTensorDesc();
auto bias_tensor_wrapper = inputs[2].dynamicCast<CannBackendWrapper>();
auto bias_tensor_desc = bias_tensor_wrapper->getTensorDesc();
auto last_node = nodes[0].dynamicCast<CannBackendNode>()->getOp();
auto scale_node = nodes[1].dynamicCast<CannBackendNode>()->getOp();
auto bias_node = nodes[2].dynamicCast<CannBackendNode>()->getOp();
auto op = std::make_shared<ge::op::InstanceNorm>(name);
// set attrs
op->set_attr_epsilon(epsilon);
// set inputs
// set inputs : x
op->set_input_x_by_name(*last_node, input_tensor_wrapper->name.c_str());
op->update_input_desc_x(*input_tensor_desc);
// set inputs : gamma
op->set_input_gamma_by_name((*scale_node), scale_tensor_wrapper->name.c_str());
op->update_input_desc_gamma(*scale_tensor_desc);
// set inputs : beta
op->set_input_beta_by_name(*bias_node, bias_tensor_wrapper->name.c_str());
op->update_input_desc_beta(*bias_tensor_desc);
// set outputs
auto output_desc_y = std::make_shared<ge::TensorDesc>(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT);
op->update_output_desc_y(*output_desc_y);
auto output_desc_mean = std::make_shared<ge::TensorDesc>(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT);
op->update_output_desc_mean(*output_desc_mean);
auto output_desc_var = std::make_shared<ge::TensorDesc>(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT);
op->update_output_desc_variance(*output_desc_var);
return Ptr<BackendNode>(new CannBackendNode(op));
}
#endif // HAVE_CANN
#ifdef HAVE_DNN_NGRAPH
virtual Ptr<BackendNode> initNgraph(const std::vector<Ptr<BackendWrapper> >& inputs,
const std::vector<Ptr<BackendNode> >& nodes) CV_OVERRIDE {
// onnx to openvino convertion: https://github.com/openvinotoolkit/openvino/blob/2023.1.0/src/frontends/onnx/frontend/src/op/instance_norm.cpp
auto ieInpNode = nodes[0].dynamicCast<InfEngineNgraphNode>()->node;
const auto &input_shape = ieInpNode.get_shape();
std::shared_ptr<ngraph::Node> mvn, result;
// mvn
#if INF_ENGINE_VER_MAJOR_LE(INF_ENGINE_RELEASE_2021_2)
// https://docs.openvino.ai/2021.4/api/ngraph_python_api/_autosummary/ngraph.opset3.mvn.html?highlight=mvn#ngraph.opset3.mvn
bool across_channels = false;
bool normalize_variance = true;
mvn = std::make_shared<ngraph::op::MVN>(ieInpNode, across_channels, normalize_variance, epsilon);
#else
// https://docs.openvino.ai/2023.1/openvino_docs_ops_normalization_MVN_6.html
std::vector<int64_t> axes_v(input_shape.size() - 2);
std::iota(axes_v.begin(), axes_v.end(), 2); // {2, 3, ...} for nd input tensor, n>=3
auto axes = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{axes_v.size()}, axes_v.data());
bool normalize_variance = true;
mvn = std::make_shared<ngraph::op::v6::MVN>(ieInpNode, axes, normalize_variance, epsilon, ngraph::op::MVNEpsMode::INSIDE_SQRT);
#endif
// instance norm = scale * mvn + bias
auto scale = nodes[1].dynamicCast<InfEngineNgraphNode>()->node;
std::vector<int64_t> shared_shape_v(input_shape.size(), 1);
shared_shape_v[1] = -1;
auto shared_shape = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{shared_shape_v.size()}, shared_shape_v.data());
scale = std::make_shared<ngraph::op::v1::Reshape>(scale, shared_shape, true);
result = std::make_shared<ngraph::op::v1::Multiply>(mvn, scale);
auto bias = nodes[2].dynamicCast<InfEngineNgraphNode>()->node;
bias = std::make_shared<ngraph::op::v1::Reshape>(bias, shared_shape, true);
result = std::make_shared<ngraph::op::v1::Add>(result, bias);
return Ptr<BackendNode>(new InfEngineNgraphNode(result));
}
#endif // HAVE_DNN_NGRAPH
#ifdef HAVE_CUDA
Ptr<BackendNode> initCUDA(void *context_,
const std::vector<Ptr<BackendWrapper>>& inputs,
const std::vector<Ptr<BackendWrapper>>& outputs) override {
auto context = reinterpret_cast<csl::CSLContext*>(context_);
auto input_wrapper = inputs[0].dynamicCast<CUDABackendWrapper>();
auto input_shape = input_wrapper->getShape();
size_t loops = static_cast<size_t>(total(input_shape, 0, 2));
return make_cuda_node<cuda4dnn::InstanceNormOp>(preferableTarget, std::move(context->stream), epsilon, loops);
}
#endif // HAVE_CUDA
};
Ptr<InstanceNormLayer> InstanceNormLayer::create(const LayerParams &params) {
return Ptr<InstanceNormLayer>(new InstanceNormLayerImpl(params));
}
}} // cv::dnn
modules/dnn/src/layers/layer_norm.cpp
+230 -1
View File
@@ -6,8 +6,29 @@
#include "layers_common.hpp"
#include "cpu_kernels/fast_norm.hpp"
// CANN backend
#include "../op_cann.hpp"
// OpenVINO backend
#include "../op_inf_engine.hpp"
#include "../ie_ngraph.hpp"
// CUDA backend
#include "../op_cuda.hpp"
#ifdef HAVE_CUDA
#include "../cuda4dnn/primitives/layer_norm.hpp"
using namespace cv::dnn::cuda4dnn;
#endif
// OpenCL backend
#ifdef HAVE_OPENCL
#include "../ocl4dnn/include/math_functions.hpp"
#include "opencl_kernels_dnn.hpp"
#endif
namespace cv { namespace dnn {
// https://github.com/onnx/onnx/blob/main/docs/Operators.md#LayerNormalization
class LayerNormLayerImpl CV_FINAL : public LayerNormLayer
{
public:
@@ -22,7 +43,13 @@ public:
virtual bool supportBackend(int backendId) CV_OVERRIDE
{
return backendId == DNN_BACKEND_OPENCV;
#ifdef HAVE_INF_ENGINE
if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)
return true;
#endif
return backendId == DNN_BACKEND_OPENCV ||
backendId == DNN_BACKEND_CUDA ||
(backendId == DNN_BACKEND_CANN && axis != -1); // axis=-1 not supported due to 1d mat shape problem
}
virtual bool getMemoryShapes(const std::vector<MatShape> &inputs,
@@ -69,6 +96,9 @@ public:
CV_TRACE_FUNCTION();
CV_TRACE_ARG_VALUE(name, "name", name.c_str());
CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget),
forward_ocl(inputs_arr, outputs_arr, internals_arr))
if (inputs_arr.depth() == CV_16S)
{
forward_fallback(inputs_arr, outputs_arr, internals_arr);
@@ -90,6 +120,205 @@ public:
fastNorm(input, scale, output, epsilon, static_cast<size_t>(axis));
}
}
#ifdef HAVE_OPENCL
bool forward_ocl(InputArrayOfArrays inputs_, OutputArrayOfArrays outputs_, OutputArrayOfArrays internals_) {
std::vector<UMat> inputs;
std::vector<UMat> outputs;
inputs_.getUMatVector(inputs);
outputs_.getUMatVector(outputs);
const auto &input = inputs[0], &scale = inputs[1]; // &bias = inputs[2]; // bias is optional
auto &output = outputs[0];
const auto input_shape = shape(input);
size_t loops = static_cast<size_t>(total(input_shape, 0, axis)),
norm_size = static_cast<size_t>(total(input_shape, axis));
float inv_norm_size = 1.f / norm_size;
const auto &bias = inputs.size() == 3 ? inputs[2] : UMat::zeros(norm_size, 1, CV_32F);
// no fp16 support
if (input.depth() == CV_16S) {
return false;
}
String base_opts = format(" -DT=float -DT4=float4 -Dconvert_T=convert_float4");
// Calculate mean
UMat one = UMat::ones(norm_size, 1, CV_32F);
UMat mean = UMat(loops, 1, CV_32F);
UMat mean_square = UMat(loops, 1, CV_32F);
UMat tmp = UMat(loops, norm_size, CV_32F);
bool ret = ocl4dnn::ocl4dnnGEMV<float>(ocl4dnn::CblasNoTrans, loops, norm_size, inv_norm_size,
input, 0, one, 0, 0.f, mean, 0);
if (!ret) {
return false;
}
// Calculate mean_square
int num_vector = (norm_size % 8 == 0) ? 8 : ((norm_size % 4 == 0) ? 4 : 1);
size_t global[] = {loops, static_cast<size_t>(norm_size / num_vector)};
String build_opt = format(" -DNUM=%d", num_vector) + base_opts;
String mean_square_kernel_name = format("calc_mean%d", num_vector);
ocl::Kernel mean_square_kernel(mean_square_kernel_name.c_str(), ocl::dnn::mvn_oclsrc, build_opt + " -DKERNEL_MEAN");
if (mean_square_kernel.empty()) {
return false;
}
mean_square_kernel.set(0, ocl::KernelArg::PtrReadOnly(input));
mean_square_kernel.set(1, (int)loops);
mean_square_kernel.set(2, (int)norm_size);
mean_square_kernel.set(3, ocl::KernelArg::PtrReadOnly(mean));
mean_square_kernel.set(4, ocl::KernelArg::PtrWriteOnly(tmp));
ret = mean_square_kernel.run(2, global, NULL, false);
if (!ret) {
return false;
}
ret = ocl4dnn::ocl4dnnGEMV<float>(ocl4dnn::CblasNoTrans, loops, norm_size, inv_norm_size,
tmp, 0, one, 0, 0.f, mean_square, 0);
if (!ret) {
return false;
}
// Calculate instance norm: output = scale * (x - mean) / sqrt(var + eps) + bias
String mvn_kernel_name = format("mvn%d", num_vector);
build_opt += " -DNORM_VARIANCE -DLAYER_NORM -DKERNEL_MVN";
ocl::Kernel mvn_kernel(mvn_kernel_name.c_str(), ocl::dnn::mvn_oclsrc, build_opt);
if (mvn_kernel.empty()) {
return false;
}
mvn_kernel.set(0, ocl::KernelArg::PtrReadOnly(input));
mvn_kernel.set(1, (int)loops);
mvn_kernel.set(2, (int)norm_size);
mvn_kernel.set(3, (float)epsilon);
mvn_kernel.set(4, ocl::KernelArg::PtrReadOnly(mean));
mvn_kernel.set(5, ocl::KernelArg::PtrReadOnly(mean_square));
mvn_kernel.set(6, ocl::KernelArg::PtrReadOnly(scale));
mvn_kernel.set(7, ocl::KernelArg::PtrReadOnly(bias));
mvn_kernel.set(8, (int)1);
mvn_kernel.set(9, (float)0.f);
mvn_kernel.set(10, ocl::KernelArg::PtrWriteOnly(output));
ret = mvn_kernel.run(2, global, NULL, false);
if (!ret) {
return false;
}
return true;
}
#endif
#ifdef HAVE_CANN
virtual Ptr<BackendNode> initCann(const std::vector<Ptr<BackendWrapper> > &inputs,
const std::vector<Ptr<BackendWrapper> > &outputs,
const std::vector<Ptr<BackendNode> >& nodes) CV_OVERRIDE {
CV_CheckEQ(inputs.size(), static_cast<size_t>(3), "LayerNorm/CANN: requires three input wrappers");
CV_CheckEQ(nodes.size(), static_cast<size_t>(3), "LayerNorm/CANN: requires three input nodes");
auto input_tensor_wrapper = inputs[0].dynamicCast<CannBackendWrapper>();
auto input_tensor_desc = input_tensor_wrapper->getTensorDesc();
CV_CheckNE(axis, static_cast<int>(input_tensor_desc->GetShape().GetDimNum() - 1), "LayerNorm: CANN does not support axis set as last axis due to 1D mat compatibility issue");
auto scale_tensor_wrapper = inputs[1].dynamicCast<CannBackendWrapper>();
auto scale_tensor_desc = scale_tensor_wrapper->getTensorDesc();
auto bias_tensor_wrapper = inputs[2].dynamicCast<CannBackendWrapper>();
auto bias_tensor_desc = bias_tensor_wrapper->getTensorDesc();
auto last_node = nodes[0].dynamicCast<CannBackendNode>()->getOp();
auto scale_node = nodes[1].dynamicCast<CannBackendNode>()->getOp();
auto bias_node = nodes[2].dynamicCast<CannBackendNode>()->getOp();
auto op = std::make_shared<ge::op::LayerNorm>(name);
// set attrs
op->set_attr_begin_norm_axis(axis);
op->set_attr_begin_params_axis(axis);
op->set_attr_epsilon(epsilon);
// set inputs
// set inputs : x
op->set_input_x_by_name(*last_node, input_tensor_wrapper->name.c_str());
op->update_input_desc_x(*input_tensor_desc);
// set inputs : gamma
op->set_input_gamma_by_name(*scale_node, scale_tensor_wrapper->name.c_str());
op->update_input_desc_gamma(*scale_tensor_desc);
// set inputs : beta
op->set_input_beta_by_name(*bias_node, bias_tensor_wrapper->name.c_str());
op->update_input_desc_beta(*bias_tensor_desc);
// set outputs
auto output_desc_y = std::make_shared<ge::TensorDesc>(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT);
op->update_output_desc_y(*output_desc_y);
auto output_desc_mean = std::make_shared<ge::TensorDesc>(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT);
op->update_output_desc_mean(*output_desc_mean);
auto output_desc_var = std::make_shared<ge::TensorDesc>(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT);
op->update_output_desc_variance(*output_desc_var);
return Ptr<BackendNode>(new CannBackendNode(op));
}
#endif // HAVE_CANN
#ifdef HAVE_DNN_NGRAPH
virtual Ptr<BackendNode> initNgraph(const std::vector<Ptr<BackendWrapper> >& inputs,
const std::vector<Ptr<BackendNode> >& nodes) CV_OVERRIDE {
auto ieInpNode = nodes[0].dynamicCast<InfEngineNgraphNode>()->node;
const auto &input_shape = ieInpNode.get_shape();
std::shared_ptr<ngraph::Node> mvn, result;
// mvn
#if INF_ENGINE_VER_MAJOR_LE(INF_ENGINE_RELEASE_2021_2)
// https://docs.openvino.ai/2021.4/api/ngraph_python_api/_autosummary/ngraph.opset3.mvn.html?highlight=mvn#ngraph.opset3.mvn
bool across_channels = false;
bool normalize_variance = true;
mvn = std::make_shared<ngraph::op::MVN>(ieInpNode, across_channels, normalize_variance, epsilon);
#else
// https://docs.openvino.ai/2023.1/openvino_docs_ops_normalization_MVN_6.html
std::vector<int64_t> axes_v(input_shape.size() - axis);
std::iota(axes_v.begin(), axes_v.end(), axis);
auto axes = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{axes_v.size()}, axes_v.data());
bool normalize_variance = true;
mvn = std::make_shared<ngraph::op::v6::MVN>(ieInpNode, axes, normalize_variance, epsilon, ngraph::op::MVNEpsMode::INSIDE_SQRT);
#endif
// layer norm = scale * mvn + bias
auto scale = nodes[1].dynamicCast<InfEngineNgraphNode>()->node;
ngraph::Output<ngraph::Node> bias;
if (nodes.size() == 3) {
bias = nodes[2].dynamicCast<InfEngineNgraphNode>()->node;
}
if (axis == -1 || axis == input_shape.size() - 1) { // special case for 1D tensor (2D mat)
std::vector<int64_t> shared_shape_v(input_shape.size(), 1);
shared_shape_v.back() = -1;
auto shared_shape = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{shared_shape_v.size()}, shared_shape_v.data());
scale = std::make_shared<ngraph::op::v1::Reshape>(scale, shared_shape, true);
if (nodes.size() == 3) {
bias = std::make_shared<ngraph::op::v1::Reshape>(bias, shared_shape, true);
}
}
result = std::make_shared<ngraph::op::v1::Multiply>(mvn, scale);
if (nodes.size() == 3) {
result = std::make_shared<ngraph::op::v1::Add>(result, bias);
}
return Ptr<BackendNode>(new InfEngineNgraphNode(result));
}
#endif // HAVE_DNN_NGRAPH
#ifdef HAVE_CUDA
Ptr<BackendNode> initCUDA(void *context_,
const std::vector<Ptr<BackendWrapper>>& inputs,
const std::vector<Ptr<BackendWrapper>>& outputs) override {
auto context = reinterpret_cast<csl::CSLContext*>(context_);
auto input_wrapper = inputs[0].dynamicCast<CUDABackendWrapper>();
auto input_shape = input_wrapper->getShape();
size_t loops = static_cast<size_t>(total(input_shape, 0, axis));
return make_cuda_node<cuda4dnn::LayerNormOp>(preferableTarget, std::move(context->stream), axis, epsilon, loops);
}
#endif // HAVE_CUDA
};
Ptr<LayerNormLayer> LayerNormLayer::create(const LayerParams& params)
modules/dnn/src/layers/layers_common.cpp
+1 -1
View File
@@ -195,7 +195,7 @@ void getConvolutionKernelParams(const LayerParams &params, std::vector<size_t>&
util::getStrideAndPadding(params, pads_begin, pads_end, strides, padMode, kernel.size());
util::getParameter(params, "dilation", "dilation", dilations, true, std::vector<size_t>(kernel.size(), 1));
util::getParameter(params, "adj", "adj", adjust_pads, true, std::vector<size_t>(kernel.size(), 0));
useWinograd = params.get<bool>("use_winograd", true);
useWinograd = params.get<bool>("use_winograd", useWinograd);
for (int i = 0; i < dilations.size(); i++)
CV_Assert(dilations[i] > 0);
modules/dnn/src/layers/matmul_layer.cpp
+326
View File
@@ -0,0 +1,326 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#include "../precomp.hpp"
#include <opencv2/dnn/shape_utils.hpp>
#include "cpu_kernels/fast_gemm.hpp"
// OpenVINO backend
#include "../op_inf_engine.hpp"
#include "../ie_ngraph.hpp"
// Vulkan backend
#include "../op_vkcom.hpp"
// CUDA backend
#ifdef HAVE_CUDA
#include "../cuda4dnn/primitives/matmul_broadcast.hpp"
using namespace cv::dnn::cuda4dnn;
#endif
// CANN backend
#include "../op_cann.hpp"
namespace cv { namespace dnn {
class MatMulLayerImpl CV_FINAL : public MatMulLayer {
public:
MatMulLayerImpl(const LayerParams& params) {
setParamsFrom(params);
trans_a = params.get<bool>("transA", false);
trans_b = params.get<bool>("transB", false);
alpha = params.get<float>("alpha", 1.f);
beta = params.get<float>("beta", 1.f);
}
virtual bool supportBackend(int backendId) CV_OVERRIDE {
return backendId == DNN_BACKEND_OPENCV ||
backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH ||
(backendId == DNN_BACKEND_VKCOM && haveVulkan() && !trans_a && !trans_b) ||
backendId == DNN_BACKEND_CUDA ||
backendId == DNN_BACKEND_CANN;
}
virtual bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int requiredOutputs,
std::vector<MatShape> &outputs,
std::vector<MatShape> &internals) const CV_OVERRIDE {
CV_CheckGE(inputs.size(), static_cast<size_t>(1), "DNN/MatMul: one varible input at least");
CV_CheckLE(inputs.size(), static_cast<size_t>(2), "DNN/MatMul: two variable inputs at most");
const auto shape_A = inputs[0], shape_B = blobs.empty() ? inputs[1] : shape(blobs[0]);
CV_CheckGE(shape_A.size(), static_cast<size_t>(2), "DNN/MatMul: invalid shape of input A");
CV_CheckGE(shape_B.size(), static_cast<size_t>(2), "DNN/MatMul: invalid shape of input B");
// Check legal matrix multiplication
int mA = shape_A[shape_A.size() - 2], nA = shape_A.back();
int mB = shape_B[shape_B.size() - 2], nB = shape_B.back();
int M = trans_a ? nA : mA;
int N = trans_b ? mB : nB;
int K_A = trans_a ? mA : nA;
int K_B = trans_b ? nB : mB;
CV_CheckEQ(K_A, K_B, "DNN/MatMul: invalid dimension K");
// Check legal broadcast. It is legal for sure if A and B are 2d, or one of them is 2d.
MatShape common_shape;
if (shape_A.size() != 2 || shape_B.size() != 2) {
const auto &shape_more_dims = shape_A.size() > shape_B.size() ? shape_A : shape_B;
const auto &shape_less_dims = shape_A.size() > shape_B.size() ? shape_B : shape_A;
size_t diff_dims = shape_more_dims.size() - shape_less_dims.size();
common_shape = shape_more_dims;
for (size_t i = 0; i < shape_less_dims.size() - 2; i++) {
const auto dl = shape_less_dims[i], dm = shape_more_dims[i + diff_dims];
if (dl != 1 && dm != 1 && dl != dm) {
CV_Error(Error::StsBadSize, format("DNN/MatMul: invalid shape for broadcasting, shape_A[%zu]=%d, shape_B[%zu]=%d\n", i, shape_less_dims[i], i, shape_more_dims[i + diff_dims]));
}
if (dm == 1) {
common_shape[i + diff_dims] = dl;
}
}
common_shape[common_shape.size() - 2] = M;
common_shape[common_shape.size() - 1] = N;
} else {
common_shape.resize(2);
common_shape[0] = M;
common_shape[1] = N;
}
outputs.assign(1, common_shape);
return false;
}
virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE {
opt.init();
std::vector<Mat> inputs, outputs;
inputs_arr.getMatVector(inputs);
outputs_arr.getMatVector(outputs);
const auto A_shape = shape(inputs[0]),
B_shape = blobs.empty() ? shape(inputs[1]) : shape(blobs[0]),
C_shape = shape(outputs[0]);
helper.compute(trans_a, trans_b, A_shape, B_shape, C_shape);
if (!blobs.empty()) {
fastGemmPackB(blobs[0], packed_input_B, trans_b, opt);
helper.updatePackedBOffsets(packed_input_B.size());
}
}
// works like Y = numpy.matmul(A, B)
void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE {
CV_TRACE_FUNCTION();
CV_TRACE_ARG_VALUE(name, "name", name.c_str());
CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget),
forward_ocl(inputs_arr, outputs_arr, internals_arr))
if (inputs_arr.depth() == CV_16S)
{
forward_fallback(inputs_arr, outputs_arr, internals_arr);
return;
}
std::vector<Mat> inputs, outputs;
inputs_arr.getMatVector(inputs);
outputs_arr.getMatVector(outputs);
const auto &A = inputs[0];
auto &Y = outputs[0];
const auto *a = A.ptr<const float>();
auto *y = Y.ptr<float>();
std::memset(y, 0, Y.total() * sizeof(float));
if (blobs.empty()) {
const auto &B = inputs[1];
const auto *b = B.ptr<const float>();
fastGemmBatch(helper.batch, helper.A_offsets.data(), helper.B_offsets.data(), helper.C_offsets.data(),
helper.M, helper.N, helper.K, alpha, a, helper.lda0, helper.lda1,
b, helper.ldb0, helper.ldb1, beta, y, helper.ldc, opt);
} else {
fastGemmBatch(helper.batch, helper.A_offsets.data(), helper.packed_B_offsets.data(), helper.C_offsets.data(),
helper.M, helper.N, helper.K, alpha, a, helper.lda0, helper.lda1,
packed_input_B.data(), beta, y, helper.ldc, opt);
}
}
#ifdef HAVE_OPENCL
bool forward_ocl(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, InputArrayOfArrays internals) {
std::vector<UMat> inputs;
std::vector<UMat> outputs;
bool use_half = (inputs_arr.depth() == CV_16S);
inputs_arr.getUMatVector(inputs);
outputs_arr.getUMatVector(outputs);
const auto &input_A = inputs[0];
UMat input_B;
if (blobs.empty()) {
input_B = inputs[1];
} else {
blobs[0].copyTo(input_B);
}
auto &output = outputs[0];
int M = static_cast<int>(helper.M),
N = static_cast<int>(helper.N),
K = static_cast<int>(helper.K),
batch = static_cast<int>(helper.batch);
int batch_A = total(shape(input_A)) / (M * K),
batch_B = total(shape(input_B)) / (N * K);
MatShape new_shape_A{batch_A, M * K}, new_shape_B{batch_B, N * K}, new_shape_output{batch, M * N};
const auto input_A_2d = input_A.reshape(1, new_shape_A.size(), &new_shape_A[0]),
input_B_2d = input_B.reshape(1, new_shape_B.size(), &new_shape_B[0]);
auto output_2d = output.reshape(1, new_shape_output.size(), &new_shape_output[0]);
UMat A, B, C, A_fp32, B_fp32, C_fp32;
for (int i = 0; i < batch; i++) {
A = input_A_2d.row(helper.A_rows[i]).reshape(1, trans_a ? K : M);
B = input_B_2d.row(helper.B_rows[i]).reshape(1, trans_b ? K : N);
C = output_2d.row(helper.C_rows[i]).reshape(1, M);
if (trans_a) {
A = A.t();
}
if (trans_b) {
B = B.t();
}
if (use_half) {
convertFp16(A, A_fp32);
convertFp16(B, B_fp32);
convertFp16(C, C_fp32);
} else {
A_fp32 = A;
B_fp32 = B;
C_fp32 = C;
}
cv::gemm(A_fp32, B_fp32, 1.f, noArray(), 0.f, C_fp32);
if (use_half) {
convertFp16(A_fp32, A);
convertFp16(B_fp32, B);
convertFp16(C_fp32, C);
}
}
return true;
}
#endif // HAVE_OPENCL
#ifdef HAVE_DNN_NGRAPH
virtual Ptr<BackendNode> initNgraph(const std::vector<Ptr<BackendWrapper> >& inputs,
const std::vector<Ptr<BackendNode> >& nodes) CV_OVERRIDE {
auto& input_A_node = nodes[0].dynamicCast<InfEngineNgraphNode>()->node;
std::shared_ptr<ngraph::Node> matmul;
if (nodes.size() == 2) {
auto &input_B_node = nodes[1].dynamicCast<InfEngineNgraphNode>()->node;
matmul = std::make_shared<ngraph::op::MatMul>(input_A_node, input_B_node, trans_a, trans_b);
} else {
auto input_B_shape = getShape<size_t>(blobs[0]);
auto input_B_node = std::make_shared<ngraph::op::Constant>(ngraph::element::f32, input_B_shape, blobs[0].data);
matmul = std::make_shared<ngraph::op::MatMul>(input_A_node, input_B_node, trans_a, trans_b);
}
return Ptr<BackendNode>(new InfEngineNgraphNode(matmul));
}
#endif // HAVE_DNN_NGRAPH
#ifdef HAVE_VULKAN
virtual Ptr<BackendNode> initVkCom(const std::vector<Ptr<BackendWrapper> > &inputs,
std::vector<Ptr<BackendWrapper> > &outputs) CV_OVERRIDE {
auto input_A_wrapper = inputs[0].dynamicCast<VkComBackendWrapper>();
auto output_wrapper = outputs[0].dynamicCast<VkComBackendWrapper>();
const auto input_A_shape = shape(*input_A_wrapper->getMat());
const auto output_shape = shape(*output_wrapper->getMat());
if (output_shape.size() != 2) {
return Ptr<BackendNode>();
}
std::vector<Mat> constants;
if (!blobs.empty()) {
constants.push_back(blobs[0]);
}
Ptr<vkcom::OpBase> op = new vkcom::OpMatMul(constants, input_A_shape[0], input_A_shape[1], output_shape[1]);
return Ptr<BackendNode>(new VkComBackendNode(inputs, op, outputs));
}
#endif
#ifdef HAVE_CUDA
Ptr<BackendNode> initCUDA(void *context_,
const std::vector<Ptr<BackendWrapper>>& inputs,
const std::vector<Ptr<BackendWrapper>>& outputs) override {
auto context = reinterpret_cast<csl::CSLContext*>(context_);
auto input_B = blobs.empty() ? Mat() : blobs[0];
CV_CheckFalse(helper.empty(), "DNN/MatMul/CUDA: MatMulHelper is not initialized");
return make_cuda_node<cuda4dnn::MatMulBroadcastOp>(preferableTarget, std::move(context->stream), std::move(context->cublas_handle), input_B, trans_a, trans_b, helper.A_offsets, helper.B_offsets, helper.C_offsets, helper.batch);
}
#endif // HAVE_CUDA
#ifdef HAVE_CANN
virtual Ptr<BackendNode> initCann(const std::vector<Ptr<BackendWrapper> > &inputs,
const std::vector<Ptr<BackendWrapper> > &outputs,
const std::vector<Ptr<BackendNode> >& nodes) CV_OVERRIDE {
auto input_A_wrapper = inputs[0].dynamicCast<CannBackendWrapper>();
auto input_A_desc = input_A_wrapper->getTensorDesc();
auto input_A_node = nodes[0].dynamicCast<CannBackendNode>()->getOp();
auto op = std::make_shared<ge::op::BatchMatMul>(name);
// set attributes
op->set_attr_adj_x1(trans_a);
op->set_attr_adj_x2(trans_b);
// set inputs
// set inputs : x1
op->set_input_x1_by_name(*input_A_node, input_A_wrapper->name.c_str());
op->update_input_desc_x1(*input_A_desc);
// set inputs : x2
if (blobs.empty()) { // varaible input B
auto input_B_wrapper = inputs[1].dynamicCast<CannBackendWrapper>();
auto input_B_desc = input_B_wrapper->getTensorDesc();
auto input_B_node = nodes[1].dynamicCast<CannBackendNode>()->getOp();
op->set_input_x2_by_name(*input_B_node, "y");
op->update_input_desc_x2(*input_B_desc);
} else { // constant input B
auto B = blobs[0];
auto const_B_node = std::make_shared<CannConstOp>(B.data, B.type(), shape(B), cv::format("%s_B", name.c_str()));
op->set_input_x2_by_name(*(const_B_node->getOp()), "y");
op->update_input_desc_x2(*(const_B_node->getTensorDesc()));
}
// set outputs
auto output_desc = std::make_shared<ge::TensorDesc>(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT);
op->update_output_desc_y(*output_desc);
return Ptr<BackendNode>(new CannBackendNode(op));
}
#endif // HAVE_CANN
private:
bool trans_a;
bool trans_b;
float alpha;
float beta;
std::vector<float> packed_input_B;
FastGemmOpt opt;
MatMulHelper helper;
};
Ptr<MatMulLayer> MatMulLayer::create(const LayerParams& params)
{
return makePtr<MatMulLayerImpl>(params);
}
}} // cv::dnn
modules/dnn/src/layers/nary_eltwise_layers.cpp
+8 -3
View File
@@ -114,9 +114,11 @@ public:
op == OPERATION::GREATER_EQUAL ||
op == OPERATION::LESS_EQUAL
);
if (op == OPERATION::MAX || op == OPERATION::MIN || op == OPERATION::SUM ||
op == OPERATION::PROD || op == OPERATION::DIV || op == OPERATION::ADD)
return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA;
if (backendId == DNN_BACKEND_CUDA) {
return op == OPERATION::MAX || op == OPERATION::MIN || op == OPERATION::SUM ||
op == OPERATION::PROD || op == OPERATION::DIV || op == OPERATION::ADD ||
op == OPERATION::SUB;
}
return backendId == DNN_BACKEND_OPENCV;
}
@@ -832,6 +834,9 @@ public:
case OPERATION::ADD:
op_ = cuda4dnn::EltwiseOpType::SUM;
break;
case OPERATION::SUB:
op_ = cuda4dnn::EltwiseOpType::SUB;
break;
default: return Ptr<BackendNode>(); // return empty cuda_node if the EltwiseOpType is unsupported type.
};
modules/dnn/src/layers/region_layer.cpp
+3 -4
View File
@@ -45,6 +45,7 @@
#include <opencv2/dnn/shape_utils.hpp>
#include <opencv2/dnn/all_layers.hpp>
#include "../nms.inl.hpp"
#include "cpu_kernels/softmax.hpp"
#ifdef HAVE_OPENCL
#include "opencl_kernels_dnn.hpp"
@@ -280,10 +281,8 @@ public:
}
if (useSoftmax) { // Yolo v2
for (int i = 0; i < batch_size*rows*cols*anchors; ++i) {
int index = cell_size*i;
softmax_activate(srcData + index + 5, classes, 1, dstData + index + 5);
}
Mat _inpBlob = inpBlob.reshape(0, outBlob.dims, outBlob.size);
softmax(outBlob, _inpBlob, -1, 5, classes);
}
else if (useLogistic) { // Yolo v3
for (int i = 0; i < batch_size*rows*cols*anchors; ++i){
modules/dnn/src/layers/softmax_layer.cpp
+6 -79
View File
@@ -51,6 +51,7 @@
#include <algorithm>
#include <stdlib.h>
#include <opencv2/core/utils/logger.hpp>
#include "cpu_kernels/softmax.hpp"
using std::max;
#ifdef HAVE_OPENCL
@@ -74,7 +75,7 @@ public:
SoftMaxLayerImpl(const LayerParams& params)
{
axisRaw = params.get<int>("axis", 1);
axisRaw = params.get<int>("axis", -1);
logSoftMax = params.get<bool>("log_softmax", false);
setParamsFrom(params);
}
@@ -223,89 +224,15 @@ public:
std::vector<Mat> inputs, outputs, internals;
inputs_arr.getMatVector(inputs);
outputs_arr.getMatVector(outputs);
internals_arr.getMatVector(internals);
const Mat &src = inputs[0];
Mat &dst = outputs[0];
int axis = normalize_axis(axisRaw, src.dims);
size_t outerSize = src.total(0, axis), channels = src.size[axis],
innerSize = src.total(axis + 1);
CV_Assert(src.type() == CV_32F);
CV_Assert(src.isContinuous() && dst.isContinuous());
const float *srcPtr = src.ptr<float>();
float *dstPtr = dst.ptr<float>();
float *bufPtr = internals[0].ptr<float>();
size_t outerStep = src.total(axis);
size_t cnStep = src.total(axis + 1);
//compute max along axis
for (size_t outerDim = 0; outerDim < outerSize; outerDim++)
{
size_t srcOffset = outerDim * outerStep;
size_t bufOffset = outerDim * cnStep;
memcpy(bufPtr + bufOffset, srcPtr + srcOffset, innerSize * sizeof(float));
for (size_t cnDim = 1; cnDim < channels; cnDim++)
{
for (size_t i = 0; i < innerSize; i++)
bufPtr[bufOffset + i] = std::max(bufPtr[bufOffset + i], srcPtr[srcOffset + cnDim * cnStep + i]);
}
}
//subtract max
for (size_t outerDim = 0; outerDim < outerSize; outerDim++)
{
size_t srcOffset = outerDim * outerStep;
size_t bufOffset = outerDim * cnStep;
for (size_t cnDim = 0; cnDim < channels; cnDim++)
{
const int offset = srcOffset + cnDim * cnStep;
for (size_t i = 0; i < innerSize; i++)
dstPtr[offset + i] = srcPtr[offset + i] - bufPtr[bufOffset + i];
}
}
cv::exp(dst, dst);
for (size_t outerDim = 0; outerDim < outerSize; outerDim++)
{
size_t srcOffset = outerDim * outerStep;
size_t bufOffset = outerDim * cnStep;
//sum exp along axis
for (size_t i = 0; i < innerSize; i++)
bufPtr[bufOffset + i] = 0.f;
for (size_t cnDim = 0; cnDim < channels; cnDim++)
{
const int offset = srcOffset + cnDim * cnStep;
for (size_t i = 0; i < innerSize; i++)
bufPtr[bufOffset + i] += dstPtr[offset + i];
}
//divide by computed sum
for (size_t cnDim = 0; cnDim < channels; cnDim++)
{
const int offset = srcOffset + cnDim * cnStep;
for (size_t i = 0; i < innerSize; i++)
dstPtr[offset + i] /= bufPtr[bufOffset + i];
}
if (logSoftMax)
{
for (size_t cnDim = 0; cnDim < channels; cnDim++)
{
const int offset = srcOffset + cnDim * cnStep;
for (size_t i = 0; i < innerSize; i++)
dstPtr[offset + i] = log(dstPtr[offset + i]);
}
}
}
if(logSoftMax)
logSoftmax(dst, src, axis);
else
softmax(dst, src, axis);
}
#ifdef HAVE_CUDA
modules/dnn/src/model.cpp
+9
View File
@@ -37,6 +37,7 @@ public:
virtual void setPreferableBackend(Backend backendId) { net.setPreferableBackend(backendId); }
virtual void setPreferableTarget(Target targetId) { net.setPreferableTarget(targetId); }
virtual void enableWinograd(bool useWinograd) { net.enableWinograd(useWinograd); }
virtual
void initNet(const Net& network)
@@ -151,6 +152,7 @@ Model& Model::setPreferableBackend(Backend backendId)
impl->setPreferableBackend(backendId);
return *this;
}
Model& Model::setPreferableTarget(Target targetId)
{
CV_DbgAssert(impl);
@@ -158,6 +160,13 @@ Model& Model::setPreferableTarget(Target targetId)
return *this;
}
Model& Model::enableWinograd(bool useWinograd)
{
CV_DbgAssert(impl);
impl->enableWinograd(useWinograd);
return *this;
}
Model& Model::setInputSize(const Size& size)
{
CV_DbgAssert(impl);
modules/dnn/src/net_impl_backend.cpp
+8
View File
@@ -236,6 +236,14 @@ void Net::Impl::setPreferableTarget(int targetId)
#endif
clear();
if (targetId == DNN_TARGET_CPU_FP16)
{
if (useWinograd) {
CV_LOG_INFO(NULL, "DNN: DNN_TARGET_CPU_FP16 is set => Winograd convolution is disabled by default to preserve accuracy. If needed, enable it explicitly using enableWinograd(true).");
enableWinograd(false);
}
}
}
}
modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp
+10
View File
@@ -1462,6 +1462,16 @@ void OCL4DNNConvSpatial<float>::generate_gemmlike_tuneritems(std::vector< cv::Pt
return;
}
// issue #24734
// OpenCL 1.2: https://registry.khronos.org/OpenCL/specs/opencl-1.2.pdf
// section 6.1.2 page 200: "Supported values of n are 2, 3, 4, 8, and 16 for all vector data types."
// besides of builtin types, kernel code defines extra types up to float15 (see float15 definition)
if (kernel_w_ > 16)
{
CV_LOG_DEBUG(NULL, "DNN/OCL: skip KERNEL_TYPE_GEMM_LIKE with blockMKN=[" << blockM << ", " << blockK << ", " << blockN << "] kernel=" << kernel_w_ << " x " << kernel_h_);
return;
}
tunerItems.push_back(makePtr<tunerParam>(KERNEL_TYPE_GEMM_LIKE, blockM, blockK, blockN));
}
modules/dnn/src/onnx/onnx_graph_simplifier.cpp
+528 -203
View File
File diff suppressed because it is too large Load Diff
modules/dnn/src/onnx/onnx_importer.cpp
+141 -86
View File
@@ -207,6 +207,7 @@ private:
void parseQConcat (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseQGemm (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseQSoftmax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseAttention (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
// '???' domain or '???' layer type
void parseCustomLayer (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
@@ -1235,7 +1236,7 @@ void ONNXImporter::parseSlice(LayerParams& layerParams, const opencv_onnx::NodeP
starts_ = DictValue::arrayInt(start_blob.begin<int>(), start_blob.total());
ends_ = DictValue::arrayInt(end_blob.begin<int>(), end_blob.total());
if (inp_size > 3)
if (inp_size > 3 && !getBlob(node_proto, 3).empty())
{
Mat axes_blob = getBlob(node_proto, 3);
CV_Assert(axes_blob.total() == start_blob.total());
@@ -1244,7 +1245,7 @@ void ONNXImporter::parseSlice(LayerParams& layerParams, const opencv_onnx::NodeP
has_axes = true;
}
if (inp_size == 5)
if (inp_size == 5 && !getBlob(node_proto, 4).empty())
{
Mat step_blob = getBlob(node_proto, 4);
CV_Assert(step_blob.total() == start_blob.total());
@@ -1848,44 +1849,43 @@ void ONNXImporter::parseLRN(LayerParams& layerParams, const opencv_onnx::NodePro
addLayer(layerParams, node_proto);
}
void ONNXImporter::parseInstanceNormalization(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_)
{
opencv_onnx::NodeProto node_proto = node_proto_;
if (node_proto.input_size() != 3)
CV_Error(Error::StsNotImplemented,
"Expected input, scale, bias");
void ONNXImporter::parseInstanceNormalization(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) {
int num_inputs = node_proto.input_size();
CV_CheckEQ(num_inputs, 3, "DNN/ONNXImporter - InstanceNorm: three inputs are required");
layerParams.blobs.resize(4);
layerParams.blobs[2] = getBlob(node_proto, 1); // weightData
layerParams.blobs[3] = getBlob(node_proto, 2); // biasData
layerParams.set("has_bias", true);
layerParams.set("has_weight", true);
bool found_input = constBlobs.find(node_proto.input(0)) != constBlobs.end();
bool found_scale = constBlobs.find(node_proto.input(1)) != constBlobs.end();
bool found_bias = constBlobs.find(node_proto.input(2)) != constBlobs.end();
// Get number of channels in input
int size = layerParams.blobs[2].total();
layerParams.blobs[0] = Mat::zeros(size, 1, CV_32F); // mean
layerParams.blobs[1] = Mat::ones(size, 1, CV_32F); // std
if (found_input && found_scale && found_bias) {
std::vector<Mat> inputs, output;
LayerParams mvnParams;
mvnParams.name = layerParams.name + "/MVN";
mvnParams.type = "MVN";
mvnParams.set("eps", layerParams.get<float>("epsilon"));
layerParams.erase("epsilon");
Mat input = getBlob(node_proto, 0);
Mat scale = getBlob(node_proto, 1);
Mat bias = getBlob(node_proto, 2);
inputs.push_back(input);
inputs.push_back(scale);
inputs.push_back(bias);
//Create MVN layer
int id = dstNet.addLayer(mvnParams.name, mvnParams.type, mvnParams);
//Connect to input
IterLayerId_t layerId = layer_id.find(node_proto.input(0));
CV_Assert(layerId != layer_id.end());
dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0);
//Add shape
layer_id.insert(std::make_pair(mvnParams.name, LayerInfo(id, 0)));
outShapes[mvnParams.name] = outShapes[node_proto.input(0)];
runLayer(layerParams, inputs, output);
addConstant(node_proto.output(0), output[0]);
} else {
auto add_const_node = [&] (int i) {
LayerParams const_params;
const_params.name = node_proto.input(i);
const_params.type = "Const";
Mat blob = getBlob(node_proto, i);
const_params.blobs.push_back(blob);
//Replace Batch Norm's input to MVN
node_proto.set_input(0, mvnParams.name);
layerParams.type = "BatchNorm";
addLayer(layerParams, node_proto);
opencv_onnx::NodeProto proto;
proto.add_output(const_params.name);
addLayer(const_params, proto);
};
if (found_input && layer_id.find(node_proto.input(0)) == layer_id.end()) { add_const_node(0); }
if (found_scale && layer_id.find(node_proto.input(1)) == layer_id.end()) { add_const_node(1); }
if (found_bias && layer_id.find(node_proto.input(2)) == layer_id.end()) { add_const_node(2); }
addLayer(layerParams, node_proto);
}
}
void ONNXImporter::parseBatchNormalization(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto)
@@ -1962,50 +1962,33 @@ void ONNXImporter::parseGemm(LayerParams& layerParams, const opencv_onnx::NodePr
addLayer(layerParams, node_proto);
}
void ONNXImporter::parseMatMul(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_)
{
opencv_onnx::NodeProto node_proto = node_proto_;
CV_Assert(node_proto.input_size() == 2);
layerParams.type = "InnerProduct";
layerParams.set("bias_term", false);
int firstInpDims, secondInpDims;
void ONNXImporter::parseMatMul(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) {
auto node_proto = node_proto_;
CV_CheckEQ(node_proto.input_size(), 2, "ONNXImporter/MatMul: two inputs required");
if (constBlobs.find(node_proto.input(0)) != constBlobs.end())
{
Mat blob = getBlob(node_proto, 0);
firstInpDims = blob.dims;
LayerParams constParams;
constParams.name = layerParams.name + "/const_0";
constParams.type = "Const";
constParams.blobs.push_back(blob);
for (int i = 0; i < node_proto.input_size(); i++) {
if (constBlobs.find(node_proto.input(i)) == constBlobs.end()) {
continue;
}
opencv_onnx::NodeProto tmpProto;
tmpProto.add_output(constParams.name);
addLayer(constParams, tmpProto);
Mat blob = getBlob(node_proto, i);
node_proto.set_input(0, constParams.name);
if (i == 1) {
layerParams.blobs.push_back(blob);
} else {
LayerParams const_params;
const_params.name = node_proto.input(i);
const_params.type = "Const";
const_params.blobs.push_back(blob);
opencv_onnx::NodeProto const_node_proto;
const_node_proto.add_output(const_params.name);
addLayer(const_params, const_node_proto);
node_proto.set_input(i, const_params.name);
}
}
else
firstInpDims = outShapes[node_proto.input(0)].size();
if (constBlobs.find(node_proto.input(1)) != constBlobs.end())
{
Mat blob = getBlob(node_proto, 1);
Mat transBlob;
secondInpDims = blob.dims;
// create order transposing last 2 dimensions
std::vector<int> order(secondInpDims);
std::iota(order.begin(), order.end(), 0);
std::swap(order[secondInpDims - 2], order[secondInpDims - 1]);
transposeND(blob, order, transBlob);
layerParams.blobs.push_back(transBlob);
int numOutput = layerParams.blobs[0].total(0, secondInpDims - 1);
layerParams.set("num_output", numOutput);
layerParams.set("is_matmul", secondInpDims > 2);
} else
secondInpDims = outShapes[node_proto.input(1)].size();
layerParams.set("axis", firstInpDims - 1);
addLayer(layerParams, node_proto);
}
@@ -2789,6 +2772,13 @@ void ONNXImporter::parseUpsample(LayerParams& layerParams, const opencv_onnx::No
void ONNXImporter::parseSoftMax(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto)
{
const std::string& layer_type = node_proto.op_type();
int axis;
if (onnx_opset != 0 && onnx_opset <= 11) {
axis = layerParams.get<int>("axis", 1);
} else {
axis = layerParams.get<int>("axis", -1);
}
layerParams.set<int>("axis", axis);
layerParams.type = "Softmax";
layerParams.set("log_softmax", layer_type == "LogSoftmax");
addLayer(layerParams, node_proto);
@@ -3182,10 +3172,19 @@ void ONNXImporter::parseLayerNorm(LayerParams& layerParams, const opencv_onnx::N
// Remove additional outputs (Mean, InvStdDev)
if (node_proto.output_size() > 1)
{
// remove from graph proto
for (size_t i = 1; i < node_proto.output_size(); i++) {
for (int j = graph_proto.output_size() - 1; j >= 0; j--) {
if (graph_proto.output(j).name() == node_proto.output(i)) {
graph_proto.mutable_output()->DeleteSubrange(j, 1);
break;
}
}
}
// remove from node proto
auto outputName = node_proto.output(0);
opencv_onnx::NodeProto node_proto_ = node_proto;
node_proto_.clear_output();
node_proto_.add_output(outputName);
node_proto_.mutable_output()->DeleteSubrange(1, node_proto_.output_size() - 1);
addLayer(layerParams, node_proto_);
}
else
@@ -3222,19 +3221,44 @@ void ONNXImporter::parseSimpleLayers(LayerParams& layerParams, const opencv_onnx
addLayer(layerParams, node_proto);
}
void ONNXImporter::parseEinsum(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto)
{
std::vector<MatShape> einsumInpShapes;
for (int j = 0; j < node_proto.input_size(); j++)
{
const auto& inputLayerName = node_proto.input(j);
auto it = outShapes.find(inputLayerName);
if (it != outShapes.end())
{
einsumInpShapes.emplace_back(it->second);
// create Const layer for constants and mark its shape
std::vector<int> input_shape;
if (layer_id.find(node_proto.input(j)) == layer_id.end()) {
Mat blob = getBlob(node_proto, j);
LayerParams const_params;
const_params.name = node_proto.input(j);
const_params.type = "Const";
const_params.blobs.push_back(blob);
opencv_onnx::NodeProto proto;
proto.add_output(const_params.name);
addLayer(const_params, proto);
input_shape.resize(blob.dims);
for (size_t i = 0; i < input_shape.size(); i++) {
input_shape[i] = blob.size[i];
}
}
// also try getting shape from inferred shapes
if (input_shape.empty()) {
const auto& inputLayerName = node_proto.input(j);
auto it = outShapes.find(inputLayerName);
if (it != outShapes.end()) {
input_shape = it->second;
}
}
if (input_shape.empty()) {
CV_Error(Error::StsAssert, format("ERROR input shape of %s not found", node_proto.input(j).c_str()));
} else {
CV_Error(Error::StsAssert, "ERROR input shape not found");
einsumInpShapes.emplace_back(input_shape);
}
}
@@ -3525,7 +3549,7 @@ void ONNXImporter::parseQGemm(LayerParams& layerParams, const opencv_onnx::NodeP
Mat bias;
if (constBlobs.find(node_proto.input(6)) != constBlobs.end())
bias = getBlob(node_proto, 6);
else
if (bias.empty())
bias = Mat::zeros(1, outCn, CV_32S);
Mat biasFused(1, outCn, CV_32S);
@@ -3897,6 +3921,31 @@ void ONNXImporter::parseQSoftmax(LayerParams& layerParams, const opencv_onnx::No
addLayer(layerParams, node_proto);
}
void ONNXImporter::parseAttention(LayerParams& params, const opencv_onnx::NodeProto& node_proto) {
CV_CheckTrue(params.has("num_heads"), "ONNXImporter/parseAttention: num_heads is required but missing");
CV_CheckTrue(params.has("qkv_hidden_sizes"), "ONNXImporter/parseAttention: qkv_hidden_sizes is required but missing");
auto param_qkv_hidden_sizes = params.get("qkv_hidden_sizes");
CV_CheckEQ(param_qkv_hidden_sizes.size(), 3, "ONNXImporter/parseAttention: qkv_hidden_sizes is must and only have three elements");
for (size_t i = 1; i < node_proto.input_size(); i++) {
if (layer_id.find(node_proto.input(i)) == layer_id.end()) {
Mat tensor = getBlob(node_proto, i);
LayerParams const_params;
const_params.name = node_proto.input(i);
const_params.type = "Const";
const_params.blobs.push_back(tensor);
opencv_onnx::NodeProto proto;
proto.add_output(const_params.name);
addLayer(const_params, proto);
}
}
addLayer(params, node_proto);
}
// Domain: ai.onnx (default)
// URL: https://github.com/onnx/onnx/blob/master/docs/Operators.md
void ONNXImporter::buildDispatchMap_ONNX_AI(int opset_version)
@@ -3948,7 +3997,7 @@ void ONNXImporter::buildDispatchMap_ONNX_AI(int opset_version)
dispatch["Concat"] = &ONNXImporter::parseConcat;
dispatch["Resize"] = &ONNXImporter::parseResize;
dispatch["Upsample"] = &ONNXImporter::parseUpsample;
dispatch["SoftMax"] = dispatch["LogSoftmax"] = &ONNXImporter::parseSoftMax;
dispatch["SoftMax"] = dispatch["Softmax"] = dispatch["LogSoftmax"] = &ONNXImporter::parseSoftMax;
dispatch["DetectionOutput"] = &ONNXImporter::parseDetectionOutput;
dispatch["CumSum"] = &ONNXImporter::parseCumSum;
dispatch["SpaceToDepth"] = dispatch["DepthToSpace"] = &ONNXImporter::parseDepthToSpace;
@@ -3967,7 +4016,7 @@ void ONNXImporter::buildDispatchMap_ONNX_AI(int opset_version)
std::vector<std::string> simpleLayers{"Acos", "Acosh", "Asin", "Asinh", "Atan", "Atanh", "Ceil", "Celu", "Cos",
"Cosh", "Dropout", "Erf", "Exp", "Floor", "HardSigmoid", "HardSwish",
"Identity", "Log", "Round", "Reciprocal", "Selu", "Sign", "Sigmoid", "Sin", "Sinh", "Softmax",
"Identity", "Log", "Round", "Reciprocal", "Selu", "Sign", "Sigmoid", "Sin", "Sinh",
"Softplus", "Softsign", "Shrink", "Sqrt", "Tan", "ThresholdedRelu", "Gelu",
"GeluApproximation"};
for (const auto& name : simpleLayers)
@@ -3980,6 +4029,11 @@ void ONNXImporter::buildDispatchMap_ONNX_AI(int opset_version)
dispatch["QLinearConv"] = &ONNXImporter::parseQConv;
dispatch["QLinearMatMul"] = &ONNXImporter::parseQMatMul;
// com.microsft: This operator is added for compatibility via onnx graph simplifier.
// Opset domain cannot be modified from onnx_graph_simplifier.cpp so this
// operator cannot be parsed if only added in buildDispatchMap_COM_MICROSOFT
dispatch["Attention"] = &ONNXImporter::parseAttention;
domain_dispatch_map[str_domain_ai_onnx] = dispatch;
}
@@ -3997,6 +4051,7 @@ void ONNXImporter::buildDispatchMap_COM_MICROSOFT(int opset_version)
dispatch["QLinearConcat"] = &ONNXImporter::parseQConcat;
dispatch["QGemm"] = &ONNXImporter::parseQGemm;
dispatch["QLinearSoftmax"] = &ONNXImporter::parseQSoftmax;
dispatch["Attention"] = &ONNXImporter::parseAttention;
domain_dispatch_map["com.microsoft"] = dispatch;
}
modules/dnn/src/opencl/conv_layer_spatial.cl
+17 -17
View File
@@ -95,23 +95,23 @@
#define __CAT(x, y) x##y
#define CAT(x, y) __CAT(x, y)
#define LOOP0(VAR, STMT)
#define LOOP1(VAR, STMT) (STMT); (VAR)++;
#define LOOP2(VAR, STMT) LOOP1(VAR, STMT); (STMT); (VAR)++;
#define LOOP3(VAR, STMT) LOOP2(VAR, STMT); (STMT); (VAR)++;
#define LOOP4(VAR, STMT) LOOP3(VAR, STMT); (STMT); (VAR)++;
#define LOOP5(VAR, STMT) LOOP4(VAR, STMT); (STMT); (VAR)++;
#define LOOP6(VAR, STMT) LOOP5(VAR, STMT); (STMT); (VAR)++;
#define LOOP7(VAR, STMT) LOOP6(VAR, STMT); (STMT); (VAR)++;
#define LOOP8(VAR, STMT) LOOP7(VAR, STMT); (STMT); (VAR)++;
#define LOOP9(VAR, STMT) LOOP8(VAR, STMT); (STMT); (VAR)++;
#define LOOP10(VAR, STMT) LOOP9(VAR, STMT); (STMT); (VAR)++;
#define LOOP11(VAR, STMT) LOOP10(VAR, STMT); (STMT); (VAR)++;
#define LOOP12(VAR, STMT) LOOP11(VAR, STMT); (STMT); (VAR)++;
#define LOOP13(VAR, STMT) LOOP12(VAR, STMT); (STMT); (VAR)++;
#define LOOP14(VAR, STMT) LOOP13(VAR, STMT); (STMT); (VAR)++;
#define LOOP15(VAR, STMT) LOOP14(VAR, STMT); (STMT); (VAR)++;
#define LOOP16(VAR, STMT) LOOP15(VAR, STMT); (STMT); (VAR)++;
#define LOOP(N, VAR, STMT) CAT(LOOP, N)((VAR), (STMT))
#define LOOP1(VAR, STMT) STMT; (VAR)++;
#define LOOP2(VAR, STMT) LOOP1(VAR, STMT); STMT; (VAR)++;
#define LOOP3(VAR, STMT) LOOP2(VAR, STMT); STMT; (VAR)++;
#define LOOP4(VAR, STMT) LOOP3(VAR, STMT); STMT; (VAR)++;
#define LOOP5(VAR, STMT) LOOP4(VAR, STMT); STMT; (VAR)++;
#define LOOP6(VAR, STMT) LOOP5(VAR, STMT); STMT; (VAR)++;
#define LOOP7(VAR, STMT) LOOP6(VAR, STMT); STMT; (VAR)++;
#define LOOP8(VAR, STMT) LOOP7(VAR, STMT); STMT; (VAR)++;
#define LOOP9(VAR, STMT) LOOP8(VAR, STMT); STMT; (VAR)++;
#define LOOP10(VAR, STMT) LOOP9(VAR, STMT); STMT; (VAR)++;
#define LOOP11(VAR, STMT) LOOP10(VAR, STMT); STMT; (VAR)++;
#define LOOP12(VAR, STMT) LOOP11(VAR, STMT); STMT; (VAR)++;
#define LOOP13(VAR, STMT) LOOP12(VAR, STMT); STMT; (VAR)++;
#define LOOP14(VAR, STMT) LOOP13(VAR, STMT); STMT; (VAR)++;
#define LOOP15(VAR, STMT) LOOP14(VAR, STMT); STMT; (VAR)++;
#define LOOP16(VAR, STMT) LOOP15(VAR, STMT); STMT; (VAR)++;
#define LOOP(N, VAR, STMT) CAT(LOOP, N)(VAR, STMT)
#if defined(convolve_simd) || defined(Conv_Interleaved)
#if TYPE == TYPE_HALF
modules/dnn/src/opencl/mvn.cl
+6
View File
@@ -126,12 +126,18 @@ __kernel void MVN(__global const Dtype* src,
alpha = 1;
#endif
#ifdef LAYER_NORM
vec_type w = load(bnorm_weight, y), b = load(bnorm_bias, y);
#else
Dtype w = 1.f, b = 0.f;
#ifdef FUSE_BATCH_NORM
w = bnorm_weight[x % channels];
b = bnorm_bias[x % channels];
#endif
#endif // LAYER_NORM
vec_type src_vec = load(src, index) - (vec_type)mean_val;
vec_type dst_vec = src_vec * alpha;
dst_vec = dst_vec * w + (vec_type)b;
modules/dnn/src/tensorflow/tf_graph_simplifier.cpp
+18 -9
View File
@@ -98,6 +98,14 @@ public:
net.mutable_node()->DeleteSubrange(idx, 1);
}
virtual inline bool isCommutativeOp(const std::string& type) const CV_OVERRIDE
{
return type == "Add" || type == "Sum" ||
type == "Mul" || type == "Prod" ||
type == "Max" || type == "Maximum" || type == "Minimum" ||
type == "Mean" || type == "SquaredDifference";
}
tensorflow::GraphDef& net;
};
@@ -282,24 +290,26 @@ public:
{
int input = addNodeToMatch("");
int relu = addNodeToMatch("Relu", input);
int maxValue = addNodeToMatch("Const");
maxValueId = addNodeToMatch("Const");
int clipValue = addNodeToMatch("Const");
int minimum = addNodeToMatch("Minimum", relu, maxValue);
int minimum = addNodeToMatch("Minimum", relu, maxValueId);
addNodeToMatch("Maximum", minimum, clipValue);
setFusedNode("Relu6", input);
}
virtual bool match(const Ptr<ImportGraphWrapper>& net, int nodeId,
std::vector<int>& matchedNodesIds,
std::vector<int>& targetNodesIds) CV_OVERRIDE
std::vector<int>& matchedNodesIds) CV_OVERRIDE
{
if (!Subgraph::match(net, nodeId, matchedNodesIds, targetNodesIds))
if (!Subgraph::match(net, nodeId, matchedNodesIds))
return false;
tensorflow::NodeDef* node = net->getNode(matchedNodesIds.front() + 1).dynamicCast<TFNodeWrapper>()->node;
tensorflow::NodeDef* node = net->getNode(matchedNodesIds[maxValueId]).dynamicCast<TFNodeWrapper>()->node;
Mat maxValue = getTensorContent(node->attr().at("value").tensor());
return maxValue.type() == CV_32FC1 && maxValue.total() == 1 && maxValue.at<float>(0) == 6;
}
private:
int maxValueId;
};
// Keras' reshape stores output shape in separate Const nodes by one value.
@@ -328,15 +338,14 @@ public:
}
virtual bool match(const Ptr<ImportGraphWrapper>& net, int nodeId,
std::vector<int>& matchedNodesIds,
std::vector<int>& targetNodesIds) CV_OVERRIDE
std::vector<int>& matchedNodesIds) CV_OVERRIDE
{
Ptr<ImportNodeWrapper> node = net->getNode(nodeId);
if (node->getNumInputs() == 0)
return false;
inpName = node->getInputName(0);
return Subgraph::match(net, nodeId, matchedNodesIds, targetNodesIds);
return Subgraph::match(net, nodeId, matchedNodesIds);
}
modules/dnn/src/tensorflow/tf_importer.cpp
+6
View File
@@ -2300,6 +2300,12 @@ void TFImporter::parseSoftmax(tensorflow::GraphDef& net, const tensorflow::NodeD
CV_CheckGT(num_inputs, 0, "");
if (hasLayerAttr(layer, "axis"))
layerParams.set("axis", getLayerAttr(layer, "axis").i());
// if tf version is 2.x, use axis -1 as default
else if(netBin.has_versions() && (int)netBin.versions().producer() >= 2)
layerParams.set("axis", -1);
// else use axis 1 as default
else
layerParams.set("axis", 1);
int id = dstNet.addLayer(name, "Softmax", layerParams);
layer_id[name] = id;
modules/dnn/test/test_backends.cpp
+11 -5
View File
@@ -47,7 +47,10 @@ public:
net.setInput(inp);
net.setPreferableBackend(backend);
net.setPreferableTarget(target);
net.enableWinograd(useWinograd);
if (target == DNN_TARGET_CPU_FP16)
net.enableWinograd(false);
Mat out = net.forward(outputLayer).clone();
check(outDefault, out, outputLayer, l1, lInf, detectionConfThresh, "First run");
@@ -105,7 +108,7 @@ TEST_P(DNNTestNetwork, ResNet_50)
{
applyTestTag(
(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB),
CV_TEST_TAG_DEBUG_LONG
CV_TEST_TAG_DEBUG_VERYLONG
);
processNet("dnn/ResNet-50-model.caffemodel", "dnn/ResNet-50-deploy.prototxt",
@@ -278,8 +281,11 @@ TEST_P(DNNTestNetwork, MobileNet_SSD_v2_TensorFlow)
TEST_P(DNNTestNetwork, SSD_VGG16)
{
applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB),
CV_TEST_TAG_DEBUG_VERYLONG);
applyTestTag(
CV_TEST_TAG_MEMORY_2GB,
CV_TEST_TAG_LONG,
CV_TEST_TAG_DEBUG_VERYLONG
);
Mat sample = imread(findDataFile("dnn/street.png"));
Mat inp = blobFromImage(sample, 1.0f, Size(300, 300), Scalar(), false);
@@ -373,7 +379,7 @@ TEST_P(DNNTestNetwork, Inception_v2_SSD_TensorFlow)
{
applyTestTag(
(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB),
CV_TEST_TAG_DEBUG_LONG
CV_TEST_TAG_DEBUG_VERYLONG
);
#if defined(INF_ENGINE_RELEASE)
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD
modules/dnn/test/test_caffe_importer.cpp
+30 -4
View File
@@ -62,6 +62,10 @@ public:
findDataFile("dnn/" + model, false));
net.setPreferableBackend(backend);
net.setPreferableTarget(target);
if (target == DNN_TARGET_CPU_FP16)
net.enableWinograd(false);
Mat img = imread(findDataFile("dnn/dog416.png"));
resize(img, img, Size(800, 600));
Mat blob = blobFromImage(img, 1.0, Size(), Scalar(102.9801, 115.9465, 122.7717), false, false);
@@ -219,6 +223,9 @@ TEST_P(Reproducibility_AlexNet, Accuracy)
net.setPreferableBackend(DNN_BACKEND_OPENCV);
net.setPreferableTarget(targetId);
if (targetId == DNN_TARGET_CPU_FP16)
net.enableWinograd(false);
Mat sample = imread(_tf("grace_hopper_227.png"));
ASSERT_TRUE(!sample.empty());
@@ -263,7 +270,11 @@ TEST(Reproducibility_FCN, Accuracy)
TEST(Reproducibility_SSD, Accuracy)
{
applyTestTag(CV_TEST_TAG_MEMORY_512MB, CV_TEST_TAG_DEBUG_LONG);
applyTestTag(
CV_TEST_TAG_MEMORY_512MB,
CV_TEST_TAG_DEBUG_VERYLONG
);
Net net;
{
const string proto = findDataFile("dnn/ssd_vgg16.prototxt");
@@ -383,6 +394,9 @@ TEST_P(Reproducibility_ResNet50, Accuracy)
net.setPreferableBackend(DNN_BACKEND_OPENCV);
net.setPreferableTarget(targetId);
if (targetId == DNN_TARGET_CPU_FP16)
net.enableWinograd(false);
float l1 = (targetId == DNN_TARGET_OPENCL_FP16 || targetId == DNN_TARGET_CPU_FP16) ? 3e-5 : 1e-5;
float lInf = (targetId == DNN_TARGET_OPENCL_FP16 || targetId == DNN_TARGET_CPU_FP16) ? 6e-3 : 1e-4;
@@ -490,7 +504,10 @@ TEST(Reproducibility_GoogLeNet_fp16, Accuracy)
// https://github.com/richzhang/colorization
TEST_P(Test_Caffe_nets, Colorization)
{
applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
applyTestTag(
target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB,
CV_TEST_TAG_DEBUG_VERYLONG
);
checkBackend();
Mat inp = blobFromNPY(_tf("colorization_inp.npy"));
@@ -503,6 +520,10 @@ TEST_P(Test_Caffe_nets, Colorization)
net.setPreferableBackend(backend);
net.setPreferableTarget(target);
// This model has bad accuracy when the FP16 and Winograd are enable at same time.
if (target == DNN_TARGET_CPU_FP16)
net.enableWinograd(false);
net.getLayer(net.getLayerId("class8_ab"))->blobs.push_back(kernel);
net.getLayer(net.getLayerId("conv8_313_rh"))->blobs.push_back(Mat(1, 313, CV_32F, 2.606));
@@ -568,10 +589,15 @@ TEST_P(Test_Caffe_nets, DenseNet_121)
{
l1 = 0.11; lInf = 0.5;
}
else if (target == DNN_TARGET_CUDA_FP16 || target == DNN_TARGET_CPU_FP16)
else if (target == DNN_TARGET_CUDA_FP16)
{
l1 = 0.04; lInf = 0.2;
}
else if (target == DNN_TARGET_CPU_FP16)
{
l1 = 0.06; lInf = 0.3;
}
normAssert(outs[0], ref, "", l1, lInf);
if (target != DNN_TARGET_MYRIAD || getInferenceEngineVPUType() != CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
expectNoFallbacksFromIE(model.getNetwork_());
@@ -758,7 +784,7 @@ TEST_P(Test_Caffe_nets, FasterRCNN_zf)
#else
(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB),
#endif
CV_TEST_TAG_DEBUG_LONG
CV_TEST_TAG_DEBUG_VERYLONG
);
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000)
// IE exception: Ngraph operation Reshape with name rpn_cls_score_reshape has dynamic output shape on 0 port, but CPU plug-in supports only static shape
modules/dnn/test/test_darknet_importer.cpp
+23 -10
View File
@@ -357,7 +357,8 @@ TEST_P(Test_Darknet_nets, YoloVoc)
#else
CV_TEST_TAG_MEMORY_1GB,
#endif
CV_TEST_TAG_LONG
CV_TEST_TAG_LONG,
CV_TEST_TAG_DEBUG_VERYLONG
);
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure
@@ -473,7 +474,8 @@ TEST_P(Test_Darknet_nets, TinyYoloVoc)
1, 6, 0.928758f, 0.651024f, 0.463539f, 0.823784f, 0.654998f); // a car
double scoreDiff = 8e-5, iouDiff = 3e-4;
if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD || target == DNN_TARGET_CPU_FP16)
bool useWinograd = true;
if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD)
{
scoreDiff = 8e-3;
iouDiff = 0.018;
@@ -483,18 +485,24 @@ TEST_P(Test_Darknet_nets, TinyYoloVoc)
scoreDiff = 0.008;
iouDiff = 0.02;
}
else if (target == DNN_TARGET_CPU_FP16)
{
useWinograd = false;
scoreDiff = 8e-3;
iouDiff = 0.018;
}
std::string config_file = "tiny-yolo-voc.cfg";
std::string weights_file = "tiny-yolo-voc.weights";
{
SCOPED_TRACE("batch size 1");
testDarknetModel(config_file, weights_file, ref.rowRange(0, 2), scoreDiff, iouDiff);
testDarknetModel(config_file, weights_file, ref.rowRange(0, 2), scoreDiff, iouDiff, 0.24, 0.4, useWinograd);
}
{
SCOPED_TRACE("batch size 2");
testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff);
testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff, 0.24, 0.4, useWinograd);
}
}
@@ -890,12 +898,12 @@ TEST_P(Test_Darknet_nets, YOLOv4_tiny)
{
SCOPED_TRACE("batch size 1");
testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff, confThreshold);
testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff, confThreshold, 0.4, false);
}
{
SCOPED_TRACE("batch size 2");
testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff, confThreshold);
testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff, confThreshold, 0.4, false);
}
#if defined(INF_ENGINE_RELEASE)
@@ -912,10 +920,10 @@ TEST_P(Test_Darknet_nets, YOLOv4_tiny)
TEST_P(Test_Darknet_nets, YOLOv4x_mish)
{
applyTestTag(
CV_TEST_TAG_LONG,
CV_TEST_TAG_MEMORY_2GB,
CV_TEST_TAG_DEBUG_VERYLONG
);
CV_TEST_TAG_MEMORY_2GB,
CV_TEST_TAG_LONG,
CV_TEST_TAG_DEBUG_VERYLONG
);
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000)
// IE exception: Ngraph operation Transpose with name permute_168 has dynamic output shape on 0 port, but CPU plug-in supports only static shape
@@ -1042,6 +1050,11 @@ TEST_P(Test_Darknet_layers, avgpool_softmax)
testDarknetLayer("avgpool_softmax");
}
TEST_P(Test_Darknet_layers, crop)
{
testDarknetLayer("crop");
}
TEST_P(Test_Darknet_layers, region)
{
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000)
modules/dnn/test/test_graph_simplifier.cpp
+142
View File
@@ -0,0 +1,142 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#include "test_precomp.hpp"
namespace opencv_test { namespace {
class Test_Graph_Simplifier : public ::testing::Test {
public:
bool required;
Test_Graph_Simplifier() : required(true) {}
void test_conformance(const std::string &basename, const std::string &expected_layer) {
test(basename + std::string("/model"), std::vector<std::string>{expected_layer}, std::string("dnn/onnx/conformance/node/"));
}
void test(const std::string &basename, const std::string &expected_layer) {
test(basename, std::vector<std::string>{expected_layer});
}
void test(const std::string &basename, const std::vector<std::string> &expected_layers, const std::string &model_path_prefix = std::string("dnn/onnx/models/")) {
std::string model_path = findDataFile(model_path_prefix + basename + std::string(".onnx"), required);
auto net = readNet(model_path);
std::vector<std::string> layers;
net.getLayerTypes(layers);
// remove Const, Identity (output layer), __NetInputLayer__ (input layer)
layers.erase(std::remove_if(layers.begin(), layers.end(), [] (const std::string l) { return l == "Const" || l == "Identity" || l == "__NetInputLayer__"; }), layers.end());
EXPECT_EQ(layers, expected_layers);
}
};
TEST_F(Test_Graph_Simplifier, GeluSubGraph) {
test("gelu", "Gelu");
test("bias_gelu", std::vector<std::string>{"Gelu", "NaryEltwise"});
}
TEST_F(Test_Graph_Simplifier, GeluApproximationSubGraph) {
test("gelu_approximation", "GeluApproximation");
}
TEST_F(Test_Graph_Simplifier, LayerNormSubGraph) {
test("layer_norm_expanded", "LayerNormalization");
test("layer_norm_expanded_with_initializers", "LayerNormalization");
}
TEST_F(Test_Graph_Simplifier, ResizeSubgraph) {
/* Test for 6 subgraphs:
- GatherCastSubgraph
- MulCastSubgraph
- UpsampleSubgraph
- ResizeSubgraph1
- ResizeSubgraph2
- ResizeSubgraph3
*/
test("upsample_unfused_torch1.2", std::vector<std::string>{"BatchNorm", "Resize"});
test("resize_nearest_unfused_opset11_torch1.3", std::vector<std::string>{"BatchNorm", "Convolution", "Resize"});
test("resize_nearest_unfused_opset11_torch1.4", std::vector<std::string>{"BatchNorm", "Convolution", "Resize"});
test("upsample_unfused_opset9_torch1.4", std::vector<std::string>{"BatchNorm", "Convolution", "Resize"});
test("two_resizes_with_shared_subgraphs", std::vector<std::string>{"NaryEltwise", "Resize"});
}
TEST_F(Test_Graph_Simplifier, SoftmaxSubgraph) {
/* Test for 3 subgraphs
- SoftMaxSubgraph
- SoftMaxSubgraph2 (conformance)
- LogSoftMaxSubgraph (conformance)
*/
test("softmax_unfused", "Softmax");
test_conformance("test_softmax_example_expanded", "Softmax");
test_conformance("test_softmax_axis_2_expanded", "Softmax");
test_conformance("test_softmax_default_axis_expanded", "Softmax");
test_conformance("test_softmax_axis_0_expanded", "Softmax");
test_conformance("test_softmax_axis_1_expanded", "Softmax");
test_conformance("test_softmax_large_number_expanded", "Softmax");
test_conformance("test_softmax_negative_axis_expanded", "Softmax");
test_conformance("test_logsoftmax_axis_2_expanded", "Softmax");
test_conformance("test_logsoftmax_example_1_expanded", "Softmax");
test_conformance("test_logsoftmax_negative_axis_expanded", "Softmax");
test_conformance("test_logsoftmax_axis_0_expanded", "Softmax");
test_conformance("test_logsoftmax_axis_1_expanded", "Softmax");
test_conformance("test_logsoftmax_large_number_expanded", "Softmax");
test_conformance("test_logsoftmax_default_axis_expanded", "Softmax");
}
TEST_F(Test_Graph_Simplifier, HardSwishSubgraph) {
test_conformance("test_hardswish_expanded", "HardSwish");
}
TEST_F(Test_Graph_Simplifier, CeluSubgraph) {
test_conformance("test_celu_expanded", "Celu");
}
TEST_F(Test_Graph_Simplifier, NormalizeSubgraph) {
/* Test for 6 subgraphs
- NormalizeSubgraph1
- NormalizeSubgraph2
- NormalizeSubgraph2_2
- NormalizeSubgraph3
- NormalizeSubgraph4
- NormalizeSubgraph5
*/
test("reduceL2_subgraph_2", "Normalize");
test("reduceL2_subgraph", "Normalize");
test("normalize_fusion", "Normalize");
}
TEST_F(Test_Graph_Simplifier, BatchNormalizationSubgraph) {
/* Test for 2 subgraphs
- BatchNormalizationSubgraph1
- BatchNormalizationSubgraph2
*/
test("frozenBatchNorm2d", "BatchNorm");
test("batch_norm_subgraph", "BatchNorm");
}
TEST_F(Test_Graph_Simplifier, ExpandSubgraph) {
test("expand_neg_batch", "Expand");
}
TEST_F(Test_Graph_Simplifier, MishSubgraph) {
/* Test for 2 subgraphs
- SoftplusSubgraph
- MishSubgraph
*/
test("mish_no_softplus", "Mish");
test("mish", "Mish");
}
TEST_F(Test_Graph_Simplifier, AttentionSubgraph) {
/* Test for 2 subgraphs
- AttentionSubgraph
- AttentionSingleHeadSubgraph
*/
test("attention", "Attention");
test("attention_single_head", "Attention");
}
}}
modules/dnn/test/test_int8_layers.cpp
+8 -2
View File
@@ -752,7 +752,10 @@ TEST_P(Test_Int8_nets, GoogLeNet)
TEST_P(Test_Int8_nets, ResNet50)
{
applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
applyTestTag(
target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB,
CV_TEST_TAG_DEBUG_VERYLONG
);
if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel())
applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
@@ -1204,7 +1207,10 @@ TEST_P(Test_Int8_nets, YoloVoc)
TEST_P(Test_Int8_nets, TinyYoloVoc)
{
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
applyTestTag(
CV_TEST_TAG_MEMORY_512MB,
CV_TEST_TAG_DEBUG_VERYLONG
);
if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel())
applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
modules/dnn/test/test_misc.cpp
+111 -12
View File
@@ -13,17 +13,83 @@
namespace opencv_test { namespace {
TEST(blobRectToImageRect, DNN_PMODE_NULL)
{
Size inputSize(50 + (rand() % 100) / 4 * 4, 50 + (rand() % 100) / 4 * 4);
Size imgSize(200 + (rand() % 100) / 4 * 4, 200 + (rand() % 100) / 4 * 4);
Rect rBlob(inputSize.width / 2 - inputSize.width / 4, inputSize.height / 2 - inputSize.height / 4, inputSize.width / 2, inputSize.height / 2);
Image2BlobParams paramNet;
paramNet.scalefactor = Scalar::all(1.f);
paramNet.size = inputSize;
paramNet.ddepth = CV_32F;
paramNet.mean = Scalar();
paramNet.swapRB = false;
paramNet.datalayout = DNN_LAYOUT_NHWC;
paramNet.paddingmode = DNN_PMODE_NULL;
Rect rOri = paramNet.blobRectToImageRect(rBlob, imgSize);
Rect rImg = Rect(rBlob.x * (float)imgSize.width / inputSize.width, rBlob.y * (float)imgSize.height / inputSize.height,
rBlob.width * (float)imgSize.width / inputSize.width, rBlob.height * (float)imgSize.height / inputSize.height);
ASSERT_EQ(rImg, rOri);
}
TEST(blobRectToImageRect, DNN_PMODE_CROP_CENTER)
{
Size inputSize(50 + (rand() % 100) / 4 * 4, 50 + (rand() % 100) / 4 * 4);
Size imgSize(200 + (rand() % 100) / 4 * 4, 200 + (rand() % 100) / 4 * 4);
Rect rBlob(inputSize.width / 2 - inputSize.width / 4, inputSize.height / 2 - inputSize.height / 4, inputSize.width / 2, inputSize.height / 2);
Image2BlobParams paramNet;
paramNet.scalefactor = Scalar::all(1.f);
paramNet.size = inputSize;
paramNet.ddepth = CV_32F;
paramNet.mean = Scalar();
paramNet.swapRB = false;
paramNet.datalayout = DNN_LAYOUT_NHWC;
paramNet.paddingmode = DNN_PMODE_CROP_CENTER;
Rect rOri = paramNet.blobRectToImageRect(rBlob, imgSize);
float resizeFactor = std::max(inputSize.width / (float)imgSize.width,
inputSize.height / (float)imgSize.height);
Rect rImg = Rect((rBlob.x + 0.5 * (imgSize.width * resizeFactor - inputSize.width)) / resizeFactor, (rBlob.y + 0.5 * (imgSize.height * resizeFactor - inputSize.height)) / resizeFactor,
rBlob.width / resizeFactor, rBlob.height / resizeFactor);
ASSERT_EQ(rImg, rOri);
}
TEST(blobRectToImageRect, DNN_PMODE_LETTERBOX)
{
Size inputSize(50 + (rand() % 100) / 4 * 4, 50 + (rand() % 100) / 4 * 4);
Size imgSize(200 + (rand() % 100) / 4 * 4, 200 + (rand() % 100) / 4 * 4);
Rect rBlob(inputSize.width / 2 - inputSize.width / 4, inputSize.height / 2 - inputSize.height / 4, inputSize.width / 2, inputSize.height / 2);
Image2BlobParams paramNet;
paramNet.scalefactor = Scalar::all(1.f);
paramNet.size = inputSize;
paramNet.ddepth = CV_32F;
paramNet.mean = Scalar();
paramNet.swapRB = false;
paramNet.datalayout = DNN_LAYOUT_NHWC;
paramNet.paddingmode = DNN_PMODE_LETTERBOX;
Rect rOri = paramNet.blobRectToImageRect(rBlob, imgSize);
float resizeFactor = std::min(inputSize.width / (float)imgSize.width,
inputSize.height / (float)imgSize.height);
int rh = int(imgSize.height * resizeFactor);
int rw = int(imgSize.width * resizeFactor);
int top = (inputSize.height - rh) / 2;
int left = (inputSize.width - rw) / 2;
Rect rImg = Rect((rBlob.x - left) / resizeFactor, (rBlob.y - top) / resizeFactor, rBlob.width / resizeFactor, rBlob.height / resizeFactor);
ASSERT_EQ(rImg, rOri);
}
TEST(blobFromImage_4ch, Regression)
{
Mat ch[4];
for(int i = 0; i < 4; i++)
ch[i] = Mat::ones(10, 10, CV_8U)*i;
for (int i = 0; i < 4; i++)
ch[i] = Mat::ones(10, 10, CV_8U) * i;
Mat img;
merge(ch, 4, img);
Mat blob = dnn::blobFromImage(img, 1., Size(), Scalar(), false, false);
for(int i = 0; i < 4; i++)
for (int i = 0; i < 4; i++)
{
ch[i] = Mat(img.rows, img.cols, CV_32F, blob.ptr(0, i));
ASSERT_DOUBLE_EQ(cvtest::norm(ch[i], cv::NORM_INF), i);
@@ -32,7 +98,7 @@ TEST(blobFromImage_4ch, Regression)
TEST(blobFromImage, allocated)
{
int size[] = {1, 3, 4, 5};
int size[] = { 1, 3, 4, 5 };
Mat img(size[2], size[3], CV_32FC(size[1]));
Mat blob(4, size, CV_32F);
void* blobData = blob.data;
@@ -66,8 +132,8 @@ TEST(imagesFromBlob, Regression)
TEST(blobFromImageWithParams_4ch, NHWC_scalar_scale)
{
Mat img(10, 10, CV_8UC4, cv::Scalar(0,1,2,3));
std::vector<double> factorVec = {0.1, 0.2, 0.3, 0.4};
Mat img(10, 10, CV_8UC4, cv::Scalar(0, 1, 2, 3));
std::vector<double> factorVec = { 0.1, 0.2, 0.3, 0.4 };
Scalar scalefactor(factorVec[0], factorVec[1], factorVec[2], factorVec[3]);
@@ -77,7 +143,7 @@ TEST(blobFromImageWithParams_4ch, NHWC_scalar_scale)
Mat blob = dnn::blobFromImageWithParams(img, param); // [1, 10, 10, 4]
float* blobPtr = blob.ptr<float>(0);
std::vector<float> targetVec = {(float )factorVec[0] * 0, (float )factorVec[1] * 1, (float )factorVec[2] * 2, (float )factorVec[3] * 3}; // Target Value.
std::vector<float> targetVec = { (float)factorVec[0] * 0, (float)factorVec[1] * 1, (float)factorVec[2] * 2, (float)factorVec[3] * 3 }; // Target Value.
for (int hi = 0; hi < 10; hi++)
{
for (int wi = 0; wi < 10; wi++)
@@ -93,18 +159,51 @@ TEST(blobFromImageWithParams_4ch, NHWC_scalar_scale)
}
}
TEST(blobFromImageWithParams_CustomPadding, letter_box)
{
Mat img(40, 20, CV_8UC4, Scalar(0, 1, 2, 3));
// Custom padding value that you have added
Scalar customPaddingValue(5, 6, 7, 8); // Example padding value
Size targetSize(20, 20);
Mat targetImg = img.clone();
cv::copyMakeBorder(
targetImg, targetImg, 0, 0,
targetSize.width / 2,
targetSize.width / 2,
BORDER_CONSTANT,
customPaddingValue);
// Set up Image2BlobParams with your new functionality
Image2BlobParams param;
param.size = targetSize;
param.paddingmode = DNN_PMODE_LETTERBOX;
param.borderValue = customPaddingValue; // Use your new feature here
// Create blob with custom padding
Mat blob = dnn::blobFromImageWithParams(img, param);
// Create target blob for comparison
Mat targetBlob = dnn::blobFromImage(targetImg, 1.0, targetSize);
EXPECT_EQ(0, cvtest::norm(targetBlob, blob, NORM_INF));
}
TEST(blobFromImageWithParams_4ch, letter_box)
{
Mat img(40, 20, CV_8UC4, cv::Scalar(0,1,2,3));
Mat img(40, 20, CV_8UC4, cv::Scalar(0, 1, 2, 3));
// Construct target mat.
Mat targetCh[4];
// The letterbox will add zero at the left and right of output blob.
// After the letterbox, every row data would have same value showing as valVec.
std::vector<uint8_t> valVec = {0,0,0,0,0, 1,1,1,1,1,1,1,1,1,1, 0,0,0,0,0};
std::vector<uint8_t> valVec = { 0,0,0,0,0, 1,1,1,1,1,1,1,1,1,1, 0,0,0,0,0 };
Mat rowM(1, 20, CV_8UC1, valVec.data());
for(int i = 0; i < 4; i++)
for (int i = 0; i < 4; i++)
{
targetCh[i] = rowM * i;
}
@@ -130,7 +229,7 @@ TEST(blobFromImagesWithParams_4ch, multi_image)
param.scalefactor = scalefactor;
param.datalayout = DNN_LAYOUT_NHWC;
Mat blobs = blobFromImagesWithParams(std::vector<Mat> { img, 2*img }, param);
Mat blobs = blobFromImagesWithParams(std::vector<Mat> { img, 2 * img }, param);
vector<Range> ranges;
ranges.push_back(Range(0, 1));
ranges.push_back(Range(0, blobs.size[1]));
@@ -140,7 +239,7 @@ TEST(blobFromImagesWithParams_4ch, multi_image)
ranges[0] = Range(1, 2);
Mat blob1 = blobs(ranges);
EXPECT_EQ(0, cvtest::norm(2*blob0, blob1, NORM_INF));
EXPECT_EQ(0, cvtest::norm(2 * blob0, blob1, NORM_INF));
}
TEST(readNet, Regression)
modules/dnn/test/test_model.cpp
+14 -3
View File
@@ -40,6 +40,8 @@ public:
model.setPreferableTarget(target);
model.setNmsAcrossClasses(nmsAcrossClasses);
if (target == DNN_TARGET_CPU_FP16)
model.enableWinograd(false);
std::vector<int> classIds;
std::vector<float> confidences;
@@ -286,8 +288,9 @@ TEST_P(Test_Model, Classify)
TEST_P(Test_Model, DetectRegion)
{
applyTestTag(
CV_TEST_TAG_MEMORY_2GB,
CV_TEST_TAG_LONG,
CV_TEST_TAG_MEMORY_2GB
CV_TEST_TAG_DEBUG_VERYLONG
);
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2022010000)
@@ -346,8 +349,9 @@ TEST_P(Test_Model, DetectRegion)
TEST_P(Test_Model, DetectRegionWithNmsAcrossClasses)
{
applyTestTag(
CV_TEST_TAG_MEMORY_2GB,
CV_TEST_TAG_LONG,
CV_TEST_TAG_MEMORY_2GB
CV_TEST_TAG_DEBUG_VERYLONG
);
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2022010000)
@@ -406,6 +410,8 @@ TEST_P(Test_Model, DetectRegionWithNmsAcrossClasses)
TEST_P(Test_Model, DetectionOutput)
{
applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2022010000)
// Check 'backward_compatible_check || in_out_elements_equal' failed at core/src/op/reshape.cpp:427:
// While validating node 'v1::Reshape bbox_pred_reshape (ave_bbox_pred_rois[0]:f32{1,8,1,1}, Constant_388[0]:i64{4}) -> (f32{?,?,?,?})' with friendly_name 'bbox_pred_reshape':
@@ -629,7 +635,8 @@ TEST_P(Test_Model, Detection_normalized)
TEST_P(Test_Model, Segmentation)
{
applyTestTag(
CV_TEST_TAG_MEMORY_2GB
CV_TEST_TAG_MEMORY_2GB,
CV_TEST_TAG_DEBUG_VERYLONG
);
float norm = 0;
@@ -744,6 +751,8 @@ TEST_P(Test_Model, TextRecognitionWithCTCPrefixBeamSearch)
TEST_P(Test_Model, TextDetectionByDB)
{
applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);
if (target == DNN_TARGET_OPENCL_FP16)
applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
if (target == DNN_TARGET_CPU_FP16)
@@ -786,6 +795,8 @@ TEST_P(Test_Model, TextDetectionByDB)
TEST_P(Test_Model, TextDetectionByEAST)
{
applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);
std::string imgPath = _tf("text_det_test2.jpg");
std::string weightPath = _tf("frozen_east_text_detection.pb", false);
modules/dnn/test/test_onnx_conformance.cpp
+19
View File
@@ -339,6 +339,25 @@ static const TestCase testConformanceConfig[] = {
{"test_isinf_negative", 1, 1},
{"test_isinf_positive", 1, 1},
{"test_isnan", 1, 1},
{"test_layer_normalization_2d_axis0", 3, 1},
{"test_layer_normalization_2d_axis1", 3, 1},
{"test_layer_normalization_2d_axis_negative_1", 3, 1},
{"test_layer_normalization_2d_axis_negative_2", 3, 1},
{"test_layer_normalization_3d_axis0_epsilon", 3, 1},
{"test_layer_normalization_3d_axis1_epsilon", 3, 1},
{"test_layer_normalization_3d_axis2_epsilon", 3, 1},
{"test_layer_normalization_3d_axis_negative_1_epsilon", 3, 1},
{"test_layer_normalization_3d_axis_negative_2_epsilon", 3, 1},
{"test_layer_normalization_3d_axis_negative_3_epsilon", 3, 1},
{"test_layer_normalization_4d_axis0", 3, 1},
{"test_layer_normalization_4d_axis1", 3, 1},
{"test_layer_normalization_4d_axis2", 3, 1},
{"test_layer_normalization_4d_axis3", 3, 1},
{"test_layer_normalization_4d_axis_negative_1", 3, 1},
{"test_layer_normalization_4d_axis_negative_2", 3, 1},
{"test_layer_normalization_4d_axis_negative_3", 3, 1},
{"test_layer_normalization_4d_axis_negative_4", 3, 1},
{"test_layer_normalization_default_axis", 3, 1},
{"test_leakyrelu", 1, 1},
{"test_leakyrelu_default", 1, 1},
{"test_leakyrelu_example", 1, 1},
modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp
+38
View File
@@ -792,6 +792,44 @@ CASE(test_isinf_positive)
// no filter
CASE(test_isnan)
// no filter
CASE(test_layer_normalization_2d_axis0)
// no filter
CASE(test_layer_normalization_2d_axis1)
// no filter
CASE(test_layer_normalization_2d_axis_negative_1)
// no filter
CASE(test_layer_normalization_2d_axis_negative_2)
// no filter
CASE(test_layer_normalization_3d_axis0_epsilon)
// no filter
CASE(test_layer_normalization_3d_axis1_epsilon)
// no filter
CASE(test_layer_normalization_3d_axis2_epsilon)
// no filter
CASE(test_layer_normalization_3d_axis_negative_1_epsilon)
// no filter
CASE(test_layer_normalization_3d_axis_negative_2_epsilon)
// no filter
CASE(test_layer_normalization_3d_axis_negative_3_epsilon)
// no filter
CASE(test_layer_normalization_4d_axis0)
// no filter
CASE(test_layer_normalization_4d_axis1)
// no filter
CASE(test_layer_normalization_4d_axis2)
// no filter
CASE(test_layer_normalization_4d_axis3)
// no filter
CASE(test_layer_normalization_4d_axis_negative_1)
// no filter
CASE(test_layer_normalization_4d_axis_negative_2)
// no filter
CASE(test_layer_normalization_4d_axis_negative_3)
// no filter
CASE(test_layer_normalization_4d_axis_negative_4)
// no filter
CASE(test_layer_normalization_default_axis)
// no filter
CASE(test_leakyrelu)
// no filter
CASE(test_leakyrelu_default)
modules/dnn/test/test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp
-2
View File
@@ -43,7 +43,6 @@
"test_castlike_STRING_to_FLOAT_expanded",
"test_concat_1d_axis_negative_1",
"test_div_uint8", // output type mismatch
"test_logsoftmax_default_axis",
"test_maxpool_2d_dilations",
"test_maxpool_2d_same_lower",
"test_maxpool_2d_uint8", // output type mismatch
@@ -51,7 +50,6 @@
"test_maxpool_with_argmax_2d_precomputed_strides",
"test_maxunpool_export_with_output_shape", // exception during net.forward() call
"test_mul_uint8", // output type mismatch
"test_softmax_default_axis",
"test_sub_bcast",
"test_sub_uint8", // output type mismatch
"test_upsample_nearest",
modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp
-2
View File
@@ -159,8 +159,6 @@
"test_if",
"test_if_opt",
"test_if_seq",
"test_instancenorm_epsilon",
"test_instancenorm_example",
"test_isinf",
"test_isinf_negative",
"test_isinf_positive",
modules/dnn/test/test_onnx_importer.cpp
+345 -92
View File
@@ -12,6 +12,15 @@
#include <numeric>
namespace opencv_test { namespace {
void yoloPostProcessing(
std::vector<Mat>& outs,
std::vector<int>& keep_classIds,
std::vector<float>& keep_confidences,
std::vector<Rect2d>& keep_boxes,
float conf_threshold,
float iou_threshold,
const std::string& test_name);
template<typename TString>
static std::string _tf(TString filename, bool required = true)
{
@@ -54,7 +63,8 @@ public:
void testONNXModels(const String& basename, const Extension ext = npy,
double l1 = 0, double lInf = 0, const bool useSoftmax = false,
bool checkNoFallbacks = true, int numInps = 1)
bool checkNoFallbacks = true, int numInps = 1,
bool testShapes = true, bool useWinograd = true)
{
String onnxmodel = _tf("models/" + basename + ".onnx", required);
std::vector<Mat> inps(numInps);
@@ -76,10 +86,12 @@ public:
Net net = readNetFromONNX(onnxmodel);
ASSERT_FALSE(net.empty());
testInputShapes(net, inps);
if (testShapes)
testInputShapes(net, inps);
net.setPreferableBackend(backend);
net.setPreferableTarget(target);
net.enableWinograd(useWinograd);
std::vector<String> inputNames;
for (int i = 0; i < numInps; ++i)
@@ -122,9 +134,6 @@ public:
TEST_P(Test_ONNX_layers, InstanceNorm)
{
if(backend == DNN_BACKEND_CUDA)
applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA); /* MVN is not supported */
if (target == DNN_TARGET_MYRIAD)
testONNXModels("instancenorm", npy, 0, 0, false, false);
else
@@ -254,6 +263,10 @@ TEST_P(Test_ONNX_layers, Gather_shared_indices) {
testONNXModels("gather_shared_indices", npy, 0, 0, false, false, 1);
}
TEST_P(Test_ONNX_layers, Two_resizes_with_shared_subgraphs) {
testONNXModels("two_resizes_with_shared_subgraphs", npy, 0, 0, false, false, 3, /*testShapes*/ false);
}
TEST_P(Test_ONNX_layers, Convolution3D)
{
if (backend == DNN_BACKEND_CUDA && target == DNN_TARGET_CUDA_FP16)
@@ -1482,7 +1495,8 @@ TEST_P(Test_ONNX_layers, Einsum_5D)
testONNXModels("einsum_5d", npy, 0, 0, false, false, 2);
}
TEST_P(Test_ONNX_layers, Einsum_InnerProduct)
// https://github.com/opencv/opencv/issues/24883
TEST_P(Test_ONNX_layers, DISABLED_Einsum_InnerProduct)
{
testONNXModels("einsum_inner", npy, 0, 0, false, false, 2);
}
@@ -1507,6 +1521,10 @@ TEST_P(Test_ONNX_layers, Einsum_transpose)
testONNXModels("einsum_transpose", npy, 0, 0, false, false, 1);
}
TEST_P(Test_ONNX_layers, Einsum_const_inputs) {
testONNXModels("einsum_const_inputs", npy, 0, 0, false, false, 1);
}
TEST_P(Test_ONNX_layers, Pad2d_Unfused)
{
testONNXModels("ReflectionPad2d");
@@ -1932,7 +1950,9 @@ TEST_P(Test_ONNX_layers, ConvResizePool1d)
#endif
}
#endif
testONNXModels("conv_resize_pool_1d");
const double lInf = (target == DNN_TARGET_CPU_FP16) ? 0.024 : default_lInf;
testONNXModels("conv_resize_pool_1d", npy, default_l1, lInf);
}
TEST_P(Test_ONNX_layers, DepthWiseAdd)
@@ -2133,6 +2153,7 @@ TEST_P(Test_ONNX_nets, Alexnet)
net.setPreferableBackend(backend);
net.setPreferableTarget(target);
net.enableWinograd(false);
Mat inp = imread(_tf("../grace_hopper_227.png"));
Mat ref = blobFromNPY(_tf("../caffe_alexnet_prob.npy"));
@@ -2205,6 +2226,9 @@ TEST_P(Test_ONNX_nets, Googlenet)
net.setPreferableBackend(backend);
net.setPreferableTarget(target);
if (target == DNN_TARGET_CPU_FP16)
net.enableWinograd(false);
std::vector<Mat> images;
images.push_back( imread(_tf("../googlenet_0.png")) );
images.push_back( imread(_tf("../googlenet_1.png")) );
@@ -2349,7 +2373,7 @@ TEST_P(Test_ONNX_nets, TinyYolov2)
}
#endif
testONNXModels("tiny_yolo2", pb, l1, lInf);
testONNXModels("tiny_yolo2", pb, l1, lInf, false, true, 1, true, false);
}
TEST_P(Test_ONNX_nets, CNN_MNIST)
@@ -2377,7 +2401,7 @@ TEST_P(Test_ONNX_nets, LResNet100E_IR)
#else
(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB),
#endif
CV_TEST_TAG_DEBUG_LONG
CV_TEST_TAG_DEBUG_VERYLONG
);
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019)
{
@@ -2394,6 +2418,7 @@ TEST_P(Test_ONNX_nets, LResNet100E_IR)
double l1 = default_l1, lInf = default_lInf;
// output range: [-3; 3]
bool useWinograd = true;
if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
{
l1 = 0.009;
@@ -2409,7 +2434,14 @@ TEST_P(Test_ONNX_nets, LResNet100E_IR)
l1 = 0.009;
lInf = 0.04;
}
testONNXModels("LResNet100E_IR", pb, l1, lInf);
else if (target == DNN_TARGET_CPU_FP16)
{
useWinograd = false;
l1 = 0.009;
lInf = 0.035;
}
testONNXModels("LResNet100E_IR", pb, l1, lInf, false, true, 1, true, useWinograd);
}
TEST_P(Test_ONNX_nets, Emotion_ferplus)
@@ -2424,7 +2456,7 @@ TEST_P(Test_ONNX_nets, Emotion_ferplus)
double l1 = default_l1;
double lInf = default_lInf;
bool useWinograd = true;
// Output values are in range [-2.011, 2.111]
if ((backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) || (target == DNN_TARGET_CUDA_FP16))
l1 = 0.007;
@@ -2437,6 +2469,11 @@ TEST_P(Test_ONNX_nets, Emotion_ferplus)
l1 = 2.4e-4;
lInf = 6e-4;
}
else if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_CPU_FP16)
{
useWinograd = false;
l1 = 0.007;
}
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2020040000)
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
{
@@ -2444,7 +2481,7 @@ TEST_P(Test_ONNX_nets, Emotion_ferplus)
}
#endif
testONNXModels("emotion_ferplus", pb, l1, lInf);
testONNXModels("emotion_ferplus", pb, l1, lInf, false, true, 1, true, useWinograd);
}
TEST_P(Test_ONNX_nets, Inception_v2)
@@ -2588,30 +2625,16 @@ TEST_P(Test_ONNX_layers, CumSum)
testONNXModels("cumsum_3d_dim_2");
}
// This test is mainly to test:
// 1. identity node with constant input
// 2. limited support to range operator (all inputs are constant)
// 3. parseExpand with multiple broadcast axes
// 4. 1D mat dimension issue with the output of range operator
TEST_P(Test_ONNX_layers, YOLOv7)
static void testYOLO(const std::string& weightPath, const std::vector<int>& refClassIds,
const std::vector<float>& refScores, const std::vector<Rect2d>& refBoxes,
Image2BlobParams imgParams, float conf_threshold = 0.3, float iou_threshold = 0.5,
double scores_diff = 1e-5, double boxes_iou_diff = 1e-4, const std::string test_name = "")
{
std::string weightPath = _tf("models/yolov7_not_simplified.onnx", false);
std::string imgPath = _tf("../dog_orig_size.png");
Size targetSize{640, 640};
float conf_threshold = 0.3;
float iou_threshold = 0.5;
// Reference, which is collected with input size of 640x640
std::vector<int> refClassIds{1, 16, 7};
std::vector<float> refScores{0.9614331f, 0.9589417f, 0.8679074f};
// [x1, y1, x2, y2] x 3
std::vector<Rect2d> refBoxes{Rect2d(105.973236f, 150.16716f, 472.59012f, 466.48834f),
Rect2d(109.97953f, 246.17862f, 259.83676f, 600.76624f),
Rect2d(385.96185f, 83.02809f, 576.07355f, 189.82793f)};
Mat img = imread(imgPath);
Mat inp = blobFromImage(img, 1/255.0, targetSize, Scalar(0, 0, 0), true, false);
Mat inp = blobFromImageWithParams(img, imgParams);
Net net = readNet(weightPath);
@@ -2619,57 +2642,253 @@ TEST_P(Test_ONNX_layers, YOLOv7)
std::vector<Mat> outs;
net.forward(outs, net.getUnconnectedOutLayersNames());
Mat preds = outs[3].reshape(1, outs[3].size[1]); // [1, 25200, 85]
// Retrieve
std::vector<int> keep_classIds;
std::vector<float> keep_confidences;
std::vector<Rect2d> keep_boxes;
yoloPostProcessing(outs, keep_classIds, keep_confidences, keep_boxes, conf_threshold, iou_threshold, test_name);
normAssertDetections(
refClassIds, refScores, refBoxes,
keep_classIds, keep_confidences, keep_boxes,
"", 0.0, scores_diff, boxes_iou_diff);
}
void yoloPostProcessing(
std::vector<Mat>& outs,
std::vector<int>& keep_classIds,
std::vector<float>& keep_confidences,
std::vector<Rect2d>& keep_boxes,
float conf_threshold,
float iou_threshold,
const std::string& test_name
){
// Retrieve
std::vector<int> classIds;
std::vector<float> confidences;
std::vector<Rect2d> boxes;
if (test_name == "yolov8"){
cv::transposeND(outs[0], {0, 2, 1}, outs[0]);
}
// each row is [cx, cy, w, h, conf_obj, conf_class1, ..., conf_class80]
for (int i = 0; i < preds.rows; ++i)
{
// filter out non objects
float obj_conf = preds.row(i).at<float>(4);
if (obj_conf < conf_threshold)
continue;
for (auto preds : outs){
// get class id and conf
Mat scores = preds.row(i).colRange(5, preds.cols);
double conf;
Point maxLoc;
minMaxLoc(scores, 0, &conf, 0, &maxLoc);
conf *= obj_conf;
if (conf < conf_threshold)
continue;
preds = preds.reshape(1, preds.size[1]); // [1, 8400, 85] -> [8400, 85]
// get bbox coords
float* det = preds.ptr<float>(i);
double cx = det[0];
double cy = det[1];
double w = det[2];
double h = det[3];
// [x1, y1, x2, y2]
boxes.push_back(Rect2d(cx - 0.5 * w, cy - 0.5 * h,
cx + 0.5 * w, cy + 0.5 * h));
classIds.push_back(maxLoc.x);
confidences.push_back(conf);
for (int i = 0; i < preds.rows; ++i)
{
// filter out non objects
float obj_conf = (test_name != "yolov8") ? preds.at<float>(i, 4) : 1.0f;
if (obj_conf < conf_threshold)
continue;
Mat scores = preds.row(i).colRange((test_name != "yolov8") ? 5 : 4, preds.cols);
double conf;
Point maxLoc;
minMaxLoc(scores, 0, &conf, 0, &maxLoc);
conf = (test_name != "yolov8") ? conf * obj_conf : conf;
if (conf < conf_threshold)
continue;
// get bbox coords
float* det = preds.ptr<float>(i);
double cx = det[0];
double cy = det[1];
double w = det[2];
double h = det[3];
// [x1, y1, x2, y2]
boxes.push_back(Rect2d(cx - 0.5 * w, cy - 0.5 * h,
cx + 0.5 * w, cy + 0.5 * h));
classIds.push_back(maxLoc.x);
confidences.push_back(conf);
}
}
// NMS
std::vector<int> keep_idx;
NMSBoxes(boxes, confidences, conf_threshold, iou_threshold, keep_idx);
std::vector<int> keep_classIds;
std::vector<float> keep_confidences;
std::vector<Rect2d> keep_boxes;
for (auto i : keep_idx)
{
keep_classIds.push_back(classIds[i]);
keep_confidences.push_back(confidences[i]);
keep_boxes.push_back(boxes[i]);
}
}
normAssertDetections(refClassIds, refScores, refBoxes, keep_classIds, keep_confidences, keep_boxes);
TEST_P(Test_ONNX_nets, YOLOX)
{
applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);
std::string weightPath = _tf("models/yolox_s_inf_decoder.onnx", false);
Size targetSize{640, 640};
float conf_threshold = 0.50;
float iou_threshold = 0.50;
std::vector<int> refClassIds{1, 16, 7};
std::vector<float> refScores{0.9649f, 0.9163f, 0.6879f};
std::vector<Rect2d> refBoxes{
Rect2d(105.5384, 179.4100, 470.6339, 428.5553),
Rect2d(111.4482, 263.4098, 258.7438, 526.1140),
Rect2d(389.1421, 143.9286, 577.9495, 222.0294)
};
Image2BlobParams imgParams(
Scalar::all(1),
targetSize,
Scalar::all(0),
true,
CV_32F,
DNN_LAYOUT_NCHW,
DNN_PMODE_LETTERBOX,
Scalar::all(114)
);
testYOLO(
weightPath, refClassIds, refScores, refBoxes,
imgParams, conf_threshold, iou_threshold,
1.0e-4, 1.0e-4);
}
TEST_P(Test_ONNX_nets, YOLOv8)
{
std::string weightPath = _tf("models/yolov8n.onnx", false);
Size targetSize{640, 640};
float conf_threshold = 0.25;
float iou_threshold = 0.50;
std::vector<int> refClassIds{16, 1, 2};
std::vector<float> refScores{0.9332f, 0.8959f, 0.6157f};
// [x1, y1, x2, y2]
std::vector<Rect2d> refBoxes{
Rect2d(108.8965, 261.9094, 257.1633, 530.3049),
Rect2d(110.4020, 192.9843, 473.4418, 429.5965),
Rect2d(389.1603, 143.2506, 577.3542, 223.0615),
};
Image2BlobParams imgParams(
Scalar::all(1/255.0),
targetSize,
Scalar::all(0),
true,
CV_32F,
DNN_LAYOUT_NCHW,
DNN_PMODE_LETTERBOX,
Scalar::all(114)
);
testYOLO(
weightPath, refClassIds, refScores, refBoxes,
imgParams, conf_threshold, iou_threshold,
1.0e-4, 1.0e-4, "yolov8");
}
// This test is mainly to test:
// 1. identity node with constant input
// 2. limited support to range operator (all inputs are constant)
// 3. parseExpand with multiple broadcast axes
// 4. 1D mat dimension issue with the output of range operator
TEST_P(Test_ONNX_nets, YOLOv7)
{
applyTestTag(
CV_TEST_TAG_MEMORY_2GB,
CV_TEST_TAG_DEBUG_VERYLONG
);
std::string weightPath = _tf("models/yolov7_not_simplified.onnx", false);
// Reference, which is collected with input size of 640x640
std::vector<int> refClassIds{1, 16, 7};
std::vector<float> refScores{0.9614331f, 0.9589417f, 0.8679074f};
// [x1, y1, x2, y2] x 3
std::vector<Rect2d> refBoxes{Rect2d(105.973236f, 150.16716f, 472.59012f, 466.48834f),
Rect2d(109.97953f, 246.17862f, 259.83676f, 600.76624f),
Rect2d(385.96185f, 83.02809f, 576.07355f, 189.82793f)};
Size targetSize{640, 640};
Image2BlobParams imgParams(
Scalar::all(1/255.0),
targetSize,
Scalar::all(0),
true,
CV_32F,
DNN_LAYOUT_NCHW,
DNN_PMODE_NULL,
Scalar::all(0)
);
testYOLO(weightPath, refClassIds, refScores, refBoxes, imgParams);
}
TEST_P(Test_ONNX_nets, YOLOv6)
{
std::string weightPath = _tf("models/yolov6n.onnx", false);
Size targetSize{640, 640};
float conf_threshold = 0.30;
float iou_threshold = 0.50;
std::vector<int> refClassIds{1, 16, 7, 1};
std::vector<float> refScores{0.95031f, 0.87123f, 0.65453f, 0.34142f};
// [x1, y1, x2, y2] x 3
std::vector<Rect2d> refBoxes{Rect2d(98.84, 177.91, 473.29, 431.19),
Rect2d(109.80, 265.50, 258.86, 531.97),
Rect2d(387.79, 141.61, 576.98, 223.52),
Rect2d(105.62, 199.24, 218.37, 389.84),
};
Image2BlobParams imgParams(
Scalar::all(1/255.0),
targetSize,
Scalar::all(0),
true,
CV_32F,
DNN_LAYOUT_NCHW,
DNN_PMODE_LETTERBOX,
Scalar::all(114)
);
testYOLO(
weightPath, refClassIds, refScores, refBoxes,
imgParams, conf_threshold, iou_threshold,
1.0e-4, 1.0e-3);
}
TEST_P(Test_ONNX_nets, YOLOv5n)
{
std::string weightPath = findDataFile("dnn/yolov5n.onnx", false);
// Reference, which is collected with input size of 640x640
std::vector<int> refClassIds{16, 2, 1};
std::vector<float> refScores{0.749053f, 0.616853f, 0.32506f};
// [x1, y1, x2, y2] x 4
std::vector<Rect2d> refBoxes{Rect2d(108.088f, 239.293f, 266.196f, 607.658f),
Rect2d(392.028f, 89.9233f, 579.152f, 190.447f),
Rect2d(120.278f, 159.76, 214.481f, 241.473f)};
Size targetSize{640, 640};
Image2BlobParams imgParams(
Scalar::all(1/255.0),
targetSize,
Scalar::all(0),
true,
CV_32F,
DNN_LAYOUT_NCHW,
DNN_PMODE_NULL,
Scalar::all(0)
);
testYOLO(weightPath, refClassIds, refScores, refBoxes, imgParams);
}
TEST_P(Test_ONNX_layers, Tile)
@@ -2677,36 +2896,6 @@ TEST_P(Test_ONNX_layers, Tile)
testONNXModels("tile", pb);
}
TEST_P(Test_ONNX_layers, LayerNorm)
{
testONNXModels("test_layer_normalization_2d_axis0", pb, 0, 0, false, true, 3);
testONNXModels("test_layer_normalization_2d_axis1", pb, 0, 0, false, true, 3);
testONNXModels("test_layer_normalization_2d_axis_negative_1", pb, 0, 0, false, true, 3);
testONNXModels("test_layer_normalization_2d_axis_negative_2", pb, 0, 0, false, true, 3);
testONNXModels("test_layer_normalization_3d_axis0_epsilon", pb, 0, 0, false, true, 3);
testONNXModels("test_layer_normalization_3d_axis1_epsilon", pb, 0, 0, false, true, 3);
testONNXModels("test_layer_normalization_3d_axis2_epsilon", pb, 0, 0, false, true, 3);
testONNXModels("test_layer_normalization_3d_axis_negative_1_epsilon", pb, 0, 0, false, true, 3);
testONNXModels("test_layer_normalization_3d_axis_negative_2_epsilon", pb, 0, 0, false, true, 3);
testONNXModels("test_layer_normalization_3d_axis_negative_3_epsilon", pb, 0, 0, false, true, 3);
testONNXModels("test_layer_normalization_4d_axis0", pb, 0, 0, false, true, 3);
testONNXModels("test_layer_normalization_4d_axis1", pb, 0, 0, false, true, 3);
testONNXModels("test_layer_normalization_4d_axis2", pb, 0, 0, false, true, 3);
testONNXModels("test_layer_normalization_4d_axis3", pb, 0, 0, false, true, 3);
testONNXModels("test_layer_normalization_4d_axis_negative_1", pb, 0, 0, false, true, 3);
testONNXModels("test_layer_normalization_4d_axis_negative_2", pb, 0, 0, false, true, 3);
testONNXModels("test_layer_normalization_4d_axis_negative_3", pb, 0, 0, false, true, 3);
testONNXModels("test_layer_normalization_4d_axis_negative_4", pb, 0, 0, false, true, 3);
testONNXModels("test_layer_normalization_default_axis", pb, 0, 0, false, true, 3);
}
// for testing graph simplification
TEST_P(Test_ONNX_layers, LayerNormExpanded)
{
testONNXModels("layer_norm_expanded");
testONNXModels("layer_norm_expanded_with_initializers");
}
TEST_P(Test_ONNX_layers, Gelu)
{
testONNXModels("gelu");
@@ -2778,6 +2967,70 @@ TEST_P(Test_ONNX_layers, Expand_shape_model4) {
testONNXModels("test_expand_shape_model4", pb, 0, 0, false, true, 1);
}
TEST_P(Test_ONNX_layers, Attention) {
testONNXModels("attention");
}
TEST_P(Test_ONNX_layers, AttentionSingleHead) {
testONNXModels("attention_single_head");
}
TEST_P(Test_ONNX_nets, ViT_B_32) {
applyTestTag(CV_TEST_TAG_LONG, CV_TEST_TAG_DEBUG_LONG);
const std::string model_path = _tf("models/vit_b_32.onnx", false);
auto net = readNet(model_path);
ASSERT_FALSE(net.empty());
net.setPreferableBackend(backend);
net.setPreferableTarget(target);
auto image = imread(_tf("../googlenet_0.png"));
auto blob = blobFromImage(image, 1.f, Size(224, 224));
auto ref = blobFromNPY(_tf("data/output_vit_b_32.npy"));
checkBackend(&blob, &ref);
net.setInput(blob);
auto out = net.forward();
double l1 = default_l1;
double lInf = default_lInf;
if (target == DNN_TARGET_CUDA_FP16)
{
l1 = 0.01;
lInf = 0.06;
}
if (target == DNN_TARGET_OPENCL_FP16)
{
l1 = 0.008;
lInf = 0.04;
}
normAssert(ref, out, "ViTB_32", l1, lInf);
}
TEST_P(Test_ONNX_nets, VitTrack) {
auto image = imread(_tf("../dog_orig_size.png"));
auto input0 = blobFromImage(image, 1.f, Size(128, 128));
auto input1 = blobFromImage(image, 1.f, Size(256, 256));
auto net = readNet(_tf("models/object_tracking_vittrack_2023sep.onnx", false));
net.setInput(input0, "template");
net.setInput(input1, "search");
std::vector<std::string> output_names{"output1", "output2", "output3"};
std::vector<Mat> outputs;
net.forward(outputs, output_names);
auto ref_output1 = blobFromNPY(_tf("data/output_object_tracking_vittrack_2023sep_0.npy"));
auto ref_output2 = blobFromNPY(_tf("data/output_object_tracking_vittrack_2023sep_1.npy"));
auto ref_output3 = blobFromNPY(_tf("data/output_object_tracking_vittrack_2023sep_2.npy"));
normAssert(ref_output1, outputs[0], "VitTrack output1");
normAssert(ref_output2, outputs[1], "VitTrack output2");
normAssert(ref_output3, outputs[2], "VitTrack output3");
}
INSTANTIATE_TEST_CASE_P(/**/, Test_ONNX_nets, dnnBackendsAndTargets());
}} // namespace
modules/dnn/test/test_tf_importer.cpp
+15 -4
View File
@@ -974,6 +974,9 @@ TEST_P(Test_TensorFlow_nets, Inception_v2_SSD)
net.setPreferableBackend(backend);
net.setPreferableTarget(target);
if (target == DNN_TARGET_CPU_FP16)
net.enableWinograd(false);
net.setInput(blob);
// Output has shape 1x1xNx7 where N - number of detections.
// An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
@@ -1279,7 +1282,7 @@ TEST_P(Test_TensorFlow_nets, EAST_text_detection)
{
applyTestTag(
(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB),
CV_TEST_TAG_DEBUG_LONG
CV_TEST_TAG_DEBUG_VERYLONG
);
#if defined(INF_ENGINE_RELEASE)
@@ -1307,6 +1310,8 @@ TEST_P(Test_TensorFlow_nets, EAST_text_detection)
net.setPreferableBackend(backend);
net.setPreferableTarget(target);
if (target == DNN_TARGET_CPU_FP16)
net.enableWinograd(false);
Mat img = imread(imgPath);
Mat inp = blobFromImage(img, 1.0, Size(), Scalar(123.68, 116.78, 103.94), true, false);
@@ -1341,8 +1346,9 @@ TEST_P(Test_TensorFlow_nets, EAST_text_detection)
}
else if (target == DNN_TARGET_CPU_FP16)
{
lInf_scores = 0.1;
l1_geometry = 0.28; lInf_geometry = 5.94;
lInf_scores = 0.17;
l1_geometry = 0.28;
lInf_geometry = 5.94;
}
else
{
@@ -1798,7 +1804,10 @@ TEST_P(Test_TensorFlow_nets, Mask_RCNN)
if (target == DNN_TARGET_CUDA_FP16)
applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA_FP16);
applyTestTag(CV_TEST_TAG_MEMORY_1GB, CV_TEST_TAG_DEBUG_VERYLONG);
applyTestTag(
CV_TEST_TAG_MEMORY_2GB,
CV_TEST_TAG_DEBUG_VERYLONG
);
Mat img = imread(findDataFile("dnn/street.png"));
std::string proto = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pbtxt");
std::string model = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pb", false);
@@ -1810,6 +1819,8 @@ TEST_P(Test_TensorFlow_nets, Mask_RCNN)
net.setPreferableBackend(backend);
net.setPreferableTarget(target);
if (target == DNN_TARGET_CPU_FP16)
net.enableWinograd(false);
net.setInput(blob);

Some files were not shown because too many files have changed in this diff Show More