diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index 34f6c10e01..d11af929e6 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -395,6 +395,7 @@ CV__DNN_INLINE_NS_BEGIN int output_zp = 0; bool per_channel = true; bool input_is_u8 = false; + bool float_input = false; // accept FP32 NCHW and quantize+interleave internally // blobs[0] = quantized weights, blobs[1] = fused bias, blobs[2] = output multiplier Mat weights, bias, outputMultiplier; }; @@ -408,6 +409,7 @@ CV__DNN_INLINE_NS_BEGIN int input_zp, output_zp; float input_sc, output_sc; bool per_channel; + bool float_input = false; std::vector strides, dilations, pads; int ngroups; @@ -1387,6 +1389,7 @@ CV__DNN_INLINE_NS_BEGIN float output_sc = 1.f; int output_zp = 0; bool with_relu = false; + String operation = "add"; }; class CV_EXPORTS Eltwise2Int8Layer : public Layer diff --git a/modules/dnn/src/graph_fusion_qdq.cpp b/modules/dnn/src/graph_fusion_qdq.cpp index 7ef3da8113..667a28a636 100644 --- a/modules/dnn/src/graph_fusion_qdq.cpp +++ b/modules/dnn/src/graph_fusion_qdq.cpp @@ -174,14 +174,17 @@ struct ModelFusionQDQ int8_inputs.push_back(dq->inputs[0]); } - const int eltwise_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1; - const bool eltwise_out_int8 = (eltwise_out_type == CV_8S || eltwise_out_type == CV_8U); + const int eltwise_out_zp_type = netimpl->argData(out_zp).type; + const bool eltwise_out_int8 = (eltwise_out_zp_type == CV_8S || eltwise_out_zp_type == CV_8U); if (dq_ptrs.size() == add->inputs.size() && eltwise_out_int8) { vector in_scales(2); vector in_zps(2); bool eltwise_in_int8 = true; for (int k = 0; k < 2; k++) { - int inp_type = netimpl->argData(dq_ptrs[k]->inputs[0]).type; + const int inp_type0 = netimpl->argData(dq_ptrs[k]->inputs[0]).type; + const int zp_type = (dq_ptrs[k]->inputs.size() >= 3 && netimpl->isConstArg(dq_ptrs[k]->inputs[2])) + ? netimpl->argData(dq_ptrs[k]->inputs[2]).type : -1; + const int inp_type = (inp_type0 >= 0) ? inp_type0 : zp_type; eltwise_in_int8 = eltwise_in_int8 && (inp_type == CV_8S || inp_type == CV_8U); in_scales[k] = netimpl->argTensor(dq_ptrs[k]->inputs[1]).at(0); const Mat& zp_m = netimpl->argTensor(dq_ptrs[k]->inputs[2]); @@ -204,6 +207,8 @@ struct ModelFusionQDQ ep.input_zeropoints = in_zps; ep.output_sc = out_scale_val; ep.output_zp = out_zp_val; + ep.operation = (add->op == NaryEltwiseLayer::OPERATION::PROD) + ? "mul" : "add"; Ptr eltwiseInt8 = Eltwise2Int8Layer::create(ep); eltwiseInt8->netimpl = netimpl; fused_layer_idx = add_layer_idx; @@ -290,6 +295,81 @@ struct ModelFusionQDQ } } + { + int sig_layer_idx = -1; + SigmoidLayer* sigmoid_layer = 0; + if (getQdqPatternContext(layer_ptr, ninputs, inputs, producer_of, + newprog, q_data_in, out_scale, out_zp, + sig_layer_idx, sigmoid_layer) && + sigmoid_layer->inputs.size() == 1) { + Arg sig_in = sigmoid_layer->inputs[0]; + int dq_idx = producer_of.at(sig_in.idx); + DequantizeLinearLayer* dq = getLayer(newprog, dq_idx); + + const int sig_out_zp_type = netimpl->argData(out_zp).type; + const bool sig_out_int8 = (sig_out_zp_type == CV_8S || sig_out_zp_type == CV_8U); + const int sig_in_zp_type = (dq && dq->inputs.size() >= 3 && netimpl->isConstArg(dq->inputs[2])) + ? netimpl->argData(dq->inputs[2]).type : -1; + const bool sig_in_int8 = (sig_in_zp_type == CV_8S || sig_in_zp_type == CV_8U); + + if (dq && dq->inputs.size() >= 3 && + areDqArgsConst(dq) && + sig_in_int8 && sig_out_int8 && + usecounts.at(sig_in.idx) == 1) { + const float inp_sc = netimpl->argTensor(dq->inputs[1]).at(0); + const Mat& sig_in_zp_m = netimpl->argTensor(dq->inputs[2]); + const int inp_zp = sig_in_zp_m.depth() == CV_8U + ? (int)sig_in_zp_m.at(0) + : (int)sig_in_zp_m.at(0); + const float out_sc = netimpl->argTensor(out_scale).at(0); + const Mat& sig_out_zp_m = netimpl->argTensor(out_zp); + const int out_zp_i = sig_out_zp_m.depth() == CV_8U + ? (int)sig_out_zp_m.at(0) + : (int)sig_out_zp_m.at(0); + + if (inp_sc > 0.f && out_sc > 0.f) { + const bool isU8 = (sig_in_zp_type == CV_8U); + Mat lookUpTable(1, 256, isU8 ? CV_8U : CV_8S); + if (isU8) { + uint8_t* table = lookUpTable.ptr(); + for (int t = 0; t < 256; t++) { + float x = inp_sc * (t - inp_zp); + float y = 1.f / (1.f + std::exp(-x)); + int q = out_zp_i + cvRound(y / out_sc); + table[t] = saturate_cast(q); + } + } else { + int8_t* table = lookUpTable.ptr(); + for (int t = -128; t < 128; t++) { + float x = inp_sc * (t - inp_zp); + float y = 1.f / (1.f + std::exp(-x)); + int q = out_zp_i + cvRound(y / out_sc); + table[t + 128] = saturate_cast(q); + } + } + + ActivationInt8Params ap; + ap.name = sigmoid_layer->name; + ap.activationType = "SigmoidInt8"; + ap.input_sc = inp_sc; + ap.input_zp = inp_zp; + ap.output_sc = out_sc; + ap.output_zp = out_zp_i; + ap.activationLUT = lookUpTable; + Ptr sigmoidInt8 = ActivationLayerInt8::create(ap); + sigmoidInt8->netimpl = netimpl; + fused_layer_idx = sig_layer_idx; + newprog[sig_layer_idx] = sigmoidInt8; + fused_inputs.assign(1, dq->inputs[0]); + removed_args.push_back(q_data_in); + removed_args.push_back(sig_in); + newprog[dq_idx] = Ptr(); + break; + } + } + } + } + QuantizeLinearLayer* ql_compound = dynamic_cast(layer_ptr); if (ql_compound && ninputs == 3) { Arg q_inp = inputs[0]; @@ -390,6 +470,8 @@ struct ModelFusionQDQ ep2.output_sc = out_sc2; ep2.output_zp = out_zp_val2; ep2.with_relu = true; + ep2.operation = (add2->op == NaryEltwiseLayer::OPERATION::PROD) + ? "mul" : "add"; Ptr eltInt8 = Eltwise2Int8Layer::create(ep2); eltInt8->netimpl = netimpl; @@ -490,10 +572,14 @@ struct ModelFusionQDQ // SIMD iteration. When ngroups>1 and Kgngroups, 1); + const int inChannelsPerGroup = (w_q.dims >= 2) ? w_q.size[1] : 1; + const bool isDepthwise = (conv->ngroups == outCn) && (inChannelsPerGroup == 1) && (outCn > 1); const bool kernelSupportsGrouping = - conv->ngroups <= 1 || outChannelsPerGroup >= 8; + conv->ngroups <= 1 || outChannelsPerGroup >= 8 || isDepthwise; if (all_wzp_zero && symmetric_pads && kernelSupportsGrouping) { Mat bias = Mat::zeros(1, outCn, CV_32S); @@ -531,8 +617,9 @@ struct ModelFusionQDQ Mat bq = netimpl->argTensor(dq_b->inputs[0]); if (bq.empty() || bq.total() != (size_t)outCn || bq.depth() != CV_32S) biasOk = false; - else + else { bias = bq.reshape(1, 1); + } } } } @@ -542,7 +629,7 @@ struct ModelFusionQDQ const bool inputIsU8 = (netimpl->argData(dq_x->inputs[0]).type == CV_8U) || (x_zp_m.depth() == CV_8U); - const int inp_zp_kernel = inputIsU8 ? (inp_zp - 128) : inp_zp; + const int inp_zp_kernel = inputIsU8 ? 0 : inp_zp; Mat weights_2d = w_q.reshape(1, outCn); Mat biasFused(1, outCn, CV_32S); Mat outputMultiplier(1, outCn, CV_32F); @@ -950,55 +1037,47 @@ struct ModelFusionQDQ } } + // Fuse DQ -> MaxPool -> QL into Pool2Int8 when scales/zps match. + { int pool_layer_idx = -1; - PoolingLayer* pool = 0; - if (getQdqPatternContext(layer_ptr, ninputs, inputs, producer_of, + MaxPoolLayer* maxpool = 0; + if (getQdqPatternContext(layer_ptr, ninputs, inputs, producer_of, newprog, q_data_in, out_scale, out_zp, - pool_layer_idx, pool) && - pool->inputs.size() == 1) { - Arg pool_in = pool->inputs[0]; + pool_layer_idx, maxpool) && + maxpool->inputs.size() == 1) { + Arg pool_in = maxpool->inputs[0]; int dq_idx = producer_of.at(pool_in.idx); DequantizeLinearLayer* dq = getLayer(newprog, dq_idx); - float inp_sc = 0.f, out_sc = 0.f; - int inp_zp = 0, out_zp_i = 0; - const int pool_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1; - const bool pool_out_int8 = (pool_out_type == CV_8S || pool_out_type == CV_8U); - const int pool_in_type = (dq && !dq->inputs.empty()) ? netimpl->argData(dq->inputs[0]).type : -1; + const int pool_out_zp_type = netimpl->argData(out_zp).type; + const bool pool_out_int8 = (pool_out_zp_type == CV_8S || pool_out_zp_type == CV_8U); + const int pool_in_type = (dq && dq->inputs.size() >= 3 && netimpl->isConstArg(dq->inputs[2])) + ? netimpl->argData(dq->inputs[2]).type + : ((dq && !dq->inputs.empty()) ? netimpl->argData(dq->inputs[0]).type : -1); const bool pool_in_int8 = (pool_in_type == CV_8S || pool_in_type == CV_8U); if (dq && dq->inputs.size() >= 3 && areDqArgsConst(dq) && pool_in_int8 && pool_out_int8 && usecounts.at(pool_in.idx) == 1) { - inp_sc = netimpl->argTensor(dq->inputs[1]).at(0); + float inp_sc = netimpl->argTensor(dq->inputs[1]).at(0); const Mat& pool_zp_m = netimpl->argTensor(dq->inputs[2]); - inp_zp = pool_zp_m.depth() == CV_8U + int inp_zp = pool_zp_m.depth() == CV_8U ? (int)pool_zp_m.at(0) : (int)pool_zp_m.at(0); - out_sc = netimpl->argTensor(out_scale).at(0); + float out_sc = netimpl->argTensor(out_scale).at(0); const Mat& pool_out_zp_m = netimpl->argTensor(out_zp); - out_zp_i = pool_out_zp_m.depth() == CV_8U + int out_zp_i = pool_out_zp_m.depth() == CV_8U ? (int)pool_out_zp_m.at(0) : (int)pool_out_zp_m.at(0); - bool isGlobalAve = pool->globalPooling; - bool isMax = !isGlobalAve; - if ((isGlobalAve && inp_sc > 0.f && out_sc > 0.f) || - (isMax && std::abs(inp_sc - out_sc) < 1e-6f && inp_zp == out_zp_i)) { + if (std::abs(inp_sc - out_sc) < 1e-6f && inp_zp == out_zp_i) { Pool2Int8Params p; - p.name = pool->name; - p.kernel_shape.assign(pool->kernel_size.begin(), pool->kernel_size.end()); - p.strides.assign(pool->strides.begin(), pool->strides.end()); - if (!pool->pads_begin.empty()) { - p.pads.reserve(pool->pads_begin.size() + pool->pads_end.size()); - for (size_t v : pool->pads_begin) p.pads.push_back((int)v); - for (size_t v : pool->pads_end) p.pads.push_back((int)v); - } - if (pool->padMode == "SAME") - p.auto_pad = AUTO_PAD_SAME_UPPER; - else if (pool->padMode == "VALID") - p.auto_pad = AUTO_PAD_VALID; - p.ceil_mode = pool->ceilMode; - p.is_global_pooling = isGlobalAve; - p.is_max_pool = isMax; + p.name = maxpool->name; + p.kernel_shape = maxpool->kernel_shape; + p.strides = maxpool->strides; + p.pads = maxpool->pads; + p.auto_pad = maxpool->auto_pad; + p.ceil_mode = maxpool->ceil_mode; + p.is_global_pooling = false; + p.is_max_pool = true; p.input_sc = inp_sc; p.input_zp = inp_zp; p.output_sc = out_sc; @@ -1015,6 +1094,7 @@ struct ModelFusionQDQ } } } + } // fuse DequantizeLinear -> DataShuffling/MaxPool -> QuantizeLinear // when DQ and Q have same scale/zp, the op can work directly on int8 data. @@ -1180,6 +1260,53 @@ struct ModelFusionQDQ modified = true; } + // Fuse input QuantizeLinear into the first Conv2Int8 (float_input=true). + for (size_t j = 0; j < newprog.size(); j++) + { + if (newprog[j].empty()) continue; + Conv2Int8Layer* conv = dynamic_cast(newprog[j].get()); + if (!conv || conv->inputs.empty() || conv->float_input) + continue; + + Arg conv_in0 = conv->inputs[0]; + + size_t ql_idx = newprog.size(); + for (size_t k = 0; k < j; k++) { + if (newprog[k].empty()) continue; + QuantizeLinearLayer* ql_cand = dynamic_cast(newprog[k].get()); + if (!ql_cand || ql_cand->outputs.empty()) + continue; + if (ql_cand->outputs[0].idx == conv_in0.idx) { + ql_idx = k; + break; + } + } + if (ql_idx >= newprog.size()) continue; + QuantizeLinearLayer* ql = dynamic_cast(newprog[ql_idx].get()); + + if (ql->inputs.size() < 2) continue; + const Arg& ql_data = ql->inputs[0]; + if (netimpl->argKind(ql_data) != DNN_ARG_INPUT) continue; + int ql_out_uc = 0; + for (size_t k = 0; k < newprog.size(); k++) { + if (newprog[k].empty()) continue; + for (const Arg& a : newprog[k]->inputs) + if (a.idx == conv_in0.idx) ql_out_uc++; + } + if (ql_out_uc != 1) continue; + if (!netimpl->isConstArg(ql->inputs[1])) continue; + if (ql->inputs.size() >= 3 && !netimpl->isConstArg(ql->inputs[2])) continue; + if (ql->inputs.size() >= 3) { + int zp_type = netimpl->argData(ql->inputs[2]).type; + if (zp_type != CV_8S && zp_type != CV_8U) continue; + } + + conv->float_input = true; + conv->inputs[0] = ql_data; + newprog[ql_idx] = Ptr(); + modified = true; + } + if (modified) { std::vector uc(nargs, 0); for (const auto& layer : newprog) { diff --git a/modules/dnn/src/int8layers/conv2_int8_kernels.simd.hpp b/modules/dnn/src/int8layers/conv2_int8_kernels.simd.hpp index a1525ed1bd..93c652aca8 100644 --- a/modules/dnn/src/int8layers/conv2_int8_kernels.simd.hpp +++ b/modules/dnn/src/int8layers/conv2_int8_kernels.simd.hpp @@ -6,6 +6,9 @@ #include "opencv2/core/hal/intrin.hpp" #include "../layers/conv2_common.hpp" +#if CV_AVX2 && (defined(__x86_64__) || defined(_M_X64)) +#include +#endif #if !defined(CV_AVXVNNI_AVAILABLE) #if (CV_TRY_AVX2 || CV_AVX2) && \ @@ -94,10 +97,16 @@ static void convInt8BlockVNNI(const void* inp_, const void* residual_, size_t outtotal = outshape.total(); if ((Kg % K0) != 0) memset(out, 0, outtotal); - int total_blocks = N * ngroups * Kblk; + int nkb = N * ngroups * Kblk; + int nthreads = std::max(1, cv::getNumThreads()); + int min_ntiles = std::max(1, (nthreads * 4 + nkb - 1) / nkb); + int TILE = std::max(SPAT_BLOCK_SIZE, + ((planeblocks_ + min_ntiles - 1) / min_ntiles + SPAT_BLOCK_SIZE - 1) & ~(SPAT_BLOCK_SIZE - 1)); + int ntiles = (planeblocks_ + TILE - 1) / TILE; + int total_tasks = nkb * ntiles; const __m256i v_xor = inputIsU8 ? _mm256_setzero_si256() : _mm256_set1_epi8((char)0x80); - parallel_for_(Range(0, total_blocks), [&](const Range& range) { + parallel_for_(Range(0, total_tasks), [&](const Range& range) { constexpr int C0 = 8, K0 = 8; constexpr int C0shift = 3; int D = D_, H = H_, W = W_; @@ -115,9 +124,11 @@ static void convInt8BlockVNNI(const void* inp_, const void* residual_, memset(zbuf, (uint8_t)inp_zp, C0); for (int t = range.start; t < range.end; t++) { - int n = t / (ngroups * Kblk); - int rem = t - n * (ngroups * Kblk); - int g = rem / Kblk; + int tile = t % ntiles; + int kt = t / ntiles; + int n = kt / (ngroups * Kblk); + int rem = kt - n * (ngroups * Kblk); + int g = rem / Kblk; int kblk = rem - g * Kblk; int k_base = g * Kg + kblk * K0; @@ -141,6 +152,12 @@ static void convInt8BlockVNNI(const void* inp_, const void* residual_, alignas(32) float multbuf[K0]; memcpy(multbuf, multiplier + k_base, K0 * sizeof(float)); + const int k0_off_dw = k_base & (K0-1); + if (cs.depthwise) { + biasbuf[k0_off_dw] = biasbuf[0]; + multbuf[k0_off_dw] = multbuf[0]; + } + int D_l = D, H_l = H, W_l = W; int Di_l = Di, Hi_l = Hi, Wi_l = Wi; int iplanesize_l = iplanesize; @@ -155,11 +172,13 @@ static void convInt8BlockVNNI(const void* inp_, const void* residual_, planeblocks_l = W_l; } - int p = 0; - for (; p < planeblocks_l; p += SPAT_BLOCK_SIZE) { - if (p + SPAT_BLOCK_SIZE > planeblocks_l) { - if (p == 0) break; - p = planeblocks_l - SPAT_BLOCK_SIZE; + int p_start = tile * TILE; + int p_end = std::min(p_start + TILE, planeblocks_l); + int p = p_start; + for (; p < p_end; p += SPAT_BLOCK_SIZE) { + if (p + SPAT_BLOCK_SIZE > p_end) { + if (p == p_start) break; + p = p_end - SPAT_BLOCK_SIZE; } Vec3i pt[SPAT_BLOCK_SIZE]; @@ -275,8 +294,9 @@ static void convInt8BlockVNNI(const void* inp_, const void* residual_, __m256 facc = _mm256_add_ps( \ _mm256_mul_ps(_mm256_cvtepi32_ps(s##j), vmult), vzp); \ if (resptr) { \ - __m256i res32 = _mm256_cvtepi8_epi32( \ - _mm_loadl_epi64((const __m128i*)(resptr + (p + (j)) * K0))); \ + __m256i res32 = inputIsU8 \ + ? _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)(resptr + (p + (j)) * K0))) \ + : _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*)(resptr + (p + (j)) * K0))); \ facc = _mm256_add_ps(facc, \ _mm256_sub_ps(_mm256_cvtepi32_ps(res32), vzp)); \ } \ @@ -287,13 +307,24 @@ static void convInt8BlockVNNI(const void* inp_, const void* residual_, __m128i packed8 = inputIsU8 ? _mm_packus_epi16(packed16, packed16) \ : _mm_packs_epi16(packed16, packed16); \ int8_t* optr = outptr + (p + (j)) * K0; \ - if (activLUT) { \ - alignas(16) int8_t tmp8[16]; \ - _mm_store_si128((__m128i*)tmp8, packed8); \ - for (int k = 0; k < K0; k++) \ - optr[k] = (int8_t)activLUT[(int)tmp8[k] + 128]; \ + alignas(16) int8_t tmp8[16]; \ + _mm_store_si128((__m128i*)tmp8, packed8); \ + if (!cs.depthwise) { \ + if (activLUT) { \ + for (int k = 0; k < K0; k++) { \ + int lut_idx = inputIsU8 ? (int)(uint8_t)tmp8[k] : ((int)tmp8[k] + 128); \ + optr[k] = (int8_t)activLUT[lut_idx]; \ + } \ + } else { \ + _mm_storel_epi64((__m128i*)optr, packed8); \ + } \ } else { \ - _mm_storel_epi64((__m128i*)optr, packed8); \ + if (activLUT) { \ + int lut_idx = inputIsU8 ? (int)(uint8_t)tmp8[k0_off_dw] : ((int)tmp8[k0_off_dw] + 128); \ + optr[k0_off_dw] = (int8_t)activLUT[lut_idx]; \ + } else { \ + optr[k0_off_dw] = tmp8[k0_off_dw]; \ + } \ } \ } CONV_VNNI_STORE(0); CONV_VNNI_STORE(1); @@ -304,7 +335,7 @@ static void convInt8BlockVNNI(const void* inp_, const void* residual_, } // Scalar tail - for (; p < planeblocks_l; p++) { + for (; p < p_end; p++) { alignas(32) int32_t acc[K0]; memcpy(acc, biasbuf, K0 * sizeof(int32_t)); @@ -326,15 +357,23 @@ static void convInt8BlockVNNI(const void* inp_, const void* residual_, for (int c0 = 0; c0 < C0; c0++) { int ival = inputIsU8 ? (int)(uint8_t)inptr[c0] : (int)(uint8_t)((uint8_t)inptr[c0] ^ 0x80u); - const int8_t* wp = wptr + c0 * K0; - for (int k0 = 0; k0 < K0; k0++) - acc[k0] += ival * (int)wp[k0]; + // VNNI-repacked weights: weight(c0,k0) = wptr[k0*4 + c0] (c0<4) + // else wptr[32 + k0*4 + (c0-4)] + if (c0 < 4) { + for (int k0 = 0; k0 < K0; k0++) + acc[k0] += ival * (int)wptr[k0 * 4 + c0]; + } else { + for (int k0 = 0; k0 < K0; k0++) + acc[k0] += ival * (int)wptr[32 + k0 * 4 + (c0 - 4)]; + } } } } int8_t* optr = outptr + p * K0; const int8_t* rptr = resptr ? resptr + p * K0 : nullptr; - for (int k0 = 0; k0 < K0; k0++) { + int k0_lo = cs.depthwise ? k0_off_dw : 0; + int k0_hi = cs.depthwise ? k0_off_dw + 1 : K0; + for (int k0 = k0_lo; k0 < k0_hi; k0++) { float val = (float)acc[k0] * multbuf[k0] + (float)out_zp; if (rptr) val += (float)((int)rptr[k0] - out_zp); int ival = cvRound(val); @@ -352,6 +391,210 @@ static void convInt8BlockVNNI(const void* inp_, const void* residual_, } }); } + +static void convInt8Block1x1(const void* inp_, const void* residual_, + void* out_, const ConvState& cs, + const void* weightsVNNI_, + const int* biasVNNI, const float* multiplier, + int inp_zp, int out_zp, + const int8_t* activLUT, + bool inputIsU8) +{ + constexpr int C0 = 8, K0 = 8; + constexpr int SPAT_BLOCK_SIZE = 8; + + const MatShape& inpshape = cs.inpshape; + const MatShape& outshape = cs.outshape; + + int N = outshape[0]; + int ndims = outshape.dims; + int K = outshape.channels(); + int C = inpshape.channels(); + int K1 = (K + K0 - 1) / K0; + int C1 = (C + C0 - 1) / C0; + + int D_ = ndims >= 6 ? outshape[ndims-4] : 1; + int H_ = ndims >= 5 ? outshape[ndims-3] : 1; + int W_ = outshape[ndims-2]; + int P = D_ * H_ * W_; + + int Di_ = ndims >= 6 ? inpshape[ndims-4] : 1; + int Hi_ = ndims >= 5 ? inpshape[ndims-3] : 1; + int Wi_ = inpshape[ndims-2]; + int Pi = Di_ * Hi_ * Wi_; + + int ngroups = cs.ngroups; + int Kg = K / ngroups; + int Cg = C / ngroups; + int Kblk = cs.wshape[1]; + int C1Max = cs.wshape[3]; + + const int8_t* inp = (const int8_t*)inp_; + const int8_t* residual = (const int8_t*)residual_; + int8_t* out = (int8_t*)out_; + const int8_t* wdata = (const int8_t*)weightsVNNI_; + + int iplanesize = Pi * C0; + int planesize = P * K0; + + if ((Kg % K0) != 0) + memset(out, 0, outshape.total()); + + constexpr int SB_TILE = 8; + int nthreads = std::max(1, cv::getNumThreads()); + int nkb = N * ngroups * Kblk; + int min_ntiles = std::max(1, (nthreads * 4 + nkb - 1) / nkb); + int TILE = std::max(SB_TILE, + ((P + min_ntiles - 1) / min_ntiles + SB_TILE - 1) & ~(SB_TILE - 1)); + int ntiles = (P + TILE - 1) / TILE; + int total_tasks = nkb * ntiles; + + const __m256i v_xor = inputIsU8 ? _mm256_setzero_si256() : _mm256_set1_epi8((char)0x80); + + parallel_for_(Range(0, total_tasks), [&](const Range& range) { + constexpr int C0 = 8, K0 = 8; + constexpr int C0shift = 3; + + for (int t = range.start; t < range.end; t++) { + int tile = t % ntiles; + int kt = t / ntiles; + int n = kt / (ngroups * Kblk); + int rem = kt - n * (ngroups * Kblk); + int g = rem / Kblk; + int kblk = rem - g * Kblk; + + int k_base = g * Kg + kblk * K0; + if (k_base >= K) continue; + + int c_start = g * Cg; + int c1_start = c_start >> C0shift; + int c00 = c_start & (C0 - 1); + int cblocks = (c00 + Cg + C0 - 1) >> C0shift; + + const int8_t* inpbaseptr = inp + (size_t)(n * C1 + c1_start) * iplanesize; + const int8_t* wbaseptr = wdata + (size_t)(g * Kblk + kblk) * C1Max * C0 * K0; + + int k1 = k_base >> C0shift; + int8_t* outptr = out + (size_t)(n * K1 + k1) * planesize; + const int8_t* resptr = residual ? residual + (size_t)(n * K1 + k1) * planesize : nullptr; + + alignas(32) int32_t biasbuf[K0]; + alignas(32) float multbuf[K0]; + memcpy(biasbuf, biasVNNI + k_base, K0 * sizeof(int32_t)); + memcpy(multbuf, multiplier + k_base, K0 * sizeof(float)); + + int p_start = tile * TILE; + int p_end = std::min(p_start + TILE, P); + int p = p_start; + + for (; p + SPAT_BLOCK_SIZE <= p_end; p += SPAT_BLOCK_SIZE) { + __m256i vbias = _mm256_load_si256((const __m256i*)biasbuf); + __m256i s0 = vbias, s1 = vbias, s2 = vbias, s3 = vbias; + __m256i s4 = vbias, s5 = vbias, s6 = vbias, s7 = vbias; + + const int8_t* inptr[SPAT_BLOCK_SIZE]; + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) + inptr[j] = inpbaseptr + (p + j) * C0; + + const int8_t* wptr = wbaseptr; + for (int c1 = 0; c1 < cblocks; c1++, wptr += C0 * K0) { + __m256i wg0 = _mm256_load_si256((const __m256i*)(wptr)); + __m256i wg1 = _mm256_load_si256((const __m256i*)(wptr + 32)); + + #define CONV_1x1_MAC(j) { \ + __m256i x0 = _mm256_xor_si256(_mm256_set1_epi32(*(const int32_t*)(inptr[(j)] )), v_xor); \ + __m256i x1 = _mm256_xor_si256(_mm256_set1_epi32(*(const int32_t*)(inptr[(j)] + 4)), v_xor); \ + s##j = _mm256_dpbusd_epi32(s##j, x0, wg0); \ + s##j = _mm256_dpbusd_epi32(s##j, x1, wg1); \ + } + CONV_1x1_MAC(0); CONV_1x1_MAC(1); + CONV_1x1_MAC(2); CONV_1x1_MAC(3); + CONV_1x1_MAC(4); CONV_1x1_MAC(5); + CONV_1x1_MAC(6); CONV_1x1_MAC(7); + #undef CONV_1x1_MAC + + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) + inptr[j] += iplanesize; + } + + __m256 vmult = _mm256_load_ps(multbuf); + __m256 vzp = _mm256_set1_ps((float)out_zp); + + #define CONV_1x1_STORE(j) { \ + __m256 facc = _mm256_add_ps( \ + _mm256_mul_ps(_mm256_cvtepi32_ps(s##j), vmult), vzp); \ + if (resptr) { \ + __m256i res32 = inputIsU8 \ + ? _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)(resptr + (p + (j)) * K0))) \ + : _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*)(resptr + (p + (j)) * K0))); \ + facc = _mm256_add_ps(facc, \ + _mm256_sub_ps(_mm256_cvtepi32_ps(res32), vzp)); \ + } \ + __m256i ival = _mm256_cvtps_epi32(facc); \ + __m128i lo = _mm256_castsi256_si128(ival); \ + __m128i hi = _mm256_extracti128_si256(ival, 1); \ + __m128i packed16 = _mm_packs_epi32(lo, hi); \ + __m128i packed8 = inputIsU8 ? _mm_packus_epi16(packed16, packed16) \ + : _mm_packs_epi16(packed16, packed16); \ + int8_t* optr = outptr + (p + (j)) * K0; \ + if (activLUT) { \ + alignas(16) int8_t tmp8[16]; \ + _mm_store_si128((__m128i*)tmp8, packed8); \ + for (int k = 0; k < K0; k++) { \ + int lut_idx = inputIsU8 ? (int)(uint8_t)tmp8[k] : ((int)tmp8[k] + 128); \ + optr[k] = (int8_t)activLUT[lut_idx]; \ + } \ + } else { \ + _mm_storel_epi64((__m128i*)optr, packed8); \ + } \ + } + CONV_1x1_STORE(0); CONV_1x1_STORE(1); + CONV_1x1_STORE(2); CONV_1x1_STORE(3); + CONV_1x1_STORE(4); CONV_1x1_STORE(5); + CONV_1x1_STORE(6); CONV_1x1_STORE(7); + #undef CONV_1x1_STORE + } + + for (; p < p_end; p++) { + alignas(32) int32_t acc[K0]; + memcpy(acc, biasbuf, K0 * sizeof(int32_t)); + + const int8_t* inptr_s = inpbaseptr + (size_t)p * C0; + const int8_t* wptr = wbaseptr; + for (int c1 = 0; c1 < cblocks; c1++, inptr_s += iplanesize, wptr += C0 * K0) { + for (int c0 = 0; c0 < C0; c0++) { + int iv = inputIsU8 ? (int)(uint8_t)inptr_s[c0] + : (int)(uint8_t)((uint8_t)inptr_s[c0] ^ 0x80u); + const int8_t* wp = wptr + c0 * K0; + for (int k0 = 0; k0 < K0; k0++) + acc[k0] += iv * (int)wp[k0]; + } + } + + int8_t* optr = outptr + (size_t)p * K0; + const int8_t* rptr = resptr ? resptr + (size_t)p * K0 : nullptr; + for (int k0 = 0; k0 < K0; k0++) { + float val = (float)acc[k0] * multbuf[k0] + (float)out_zp; + if (rptr) { + val += inputIsU8 ? (float)(int)(uint8_t)rptr[k0] - (float)out_zp + : (float)(int)rptr[k0] - (float)out_zp; + } + int iv = cvRound(val); + if (inputIsU8) { + iv = std::max(0, std::min(255, iv)); + if (activLUT) iv = (int)(uint8_t)activLUT[iv]; + ((uint8_t*)optr)[k0] = (uint8_t)iv; + } else { + iv = std::max(-128, std::min(127, iv)); + if (activLUT) iv = (int)activLUT[iv + 128]; + optr[k0] = (int8_t)iv; + } + } + } + } + }); +} + #ifdef __clang__ #pragma clang attribute pop #else @@ -861,6 +1104,395 @@ static void convInt8BlockNEON(const void* inp_, const void* residual_, #undef CONV_NEON_STORE #endif // CV_NEON && CV_NEON_DOT +// INT8 depthwise convolution kernel (ngroups==K==C, Kg==Cg==1). +static void convInt8BlockDepthwise(const void* inp_, const void* residual_, + void* out_, const ConvState& cs, + const void* weights_, + const int* bias, const float* multiplier, + int inp_zp, int out_zp, + const int8_t* activLUT, + bool inputIsU8) +{ + constexpr int C0 = 8, K0 = 8; + constexpr int MAX_CONV_DIMS = ConvState::MAX_CONV_DIMS; + + const MatShape& inpshape = cs.inpshape; + const MatShape& outshape = cs.outshape; + + int N = outshape[0]; + int ndims = outshape.dims; + int K = outshape.channels(); + int C = inpshape.channels(); + int K1 = (K + K0 - 1) / K0; + int C1 = (C + C0 - 1) / C0; + + int D_ = ndims >= 6 ? outshape[ndims-4] : 1; + int H_ = ndims >= 5 ? outshape[ndims-3] : 1; + int W_ = outshape[ndims-2]; + int planeblocks_ = D_ * H_ * W_; + + int Di_ = ndims >= 6 ? inpshape[ndims-4] : 1; + int Hi_ = ndims >= 5 ? inpshape[ndims-3] : 1; + int Wi_ = inpshape[ndims-2]; + + int ngroups = cs.ngroups; + int ksize_ = cs.wshape[2]; + int C1Max = cs.wshape[3]; + + int Sz = cs.strides[0], Sy = cs.strides[1], Sx = cs.strides[2]; + int padZ = cs.pads[0], padY = cs.pads[1], padX = cs.pads[2]; + + int innerZ0 = cs.inner[0], innerZ1 = cs.inner[MAX_CONV_DIMS]; + int innerY0 = cs.inner[1], innerY1 = cs.inner[MAX_CONV_DIMS+1]; + int innerX0 = cs.inner[2], innerX1 = cs.inner[MAX_CONV_DIMS+2]; + + const int8_t* inp = (const int8_t*)inp_; + const int8_t* residual = (const int8_t*)residual_; + int8_t* out = (int8_t*)out_; + const int8_t* wdata = (const int8_t*)weights_; + const int* ofsZYX = cs.coordtab.data(); + + int iplanesize_ = Di_ * Hi_ * Wi_ * C0; + int planesize_ = planeblocks_ * K0; + + // Spatial tiling: split planeblocks into ntiles tiles so the total task count + // is N*K1*ntiles, giving all threads work even when K is small (e.g. K=16 → K1=2). + constexpr int SB_TILE = 8; + int nthreads = std::max(1, cv::getNumThreads()); + int TILE = std::max(SB_TILE, + ((planeblocks_ + nthreads * 4 - 1) / (nthreads * 4) + SB_TILE - 1) & ~(SB_TILE - 1)); + int ntiles = (planeblocks_ + TILE - 1) / TILE; + int total_tasks = N * K1 * ntiles; + + parallel_for_(Range(0, total_tasks), [&](const Range& range) { + int H = H_, W = W_; + int Di = Di_, Hi = Hi_, Wi = Wi_; + int planeblocks = planeblocks_; + int iplanesize = iplanesize_; + int planesize = planesize_; + int ksize = ksize_; + + for (int task = range.start; task < range.end; task++) { + int nk = task / ntiles; + int tile = task - nk * ntiles; + int n = nk / K1; + int k1 = nk - n * K1; + + int g_start = k1 * K0; + int g_end = std::min(g_start + K0, ngroups); + int p_task_start = tile * TILE; + int p_task_end = std::min(p_task_start + TILE, planeblocks); + + const int8_t* inpbaseptr = inp + (size_t)(n * C1 + k1) * iplanesize; + int8_t* outbase = out + (size_t)(n * K1 + k1) * planesize; + const int8_t* resbase = residual ? residual + (size_t)(n * K1 + k1) * planesize + : nullptr; + + +#if CV_AVX2 && (defined(__x86_64__) || defined(_M_X64)) + if (ksize <= 128) { + int n_g = g_end - g_start; + constexpr int SB = 8; + + // Pre-gather weights for this group block into wbuf[ksize * K0]. + alignas(32) int8_t wbuf[128 * K0]; + for (int i = 0; i < ksize; i++) { + for (int j = 0; j < n_g; j++) { + int g = g_start + j; + int c0 = g % C0; + wbuf[i * K0 + j] = wdata[(size_t)g * ksize * C1Max * C0 * K0 + + (size_t)i * C1Max * C0 * K0 + + (size_t)c0 * K0]; + } + for (int j = n_g; j < K0; j++) + wbuf[i * K0 + j] = 0; + } + + __m256i vbias = _mm256_loadu_si256((const __m256i*)(bias + g_start)); + __m256 vmult = _mm256_loadu_ps(multiplier + g_start); + __m256 vzp = _mm256_set1_ps((float)out_zp); + __m128i v_xor = inputIsU8 ? _mm_setzero_si128() + : _mm_set1_epi8((char)0x80); + + int p = p_task_start; + for (; p < p_task_end; p += SB) { + if (p + SB > p_task_end) { + if (p == p_task_start) break; + p = p_task_end - SB; + } + + Vec3i pt[SB]; + bool inner_pix[SB]; + bool all_inner = true; + bool same_row = false; + + if ((p % W) + SB <= W) { + same_row = true; + int zj = p / (H * W); + int yxj = p - zj * H * W; + int yj = yxj / W; + int xj0 = yxj - yj * W; + bool zy_inner = (zj >= innerZ0 && zj < innerZ1) && + (yj >= innerY0 && yj < innerY1); + for (int j = 0; j < SB; j++) { + int xj = xj0 + j; + pt[j] = Vec3i(zj*Sz - padZ, yj*Sy - padY, xj*Sx - padX); + inner_pix[j] = zy_inner && (xj >= innerX0 && xj < innerX1); + all_inner &= inner_pix[j]; + } + } else { + for (int j = 0; j < SB; j++) { + int pj = p + j; + int zj = pj / (H * W); + int yxj = pj - zj * H * W; + int yj = yxj / W; + int xj = yxj - yj * W; + pt[j] = Vec3i(zj*Sz - padZ, yj*Sy - padY, xj*Sx - padX); + inner_pix[j] = (zj >= innerZ0 && zj < innerZ1) && + (yj >= innerY0 && yj < innerY1) && + (xj >= innerX0 && xj < innerX1); + all_inner &= inner_pix[j]; + } + } + + __m256i s0 = vbias, s1 = vbias, s2 = vbias, s3 = vbias; + __m256i s4 = vbias, s5 = vbias, s6 = vbias, s7 = vbias; + + if (all_inner && same_row) { + const int step = Sx * C0; + for (int i = 0; i < ksize; i++) { + __m128i w8 = _mm_loadl_epi64((const __m128i*)(wbuf + i * K0)); + __m256i w32 = _mm256_cvtepi8_epi32(w8); + int zi_eff = pt[0][0] + ofsZYX[i*3]; + int yi_eff = pt[0][1] + ofsZYX[i*3+1]; + int xi_eff = pt[0][2] + ofsZYX[i*3+2]; + const int8_t* ip = inpbaseptr + + (((zi_eff * Hi) + yi_eff) * Wi + xi_eff) * C0; + #define DW_HOT(j) { \ + __m256i inp32 = _mm256_cvtepu8_epi32(_mm_xor_si128( \ + _mm_loadl_epi64((const __m128i*)(ip+(j)*step)), v_xor)); \ + s##j = _mm256_add_epi32(s##j, \ + _mm256_mullo_epi32(inp32, w32)); \ + } + DW_HOT(0); DW_HOT(1); DW_HOT(2); DW_HOT(3); + DW_HOT(4); DW_HOT(5); DW_HOT(6); DW_HOT(7); + #undef DW_HOT + } + } else if (all_inner) { + for (int i = 0; i < ksize; i++) { + __m128i w8 = _mm_loadl_epi64((const __m128i*)(wbuf + i * K0)); + __m256i w32 = _mm256_cvtepi8_epi32(w8); + #define DW_INNER(j) { \ + int zi = pt[j][0] + ofsZYX[i*3]; \ + int yi = pt[j][1] + ofsZYX[i*3+1]; \ + int xi = pt[j][2] + ofsZYX[i*3+2]; \ + const int8_t* ip = inpbaseptr + \ + (((zi * Hi) + yi) * Wi + xi) * C0; \ + __m256i inp32 = _mm256_cvtepu8_epi32(_mm_xor_si128( \ + _mm_loadl_epi64((const __m128i*)ip), v_xor)); \ + s##j = _mm256_add_epi32(s##j, \ + _mm256_mullo_epi32(inp32, w32)); \ + } + DW_INNER(0); DW_INNER(1); DW_INNER(2); DW_INNER(3); + DW_INNER(4); DW_INNER(5); DW_INNER(6); DW_INNER(7); + #undef DW_INNER + } + } else { + for (int i = 0; i < ksize; i++) { + __m128i w8 = _mm_loadl_epi64((const __m128i*)(wbuf + i * K0)); + __m256i w32 = _mm256_cvtepi8_epi32(w8); + #define DW_BORDER(j) { \ + int zi = pt[j][0] + ofsZYX[i*3]; \ + int yi = pt[j][1] + ofsZYX[i*3+1]; \ + int xi = pt[j][2] + ofsZYX[i*3+2]; \ + __m256i inp32; \ + if (!((unsigned)zi >= (unsigned)Di || \ + (unsigned)yi >= (unsigned)Hi || \ + (unsigned)xi >= (unsigned)Wi)) { \ + const int8_t* ip = inpbaseptr + \ + (((zi * Hi) + yi) * Wi + xi) * C0; \ + inp32 = _mm256_cvtepu8_epi32(_mm_xor_si128( \ + _mm_loadl_epi64((const __m128i*)ip), v_xor)); \ + } else { \ + inp32 = inputIsU8 \ + ? _mm256_set1_epi32(inp_zp) \ + : _mm256_set1_epi32((int)(uint8_t)((uint8_t)inp_zp ^ 0x80u)); \ + } \ + s##j = _mm256_add_epi32(s##j, _mm256_mullo_epi32(inp32, w32)); \ + } + DW_BORDER(0); DW_BORDER(1); DW_BORDER(2); DW_BORDER(3); + DW_BORDER(4); DW_BORDER(5); DW_BORDER(6); DW_BORDER(7); + #undef DW_BORDER + } + } + + #define DW_STORE(j) { \ + __m256 facc = _mm256_add_ps( \ + _mm256_mul_ps(_mm256_cvtepi32_ps(s##j), vmult), vzp); \ + int8_t* optr = outbase + (p + (j)) * K0; \ + const int8_t* rptr = resbase \ + ? resbase + (p + (j)) * K0 : nullptr; \ + if (rptr) { \ + __m256i r32 = _mm256_cvtepi8_epi32( \ + _mm_loadl_epi64((const __m128i*)rptr)); \ + facc = _mm256_add_ps(facc, _mm256_sub_ps( \ + _mm256_cvtepi32_ps(r32), vzp)); \ + } \ + __m256i iv = _mm256_cvtps_epi32(facc); \ + __m128i lo = _mm256_castsi256_si128(iv); \ + __m128i hi = _mm256_extracti128_si256(iv, 1); \ + __m128i p16 = _mm_packs_epi32(lo, hi); \ + __m128i p8 = inputIsU8 ? _mm_packus_epi16(p16, p16) \ + : _mm_packs_epi16(p16, p16); \ + if (activLUT) { \ + alignas(16) int8_t tmp8[16]; \ + _mm_store_si128((__m128i*)tmp8, p8); \ + if (inputIsU8) { \ + for (int jj = 0; jj < n_g; jj++) \ + ((uint8_t*)optr)[jj] = \ + (uint8_t)activLUT[(int)(uint8_t)tmp8[jj]]; \ + } else { \ + for (int jj = 0; jj < n_g; jj++) \ + optr[jj] = (int8_t)activLUT[(int)tmp8[jj]+128]; \ + } \ + } else if (n_g == K0) { \ + _mm_storel_epi64((__m128i*)optr, p8); \ + } else { \ + alignas(16) int8_t tmp8[16]; \ + _mm_store_si128((__m128i*)tmp8, p8); \ + memcpy(optr, tmp8, n_g); \ + } \ + } + DW_STORE(0); DW_STORE(1); DW_STORE(2); DW_STORE(3); + DW_STORE(4); DW_STORE(5); DW_STORE(6); DW_STORE(7); + #undef DW_STORE + } + for (; p < p_task_end; p++) { + int zj = p / (H * W); + int yxj = p - zj * H * W; + int yj = yxj / W; + int xj = yxj - yj * W; + int zi_base = zj * Sz - padZ, yi_base = yj * Sy - padY, + xi_base = xj * Sx - padX; + __m256i acc = vbias; + for (int i = 0; i < ksize; i++) { + int zi = zi_base + ofsZYX[i*3], yi = yi_base + ofsZYX[i*3+1], + xi = xi_base + ofsZYX[i*3+2]; + if ((unsigned)zi>=(unsigned)Di || (unsigned)yi>=(unsigned)Hi || + (unsigned)xi>=(unsigned)Wi) continue; + const int8_t* ip = inpbaseptr + (((zi*Hi)+yi)*Wi+xi)*C0; + __m256i inp32 = _mm256_cvtepu8_epi32( + _mm_xor_si128(_mm_loadl_epi64((const __m128i*)ip), v_xor)); + __m256i w32 = _mm256_cvtepi8_epi32( + _mm_loadl_epi64((const __m128i*)(wbuf + i*K0))); + acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(inp32, w32)); + } + __m256 facc = _mm256_add_ps( + _mm256_mul_ps(_mm256_cvtepi32_ps(acc), vmult), vzp); + int8_t* optr = outbase + p * K0; + const int8_t* rptr = resbase ? resbase + p * K0 : nullptr; + if (rptr) { + facc = _mm256_add_ps(facc, _mm256_sub_ps(_mm256_cvtepi32_ps( + _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*)rptr))), vzp)); + } + __m256i iv = _mm256_cvtps_epi32(facc); + __m128i p16 = _mm_packs_epi32(_mm256_castsi256_si128(iv), + _mm256_extracti128_si256(iv, 1)); + __m128i p8 = inputIsU8 ? _mm_packus_epi16(p16, p16) + : _mm_packs_epi16(p16, p16); + if (activLUT) { + alignas(16) int8_t tmp8[16]; + _mm_store_si128((__m128i*)tmp8, p8); + if (inputIsU8) + for (int jj=0; jj= innerZ0 && zj < innerZ1) && + (yj >= innerY0 && yj < innerY1) && + (xj >= innerX0 && xj < innerX1); + + int zi_base = zj * Sz - padZ; + int yi_base = yj * Sy - padY; + int xi_base = xj * Sx - padX; + + int8_t* optr = outbase + p * K0; + const int8_t* rptr = resbase ? resbase + p * K0 : nullptr; + + for (int g = g_start; g < g_end; g++) { + int j = g - g_start; + // For Kg=1 depthwise: kin=0, k0=0, c0=g%C0 + int c0 = g % C0; + const int8_t* wbase_g = wdata + (size_t)g * ksize * C1Max * C0 * K0; + int32_t acc = bias[g]; + + if (is_inner) { + for (int i = 0; i < ksize; i++) { + int zi = zi_base + ofsZYX[i * 3]; + int yi = yi_base + ofsZYX[i * 3 + 1]; + int xi = xi_base + ofsZYX[i * 3 + 2]; + const int8_t* ip = inpbaseptr + (((zi * Hi) + yi) * Wi + xi) * C0; + int ival = inputIsU8 ? (int)(uint8_t)ip[c0] + : (int)(uint8_t)((uint8_t)ip[c0] ^ 0x80u); + acc += ival * (int)wbase_g[(size_t)i * C1Max * C0 * K0 + c0 * K0]; + } + } else { + for (int i = 0; i < ksize; i++) { + int zi = zi_base + ofsZYX[i * 3]; + int yi = yi_base + ofsZYX[i * 3 + 1]; + int xi = xi_base + ofsZYX[i * 3 + 2]; + int ival; + if ((unsigned)zi >= (unsigned)Di || + (unsigned)yi >= (unsigned)Hi || + (unsigned)xi >= (unsigned)Wi) { + ival = inputIsU8 ? inp_zp + : (int)(uint8_t)((uint8_t)inp_zp ^ 0x80u); + } else { + const int8_t* ip = inpbaseptr + (((zi * Hi) + yi) * Wi + xi) * C0; + ival = inputIsU8 ? (int)(uint8_t)ip[c0] + : (int)(uint8_t)((uint8_t)ip[c0] ^ 0x80u); + } + acc += ival * (int)wbase_g[(size_t)i * C1Max * C0 * K0 + c0 * K0]; + } + } + + float val = (float)acc * multiplier[g] + (float)out_zp; + if (rptr) val += (float)((int)rptr[j] - out_zp); + int ival = cvRound(val); + if (inputIsU8) { + ival = std::max(0, std::min(255, ival)); + if (activLUT) ival = (int)(uint8_t)activLUT[ival]; + ((uint8_t*)optr)[j] = (uint8_t)ival; + } else { + ival = std::max(-128, std::min(127, ival)); + if (activLUT) ival = (int)activLUT[ival + 128]; + optr[j] = (int8_t)ival; + } + } + } + } + }); +} + void convInt8Block(const void* inp_, const void* residual_, void* out_, const ConvState& cs, const void* weights_, @@ -871,8 +1503,21 @@ void convInt8Block(const void* inp_, const void* residual_, const int8_t* activLUT, bool inputIsU8) { + // Dispatch to dedicated depthwise kernel before the general VNNI path. + if (cs.depthwise) { + const int* dw_bias = biasVNNI_ ? biasVNNI_ : bias; + convInt8BlockDepthwise(inp_, residual_, out_, cs, weights_, + dw_bias, multiplier, inp_zp, out_zp, activLUT, inputIsU8); + return; + } #if CV_AVXVNNI_AVAILABLE if (cv::checkHardwareSupport(CV_CPU_AVX_VNNI) && weightsVNNI_ && biasVNNI_) { + if (cs.wshape[2] == 1 && + cs.strides[0] == 1 && cs.strides[1] == 1 && cs.strides[2] == 1) { + convInt8Block1x1(inp_, residual_, out_, cs, weightsVNNI_, + biasVNNI_, multiplier, inp_zp, out_zp, activLUT, inputIsU8); + return; + } convInt8BlockVNNI(inp_, residual_, out_, cs, weightsVNNI_, biasVNNI_, multiplier, inp_zp, out_zp, activLUT, inputIsU8); return; @@ -999,10 +1644,14 @@ void convInt8Block(const void* inp_, const void* residual_, } alignas(32) int32_t biasbuf[K0]; - memcpy(biasbuf, bias + k_base, K0 * sizeof(int32_t)); - alignas(32) float multbuf[K0]; - memcpy(multbuf, multiplier + k_base, K0 * sizeof(float)); + { + int k_valid = std::min(K0, K - k_base); + memcpy(biasbuf, bias + k_base, k_valid * sizeof(int32_t)); + memset(biasbuf + k_valid, 0, (K0 - k_valid) * sizeof(int32_t)); + memcpy(multbuf, multiplier + k_base, k_valid * sizeof(float)); + memset(multbuf + k_valid, 0, (K0 - k_valid) * sizeof(float)); + } #if CV_AVX2 int p = 0; @@ -1143,8 +1792,10 @@ void convInt8Block(const void* inp_, const void* residual_, if (activLUT) { alignas(16) int8_t tmp8[16]; _mm_store_si128((__m128i*)tmp8, packed8); - for (int k = 0; k < K0; k++) - optr[k] = (int8_t)activLUT[(int)tmp8[k] + 128]; + for (int k = 0; k < K0; k++) { + int lut_idx = inputIsU8 ? (int)(uint8_t)tmp8[k] : ((int)tmp8[k] + 128); + optr[k] = (int8_t)activLUT[lut_idx]; + } } else { _mm_storel_epi64((__m128i*)optr, packed8); } diff --git a/modules/dnn/src/int8layers/conv2_int8_layer.cpp b/modules/dnn/src/int8layers/conv2_int8_layer.cpp index 4ca990cfea..471eda18a6 100644 --- a/modules/dnn/src/int8layers/conv2_int8_layer.cpp +++ b/modules/dnn/src/int8layers/conv2_int8_layer.cpp @@ -172,8 +172,10 @@ class Conv2Int8LayerImpl CV_FINAL : public Conv2Int8Layer public: Mat weights; // repacked int8 weights in block format Mat weightsVNNI; // VNNI-transposed weights (pre-computed) - Mat biasInt32; // int32 fused bias - Mat biasVNNI; // int32 VNNI-adjusted bias (pre-computed) + Mat biasInt32; + Mat biasVNNI; + Mat biasU8; + Mat biasBaselineU8; Mat outMultiplier; // float32 per-channel output multiplier MatShape wshape0; // original weight shape (K x Cg x kH x kW) MatShape prevInpshape; @@ -228,6 +230,7 @@ public: output_zp = p.output_zp; per_channel = p.per_channel; inputIsU8 = p.input_is_u8; + float_input = p.float_input; addResidual = false; if (!p.weights.empty()) { @@ -243,8 +246,11 @@ public: std::vector& outputs, std::vector& internals) const CV_OVERRIDE { - // Input can be CV_8SC1 or CV_8UC1; output matches input type - int outtype = !inputs.empty() ? inputs[0] : CV_8SC1; + int outtype; + if (float_input) + outtype = inputIsU8 ? CV_8UC1 : CV_8SC1; + else + outtype = !inputs.empty() ? inputs[0] : CV_8SC1; outputs.assign(requiredOutputs, outtype); internals.clear(); } @@ -258,7 +264,14 @@ public: CV_Assert(ninputs >= 1); std::vector emptyKernelShape; - outshapes.assign(1, convInferShape(inpshapes[0], wshape0, emptyKernelShape, + MatShape inpshape = inpshapes[0]; + if (float_input && inpshape.layout != DATA_LAYOUT_BLOCK) { + if (inpshape.layout == DATA_LAYOUT_UNKNOWN) + inpshape.layout = DATA_LAYOUT_NCHW; + int C0 = getNetImpl(this)->defaultC0; + inpshape = inpshape.toLayout(DATA_LAYOUT_BLOCK, C0); + } + outshapes.assign(1, convInferShape(inpshape, wshape0, emptyKernelShape, ngroups, strides, dilations, pads, auto_pad, ceil_mode)); tempshapes.clear(); @@ -273,7 +286,7 @@ public: size_t ninputs = actualInputs.size(); CV_Assert(ninputs >= 1u && requiredOutputs == 1u); desiredInputs = actualInputs; - desiredInputs[0] = DATA_LAYOUT_BLOCK; + desiredInputs[0] = float_input ? DATA_LAYOUT_NCHW : DATA_LAYOUT_BLOCK; for (size_t i = 1; i < ninputs; i++) desiredInputs[i] = DATA_LAYOUT_UNKNOWN; outputs.assign(requiredOutputs, DATA_LAYOUT_BLOCK); @@ -301,13 +314,58 @@ public: Ptr activ_int8 = layer.dynamicCast(); if (!activ_int8.empty()) { activ = activ_int8; - if (!activ_int8->blobs.empty()) - activ_int8->blobs[0].convertTo(activationLUT, CV_8S); + if (!activ_int8->activationLUT.empty()) + activationLUT = activ_int8->activationLUT; + else if (!activ_int8->blobs.empty()) + activ_int8->blobs[0].copyTo(activationLUT); return true; } return false; } + void quantizeInterleaveBlock(const Mat& fp32_nchw, Mat& int8_block, int C0) const + { + MatShape nchw = fp32_nchw.shape(); + if (nchw.layout == DATA_LAYOUT_UNKNOWN) + nchw.layout = DATA_LAYOUT_NCHW; + CV_Assert(nchw.dims >= 2); + int N = nchw[0], C = nchw[1]; + int planesize = 1; + for (int d = 2; d < nchw.dims; d++) planesize *= nchw[d]; + int C1 = (C + C0 - 1) / C0; + + MatShape blkshape = nchw.toLayout(DATA_LAYOUT_BLOCK, C0); + int8_block.fit(blkshape, inputIsU8 ? CV_8UC1 : CV_8SC1); + + float inv_sc = input_sc > 0.f ? 1.f / input_sc : 0.f; + int zp = input_zp; + const bool is_u8 = inputIsU8; + const float* src = fp32_nchw.ptr(); + uint8_t* dst = int8_block.ptr(); + + parallel_for_(Range(0, N * C1), [&](const Range& range) { + for (int nc = range.start; nc < range.end; nc++) { + int n = nc / C1, c1 = nc % C1; + int c_base = c1 * C0; + int nzc = std::min(C0, C - c_base); + const float* src_block = src + (n * C + c_base) * planesize; + uint8_t* dst_block = dst + (size_t)(n * C1 + c1) * planesize * C0; + for (int i = 0; i < planesize; i++) { + for (int c = 0; c < nzc; c++) { + float x = src_block[c * planesize + i]; + int q = cvRound(x * inv_sc) + zp; + if (is_u8) + dst_block[i * C0 + c] = saturate_cast(q); + else + dst_block[i * C0 + c] = (uint8_t)(int8_t)saturate_cast(q); + } + for (int c = nzc; c < C0; c++) + dst_block[i * C0 + c] = 0; + } + } + }); + } + void forward(InputArrayOfArrays input_arrs, OutputArrayOfArrays output_arrs, OutputArrayOfArrays) CV_OVERRIDE @@ -315,7 +373,16 @@ public: int ninputs = (int)input_arrs.total(); CV_Assert(ninputs >= 1); - const Mat& inp = input_arrs.getMat(0); + Mat inp_storage; + Mat inp_ref = input_arrs.getMat(0); + + if (float_input) { + CV_Assert(inp_ref.type() == CV_32F); + int C0 = getNetImpl(this)->defaultC0; + quantizeInterleaveBlock(inp_ref, inp_storage, C0); + } + + const Mat& inp = float_input ? inp_storage : inp_ref; MatShape inpshape = inp.shape(); CV_Assert(inpshape.layout == DATA_LAYOUT_BLOCK); CV_Assert(inp.type() == CV_8SC1 || inp.type() == CV_8UC1); @@ -332,6 +399,20 @@ public: int Kg = K / ngroups; repackWeightsForVNNI(weights, ngroups, Kg, Cg, biasInt32, weightsVNNI, biasVNNI); + + if (inputIsU8) { + biasU8.create({K}, CV_32SC1); + biasBaselineU8.create({K}, CV_32SC1); + const int32_t* bI = biasInt32.ptr(); + const int32_t* bV = biasVNNI.ptr(); + int32_t* bU = biasU8.ptr(); + int32_t* bB = biasBaselineU8.ptr(); + for (int k = 0; k < K; k++) { + int32_t wsum = (bI[k] - bV[k]) / 128; + bU[k] = bI[k] - input_zp * wsum; + bB[k] = bI[k] + (128 - input_zp) * wsum; + } + } } Mat residual; @@ -376,11 +457,19 @@ public: const int8_t* lutptr = !activationLUT.empty() ? activationLUT.ptr() : nullptr; + const int* effectiveBiasVNNI = inputIsU8 + ? (biasU8.empty() ? nullptr : biasU8.ptr()) + : (biasVNNI.empty() ? nullptr : biasVNNI.ptr()); + + const int* baselineBias = (inputIsU8 && !biasBaselineU8.empty()) + ? biasBaselineU8.ptr() + : biasInt32.ptr(); + convInt8Block(inp.data, resptr, out.data, cs, weights.data, weightsVNNI.empty() ? nullptr : weightsVNNI.data, - biasInt32.ptr(), - biasVNNI.empty() ? nullptr : biasVNNI.ptr(), + baselineBias, + effectiveBiasVNNI, outMultiplier.ptr(), input_zp, output_zp, lutptr, inputIsU8); diff --git a/modules/dnn/src/int8layers/elementwise_layers.cpp b/modules/dnn/src/int8layers/elementwise_layers.cpp index 3826d2e553..0e307a1dde 100644 --- a/modules/dnn/src/int8layers/elementwise_layers.cpp +++ b/modules/dnn/src/int8layers/elementwise_layers.cpp @@ -307,9 +307,23 @@ public: const int nstripes = getNumThreads(); Mat &dst = outputs[i]; CV_Assert(src.size == dst.size && src.type() == dst.type() && - src.isContinuous() && dst.isContinuous() && src.type() == CV_8S); + src.isContinuous() && dst.isContinuous() && + (src.type() == CV_8S || src.type() == CV_8U)); - Activation::run(src, activationLUT, dst, nstripes); + if (src.type() == CV_8S) { + Activation::run(src, activationLUT, dst, nstripes); + } else { + const uint8_t* table = activationLUT.ptr(); + const uint8_t* srcptr = src.ptr(); + uint8_t* dstptr = dst.ptr(); + int total = (int)src.total(); + parallel_for_(Range(0, nstripes), [&](const Range& r) { + int start = (int)((size_t)r.start * total / nstripes); + int end = (int)((size_t)r.end * total / nstripes); + for (int j = start; j < end; j++) + dstptr[j] = table[srcptr[j]]; + }, nstripes); + } } else { diff --git a/modules/dnn/src/int8layers/eltwise2_int8_layer.cpp b/modules/dnn/src/int8layers/eltwise2_int8_layer.cpp index 1b9f60e54d..e3329114f4 100644 --- a/modules/dnn/src/int8layers/eltwise2_int8_layer.cpp +++ b/modules/dnn/src/int8layers/eltwise2_int8_layer.cpp @@ -18,11 +18,48 @@ namespace cv { namespace dnn { +static Mat broadcastExpand(const Mat& src, const MatShape& dstShape) +{ + MatShape srcShape = src.shape(); + const int ndst = (int)dstShape.size(); + while ((int)srcShape.size() < ndst) + srcShape.insert(srcShape.begin(), 1); + if (srcShape == dstShape) + return src; + + MatShape outShape = dstShape; + Mat out(outShape, src.type()); + const int total = (int)out.total(); + + std::vector dstStrides(ndst); + { int s = 1; for (int d = ndst-1; d >= 0; d--) { dstStrides[d] = s; s *= dstShape[d]; } } + + std::vector srcStrides(ndst, 0); + { int s = 1; for (int d = ndst-1; d >= 0; d--) { + srcStrides[d] = (srcShape[d] > 1) ? s : 0; s *= srcShape[d]; } } + + const uint8_t* sp = src.ptr(); + uint8_t* dp = out.ptr(); + for (int i = 0; i < total; i++) { + int rem = i, srcIdx = 0; + for (int d = 0; d < ndst; d++) { + int coord = rem / dstStrides[d]; rem %= dstStrides[d]; + srcIdx += coord * srcStrides[d]; + } + dp[i] = sp[srcIdx]; + } + return out; +} + class Eltwise2Int8LayerImpl CV_FINAL : public Eltwise2Int8Layer { public: + enum class Op { ADD, MUL }; + Op op; + std::vector coeffs; float offset; + float mulCoeff; bool withRelu; Mat activationLUT; @@ -36,6 +73,9 @@ public: output_sc = params.get("scales", 1.0f); withRelu = params.get("with_relu", false); + std::string opStr = params.get("operation", "add"); + op = (opStr == "mul" || opStr == "prod") ? Op::MUL : Op::ADD; + if (params.has("input_scales")) { DictValue sc = params.get("input_scales"); @@ -55,6 +95,7 @@ public: } offset = 0.f; + mulCoeff = 0.f; } Eltwise2Int8LayerImpl(const Eltwise2Int8Params& p) @@ -66,7 +107,9 @@ public: output_sc = p.output_sc; output_zp = p.output_zp; withRelu = p.with_relu; + op = (p.operation == "mul" || p.operation == "prod") ? Op::MUL : Op::ADD; offset = 0.f; + mulCoeff = 0.f; } void ensureCoeffs() @@ -79,11 +122,19 @@ public: CV_Assert(output_sc > 0.f); coeffs.resize(scales.size()); - offset = (float)output_zp; - for (size_t i = 0; i < scales.size(); i++) - { - coeffs[i] = scales[i] / output_sc; - offset -= coeffs[i] * zeropoints[i]; + if (op == Op::MUL) { + CV_Assert(scales.size() == 2); + mulCoeff = (scales[0] * scales[1]) / output_sc; + coeffs[0] = (float)zeropoints[0]; + coeffs[1] = (float)zeropoints[1]; + offset = (float)output_zp; + } else { + offset = (float)output_zp; + for (size_t i = 0; i < scales.size(); i++) + { + coeffs[i] = scales[i] / output_sc; + offset -= coeffs[i] * zeropoints[i]; + } } } @@ -93,7 +144,24 @@ public: std::vector& internals) const CV_OVERRIDE { CV_Assert(inputs.size() >= 2); - outputs.assign(1, inputs[0]); + int ndims = 0; + DataLayout outLayout = DATA_LAYOUT_UNKNOWN; + int outC = 0; + for (const auto& s : inputs) { + ndims = std::max(ndims, (int)s.size()); + if (s.layout != DATA_LAYOUT_UNKNOWN && outLayout == DATA_LAYOUT_UNKNOWN) + outLayout = s.layout; + if (s.C > outC) outC = s.C; + } + MatShape outShape(ndims, 1); + for (const auto& s : inputs) { + int off = ndims - (int)s.size(); + for (int d = 0; d < (int)s.size(); d++) + outShape[d + off] = std::max(outShape[d + off], s[d]); + } + outShape.layout = outLayout; + outShape.C = outC; + outputs.assign(1, outShape); internals.clear(); return true; } @@ -145,8 +213,39 @@ public: int ninputs = (int)input_arrs.total(); CV_Assert(ninputs >= 2 && coeffs.size() == (size_t)ninputs); - const Mat& inp0 = input_arrs.getMat(0); + std::vector inputs(ninputs); + DataLayout outLayout = DATA_LAYOUT_UNKNOWN; + int outC = 0; + { + int ndims = 0; + for (int k = 0; k < ninputs; k++) { + inputs[k] = input_arrs.getMat(k); + const MatShape& sh = inputs[k].shape(); + ndims = std::max(ndims, (int)sh.size()); + if (sh.layout != DATA_LAYOUT_UNKNOWN && outLayout == DATA_LAYOUT_UNKNOWN) + outLayout = sh.layout; + if (sh.C > outC) outC = sh.C; + } + MatShape broadShape(ndims, 1); + for (int k = 0; k < ninputs; k++) { + const MatShape& sh = inputs[k].shape(); + int off = ndims - (int)sh.size(); + for (int d = 0; d < (int)sh.size(); d++) + broadShape[d + off] = std::max(broadShape[d + off], sh[d]); + } + broadShape.layout = outLayout; + broadShape.C = outC; + for (int k = 0; k < ninputs; k++) + if (inputs[k].shape() != broadShape) + inputs[k] = broadcastExpand(inputs[k], broadShape); + } + + const Mat& inp0 = inputs[0]; MatShape outshape = inp0.shape(); + if (outshape.layout == DATA_LAYOUT_UNKNOWN && outLayout != DATA_LAYOUT_UNKNOWN) { + outshape.layout = outLayout; + outshape.C = outC; + } int outtype = inp0.type(); CV_Assert(outtype == CV_8SC1 || outtype == CV_8UC1); const bool isU8 = (outtype == CV_8UC1); @@ -175,6 +274,11 @@ public: const float c0 = cptr[0]; const float c1 = cptr[1]; + const bool isMul = (op == Op::MUL); + const float mc = mulCoeff; + const float zp0f = isMul ? (float)zeropoints[0] : 0.f; + const float zp1f = isMul ? (float)zeropoints[1] : 0.f; + const float ozpf = (float)output_zp; const int out_min = isU8 ? (withRelu ? output_zp : 0) : (withRelu ? output_zp : -128); const int out_max = isU8 ? 255 : 127; @@ -184,7 +288,7 @@ public: if (isU8) { std::vector inptrs(ninputs); for (int k = 0; k < ninputs; k++) - inptrs[k] = input_arrs.getMat(k).ptr(); + inptrs[k] = inputs[k].ptr(); const uint8_t* p0 = inptrs[0]; const uint8_t* p1 = inptrs[1]; uint8_t* outptr = out.ptr(); @@ -192,7 +296,32 @@ public: parallel_for_(Range(0, (int)total_elems), [&](const Range& r) { int i = r.start; #if CV_AVX2 - if (!lutptr && ninputs == 2) { + if (isMul && !lutptr && ninputs == 2) { + __m256 vmc = _mm256_set1_ps(mc); + __m256 vzp0 = _mm256_set1_ps(zp0f); + __m256 vzp1 = _mm256_set1_ps(zp1f); + __m256 vozp = _mm256_set1_ps(ozpf); + __m256i vmin = _mm256_set1_epi32(out_min); + __m256i vmax = _mm256_set1_epi32(out_max); + for (; i <= r.end - 8; i += 8) { + __m128i a8 = _mm_loadl_epi64((const __m128i*)(p0 + i)); + __m128i b8 = _mm_loadl_epi64((const __m128i*)(p1 + i)); + __m256i a32 = _mm256_cvtepu8_epi32(a8); + __m256i b32 = _mm256_cvtepu8_epi32(b8); + __m256 af = _mm256_sub_ps(_mm256_cvtepi32_ps(a32), vzp0); + __m256 bf = _mm256_sub_ps(_mm256_cvtepi32_ps(b32), vzp1); + __m256 val = _mm256_fmadd_ps(_mm256_mul_ps(af, bf), vmc, vozp); + __m256i ival = _mm256_cvtps_epi32(val); + ival = _mm256_max_epi32(ival, vmin); + ival = _mm256_min_epi32(ival, vmax); + __m128i lo = _mm256_castsi256_si128(ival); + __m128i hi = _mm256_extracti128_si256(ival, 1); + __m128i packed16 = _mm_packs_epi32(lo, hi); + __m128i packed8 = _mm_packus_epi16(packed16, packed16); + _mm_storel_epi64((__m128i*)(outptr + i), packed8); + } + } else + if (!isMul && !lutptr && ninputs == 2) { __m256 vc0 = _mm256_set1_ps(c0); __m256 vc1 = _mm256_set1_ps(c1); __m256 voff = _mm256_set1_ps(off); @@ -274,10 +403,14 @@ public: #endif for (; i < r.end; i++) { - float val = c0 * (float)p0[i] + c1 * (float)p1[i] + off; - - for (int k = 2; k < ninputs; k++) - val += cptr[k] * (float)inptrs[k][i]; + float val; + if (isMul) { + val = mc * ((float)p0[i] - zp0f) * ((float)p1[i] - zp1f) + ozpf; + } else { + val = c0 * (float)p0[i] + c1 * (float)p1[i] + off; + for (int k = 2; k < ninputs; k++) + val += cptr[k] * (float)inptrs[k][i]; + } int ival = cvRound(val); ival = std::max(out_min, std::min(out_max, ival)); @@ -291,7 +424,7 @@ public: } else { std::vector inptrs(ninputs); for (int k = 0; k < ninputs; k++) - inptrs[k] = input_arrs.getMat(k).ptr(); + inptrs[k] = inputs[k].ptr(); const int8_t* p0 = inptrs[0]; const int8_t* p1 = inptrs[1]; int8_t* outptr = out.ptr(); @@ -299,7 +432,32 @@ public: parallel_for_(Range(0, (int)total_elems), [&](const Range& r) { int i = r.start; #if CV_AVX2 - if (!lutptr && ninputs == 2) { + if (isMul && !lutptr && ninputs == 2) { + __m256 vmc = _mm256_set1_ps(mc); + __m256 vzp0 = _mm256_set1_ps(zp0f); + __m256 vzp1 = _mm256_set1_ps(zp1f); + __m256 vozp = _mm256_set1_ps(ozpf); + __m256i vmin = _mm256_set1_epi32(out_min); + __m256i vmax = _mm256_set1_epi32(out_max); + for (; i <= r.end - 8; i += 8) { + __m128i a8 = _mm_loadl_epi64((const __m128i*)(p0 + i)); + __m128i b8 = _mm_loadl_epi64((const __m128i*)(p1 + i)); + __m256i a32 = _mm256_cvtepi8_epi32(a8); + __m256i b32 = _mm256_cvtepi8_epi32(b8); + __m256 af = _mm256_sub_ps(_mm256_cvtepi32_ps(a32), vzp0); + __m256 bf = _mm256_sub_ps(_mm256_cvtepi32_ps(b32), vzp1); + __m256 val = _mm256_fmadd_ps(_mm256_mul_ps(af, bf), vmc, vozp); + __m256i ival = _mm256_cvtps_epi32(val); + ival = _mm256_max_epi32(ival, vmin); + ival = _mm256_min_epi32(ival, vmax); + __m128i lo = _mm256_castsi256_si128(ival); + __m128i hi = _mm256_extracti128_si256(ival, 1); + __m128i packed16 = _mm_packs_epi32(lo, hi); + __m128i packed8 = _mm_packs_epi16(packed16, packed16); + _mm_storel_epi64((__m128i*)(outptr + i), packed8); + } + } else + if (!isMul && !lutptr && ninputs == 2) { __m256 vc0 = _mm256_set1_ps(c0); __m256 vc1 = _mm256_set1_ps(c1); __m256 voff = _mm256_set1_ps(off); @@ -381,10 +539,14 @@ public: #endif for (; i < r.end; i++) { - float val = c0 * (float)p0[i] + c1 * (float)p1[i] + off; - - for (int k = 2; k < ninputs; k++) - val += cptr[k] * (float)inptrs[k][i]; + float val; + if (isMul) { + val = mc * ((float)p0[i] - zp0f) * ((float)p1[i] - zp1f) + ozpf; + } else { + val = c0 * (float)p0[i] + c1 * (float)p1[i] + off; + for (int k = 2; k < ninputs; k++) + val += cptr[k] * (float)inptrs[k][i]; + } int ival = cvRound(val); ival = std::max(out_min, std::min(out_max, ival)); diff --git a/modules/dnn/test/test_model.cpp b/modules/dnn/test/test_model.cpp index b0531fa5e1..bfb8aa6594 100644 --- a/modules/dnn/test/test_model.cpp +++ b/modules/dnn/test/test_model.cpp @@ -5,6 +5,8 @@ #include "test_precomp.hpp" #include #include "npy_blob.hpp" +#include +#include namespace opencv_test { namespace { template @@ -817,12 +819,20 @@ TEST_P(Reproducibility_ResNet50_QDQ_ONNX, Accuracy) topK(out, res, K); ASSERT_EQ(int(res.size()), K); + // Top 4 class's score must be within eps of its reference value. std::vector > ref = {{285, 10.44}, {287, 10.13}, {283, 8.89}, {278, 8.43}}; const float eps = 0.5f; - for (int i = 0; i < (int)ref.size(); i++) { - EXPECT_EQ(ref[i].first, res[i].first); - EXPECT_NEAR(ref[i].second, res[i].second, eps); + std::map res_map; + for (int i = 0; i < (int)res.size(); i++) + res_map[res[i].first] = res[i].second; + + for (const auto& r : ref) { + auto it = res_map.find(r.first); + EXPECT_NE(it, res_map.end()) << "Expected class " << r.first << " not found in top-4"; + if (it != res_map.end()) { + EXPECT_NEAR(r.second, it->second, eps) << "Score mismatch for class " << r.first; + } } } INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_ResNet50_QDQ_ONNX,