From 2ed97b9ef3f096031b55e44dff254533dc14afa2 Mon Sep 17 00:00:00 2001 From: fengyuentau Date: Fri, 5 Jan 2024 18:15:59 +0800 Subject: [PATCH] multi-threaded scatterND and refactor perf --- modules/dnn/perf/perf_layer.cpp | 168 ++++++++++----------- modules/dnn/src/layers/scatterND_layer.cpp | 78 +++++----- 2 files changed, 123 insertions(+), 123 deletions(-) diff --git a/modules/dnn/perf/perf_layer.cpp b/modules/dnn/perf/perf_layer.cpp index 04e7d04153..3e477da125 100644 --- a/modules/dnn/perf/perf_layer.cpp +++ b/modules/dnn/perf/perf_layer.cpp @@ -324,103 +324,95 @@ INSTANTIATE_TEST_CASE_P(/**/, Layer_Scatter, Combine( /* withCann= */ false) // only test on CPU )); -struct Layer_ScatterND : public TestBaseWithParam > -{ - void test_layer(const std::vector& shape, const String reduction = "none") +using Layer_ScatterND = TestBaseWithParam, std::string, tuple>>; +PERF_TEST_P_(Layer_ScatterND, scatterND) { + std::vector shape = get<0>(GetParam()); + std::string reduction = get<1>(GetParam()); + int backend_id = get<0>(get<2>(GetParam())); + int target_id = get<1>(get<2>(GetParam())); + + std::vector indices_shape(shape); + indices_shape.push_back(int(shape.size())); + Mat data(shape, CV_32FC1); + Mat indices(indices_shape, CV_32FC1); + Mat updates(shape, CV_32FC1); + + randn(data, 0.f, 1.f); + randn(updates, 0.f, 1.f); + + // initialize the indices with index tuples like [0...N, 0...C, 0...H, 0...W] + std::vector current_index_tuple(shape.size()); + int total = data.total(); + std::vector indices_step; + for (int i = 0; i < indices.dims; i++) { - int backendId = get<0>(GetParam()); - int targetId = get<1>(GetParam()); - - std::vector indices_shape(shape); - indices_shape.push_back(int(shape.size())); - Mat data(shape, CV_32FC1); - Mat indices(indices_shape, CV_32FC1); - Mat updates(shape, CV_32FC1); - - Scalar mean = 0.f; - Scalar std = 1.f; - randn(data, mean, std); - randn(updates, mean, std); - - // initialize the indices with index tuples like [0...N, 0...C, 0...H, 0...W] - std::vector current_index_tuple(shape.size()); - int total = data.total(); - std::vector indices_step; - for (int i = 0; i < indices.dims; i++) + int step = indices.step.p[i] / sizeof(float); + indices_step.push_back(step); + } + int t, j, idx, offset_at_idx, offset; + for (int i = 0; i < total; i++) + { + t = i; + for (j = shape.size() - 1; j >= 0; j--) { - int step = indices.step.p[i] / sizeof(float); - indices_step.push_back(step); - } - int t, j, idx, offset_at_idx, offset; - for (int i = 0; i < total; i++) - { - t = i; - for (j = shape.size() - 1; j >= 0; j--) - { - idx = t / shape[j]; - offset_at_idx = (int)(t - idx * shape[j]); - current_index_tuple[j] = offset_at_idx; - t = idx; - } - - offset = 0; - for (j = 0; j < shape.size(); j++) - offset += current_index_tuple[j] * indices_step[j]; - - for (j = 0; j < shape.size(); j++) - indices.at(offset + j) = current_index_tuple[j]; + idx = t / shape[j]; + offset_at_idx = (int)(t - idx * shape[j]); + current_index_tuple[j] = offset_at_idx; + t = idx; } - Net net; - LayerParams lp; - lp.type = "ScatterND"; - lp.name = "testLayer"; - lp.set("reduction", reduction); + offset = 0; + for (j = 0; j < shape.size(); j++) + offset += current_index_tuple[j] * indices_step[j]; - int id = net.addLayerToPrev(lp.name, lp.type, lp); - net.connect(0, 0, id, 0); - net.connect(0, 1, id, 1); - net.connect(0, 2, id, 2); - - // warmup - { - std::vector inpNames(3); - inpNames[0] = "data"; - inpNames[1] = "indices"; - inpNames[2] = "updates"; - net.setInputsNames(inpNames); - net.setInput(data, inpNames[0]); - net.setInput(indices, inpNames[1]); - net.setInput(updates, inpNames[2]); - - net.setPreferableBackend(backendId); - net.setPreferableTarget(targetId); - Mat out = net.forward(); - } - - TEST_CYCLE() - { - Mat res = net.forward(); - } - - SANITY_CHECK_NOTHING(); + for (j = 0; j < shape.size(); j++) + indices.at(offset + j) = current_index_tuple[j]; } - int N = 8; - int C = 256; - int H = 128; - int W = 100; -}; + Net net; + LayerParams lp; + lp.type = "ScatterND"; + lp.name = "testLayer"; + lp.set("reduction", reduction); -PERF_TEST_P_(Layer_ScatterND, DISABLED_ScatterND) -{ - test_layer({N, C, H ,W}); + int id = net.addLayerToPrev(lp.name, lp.type, lp); + net.connect(0, 0, id, 0); + net.connect(0, 1, id, 1); + net.connect(0, 2, id, 2); + + // warmup + { + std::vector input_names{"data", "indices", "updates"}; + net.setInputsNames(input_names); + net.setInput(data, input_names[0]); + net.setInput(indices, input_names[1]); + net.setInput(updates, input_names[2]); + + net.setPreferableBackend(backend_id); + net.setPreferableTarget(target_id); + Mat out = net.forward(); + } + + TEST_CYCLE() + { + Mat res = net.forward(); + } + + SANITY_CHECK_NOTHING(); } -PERF_TEST_P_(Layer_ScatterND, DISABLED_ScatterND_add) -{ - test_layer({N, C, H , W}, "add"); -} +INSTANTIATE_TEST_CASE_P(/**/, Layer_ScatterND, Combine( + Values(std::vector{2, 128, 64, 50}), + Values(std::string("none"), std::string("add")), + dnnBackendsAndTargets(/* withInferenceEngine= */ false, + /* withHalide= */ false, + /* withCpuOCV= */ true, + /* withVkCom= */ false, + /* withCUDA= */ false, + /* withNgraph= */ false, + /* withWebnn= */ false, + /* withCann= */ false) // only test on CPU +)); struct Layer_LayerNorm : public TestBaseWithParam > { @@ -795,8 +787,6 @@ INSTANTIATE_TEST_CASE_P(/**/, Layer_NaryEltwise, testing::Values(std::make_tuple #ifdef HAVE_CUDA INSTANTIATE_TEST_CASE_P(CUDA, Layer_NaryEltwise, testing::Values(std::make_tuple(DNN_BACKEND_CUDA, DNN_TARGET_CUDA))); #endif -// INSTANTIATE_TEST_CASE_P(/**/, Layer_Scatter, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU))); -INSTANTIATE_TEST_CASE_P(/**/, Layer_ScatterND, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU))); INSTANTIATE_TEST_CASE_P(/**/, Layer_LayerNorm, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU))); INSTANTIATE_TEST_CASE_P(/**/, Layer_LayerNormExpanded, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU))); INSTANTIATE_TEST_CASE_P(/**/, Layer_GatherElements, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU))); diff --git a/modules/dnn/src/layers/scatterND_layer.cpp b/modules/dnn/src/layers/scatterND_layer.cpp index 648d35fc0c..531d32f45b 100644 --- a/modules/dnn/src/layers/scatterND_layer.cpp +++ b/modules/dnn/src/layers/scatterND_layer.cpp @@ -89,49 +89,59 @@ public: // NOTE: This impl does not check whether indices have duplicate entries. // The last duplicate entry will overwrite the previous. template - void forward_impl(const Functor& rd, const Mat& data, const Mat& indices, const Mat& updates, Mat& out) - { - data.copyTo(out); + void forward_impl(const Functor &reduce_operation, const Mat &input_mat, const Mat &indices_mat, const Mat &updates_mat, Mat& output_mat) { + input_mat.copyTo(output_mat); - const int* shape = data.size.p; - const size_t* step = data.step.p; + const auto &input_mat_shape = shape(input_mat); + std::vector input_mat_step(input_mat_shape.size()); + for (int i = 0; i < input_mat.dims; i++) { + input_mat_step[i] = static_cast(input_mat.step.p[i] / sizeof(T)); + } - const int ind_ndims = indices.dims; - const int* ind_shape = indices.size.p; - const T* p_indices = indices.ptr(); + const int indices_mat_ndims = indices_mat.dims; + const auto &indices_mat_shape = shape(indices_mat); - const int upd_ndims = updates.dims; - const int* upd_shape = updates.size.p; - const T* p_updates = updates.ptr(); + const int updates_mat_ndims = updates_mat.dims; + const auto &updates_mat_shape = shape(updates_mat); - T* p_out = out.ptr(); - - int k = ind_shape[ind_ndims - 1]; // last dim of indices - size_t total = (size_t)(indices.total() / k); + int indices_last_dim = indices_mat_shape[indices_mat_ndims - 1]; // last dim of indices size_t updates_size = 1; - for (int i = ind_ndims - 1; i < upd_ndims; i++) - updates_size *= upd_shape[i]; + for (int i = indices_mat_ndims - 1; i < updates_mat_ndims; i++) + updates_size *= updates_mat_shape[i]; - size_t inp_start_offset = 0; - size_t ind_start_offset = 0; - size_t upd_start_offset = 0; - for (size_t i = 0; i < total; i++, ind_start_offset += k, upd_start_offset += updates_size) - { - const T* tmp_p_indices = p_indices + ind_start_offset; - inp_start_offset = 0; - for (int j = 0; j < k; j++) - { - CV_Assert(tmp_p_indices[j] < shape[j] && tmp_p_indices[j] > -shape[j]); - inp_start_offset += (((int)tmp_p_indices[j] + shape[j]) % shape[j]) * step[j]; + auto fn = [&](const Range &r) { + size_t input_offset = 0, + indices_offset = r.start * indices_last_dim, + updates_offset = r.start * updates_size; + for (int i = r.start; i < r.end; i++) { + const T* indices = indices_mat.ptr(); + const T* updates = updates_mat.ptr(); + T* output = output_mat.ptr(); + + input_offset = 0; + indices += indices_offset; + for (int j = 0; j < indices_last_dim; j++) { + int index = static_cast(*(indices + j)); + index = (index + input_mat_shape[j]) % input_mat_shape[j]; + CV_Assert(index < input_mat_shape[j] && index >= 0); + input_offset += index * input_mat_step[j]; + } + + updates += updates_offset; + output += input_offset; + for (int j = 0; j < updates_size; j++) { + output[j] = reduce_operation(output[j], updates[j]); + } + + indices_offset += indices_last_dim; + updates_offset += updates_size; } - inp_start_offset /= sizeof(T); + }; - const T* tmp_p_updates = p_updates + upd_start_offset; - T* tmp_p_out = p_out + inp_start_offset; - for (int j = 0; j < updates_size; j++) - tmp_p_out[j] = rd(tmp_p_out[j], tmp_p_updates[j]); - } + size_t total = (size_t)(indices_mat.total() / indices_last_dim); + double nstripes = (size_t)total * (indices_last_dim + updates_size) * (1 / 1024.0); + parallel_for_(Range(0, total), fn, nstripes); } template