Merge remote-tracking branch 'upstream/3.4' into merge-3.4

modules/ts/src/precomp.hpp

@@ -1,4 +1,6 @@
 #include "opencv2/ts.hpp"
+#include <opencv2/core/utils/logger.hpp>
+#include "opencv2/core/utility.hpp"
 #include "opencv2/core/private.hpp"
 
 #ifdef GTEST_LINKED_AS_SHARED_LIBRARY

modules/ts/src/ts.cpp

@@ -91,11 +91,31 @@
 
 #include "opencv2/core/opencl/opencl_info.hpp"
 
+#include "opencv2/core/utils/allocator_stats.hpp"
+namespace cv { namespace ocl {
+cv::utils::AllocatorStatisticsInterface& getOpenCLAllocatorStatistics();
+}}
 #endif // HAVE_OPENCL
 
-#include "opencv2/core/utility.hpp"
+#include "opencv2/core/utils/allocator_stats.hpp"
+namespace cv {
+CV_EXPORTS cv::utils::AllocatorStatisticsInterface& getAllocatorStatistics();
+}
+
 #include "opencv_tests_config.hpp"
 
+#include "ts_tags.hpp"
+
+#if defined(__GNUC__) && defined(__linux__)
+extern "C" {
+size_t malloc_peak(void) __attribute__((weak));
+void malloc_reset_peak(void) __attribute__((weak));
+} // extern "C"
+#else // stubs
+static size_t (*malloc_peak)(void) = 0;
+static void (*malloc_reset_peak)(void) = 0;
+#endif
+
 namespace opencv_test {
 bool required_opencv_test_namespace = false;  // compilation check for non-refactored tests
 }
@@ -726,6 +746,85 @@ bool skipUnstableTests = false;
 bool runBigDataTests = false;
 int testThreads = 0;
 
+
+static size_t memory_usage_base = 0;
+static uint64_t memory_usage_base_opencv = 0;
+#ifdef HAVE_OPENCL
+static uint64_t memory_usage_base_opencl = 0;
+#endif
+
+void testSetUp()
+{
+    cv::ipp::setIppStatus(0);
+    cv::theRNG().state = cvtest::param_seed;
+    cv::setNumThreads(cvtest::testThreads);
+    if (malloc_peak)  // if memory profiler is available
+    {
+        malloc_reset_peak();
+        memory_usage_base = malloc_peak(); // equal to malloc_current()
+    }
+    {
+        cv::utils::AllocatorStatisticsInterface& ocv_stats = cv::getAllocatorStatistics();
+        ocv_stats.resetPeakUsage();
+        memory_usage_base_opencv = ocv_stats.getCurrentUsage();
+    }
+#ifdef HAVE_OPENCL
+    {
+        cv::utils::AllocatorStatisticsInterface& ocl_stats = cv::ocl::getOpenCLAllocatorStatistics();
+        ocl_stats.resetPeakUsage();
+        memory_usage_base_opencl = ocl_stats.getCurrentUsage();
+    }
+#endif
+    checkTestTags();
+}
+
+void testTearDown()
+{
+    ::cvtest::checkIppStatus();
+    uint64_t memory_usage = 0;
+    uint64_t ocv_memory_usage = 0, ocv_peak = 0;
+    if (malloc_peak)  // if memory profiler is available
+    {
+        size_t peak = malloc_peak();
+        memory_usage = peak - memory_usage_base;
+        CV_LOG_INFO(NULL, "Memory_usage (malloc): " << memory_usage << " (base=" << memory_usage_base << ")");
+    }
+    {
+        // core/src/alloc.cpp: #define OPENCV_ALLOC_ENABLE_STATISTICS
+        // handle large buffers via fastAlloc()
+        // (not always accurate on heavy 3rdparty usage, like protobuf)
+        cv::utils::AllocatorStatisticsInterface& ocv_stats = cv::getAllocatorStatistics();
+        ocv_peak = ocv_stats.getPeakUsage();
+        ocv_memory_usage = ocv_peak - memory_usage_base_opencv;
+        CV_LOG_INFO(NULL, "Memory_usage (OpenCV): " << ocv_memory_usage << " (base=" << memory_usage_base_opencv << "  current=" << ocv_stats.getCurrentUsage() << ")");
+        if (memory_usage == 0)  // external profiler has higher priority (and accuracy)
+            memory_usage = ocv_memory_usage;
+    }
+#ifdef HAVE_OPENCL
+    uint64_t ocl_memory_usage = 0, ocl_peak = 0;
+    {
+        cv::utils::AllocatorStatisticsInterface& ocl_stats = cv::ocl::getOpenCLAllocatorStatistics();
+        ocl_peak = ocl_stats.getPeakUsage();
+        ocl_memory_usage = ocl_peak - memory_usage_base_opencl;
+        CV_LOG_INFO(NULL, "Memory_usage (OpenCL): " << ocl_memory_usage << " (base=" << memory_usage_base_opencl << "  current=" << ocl_stats.getCurrentUsage() << ")");
+        ::testing::Test::RecordProperty("ocl_memory_usage",
+                cv::format("%llu", (unsigned long long)ocl_memory_usage));
+    }
+#else
+    uint64_t ocl_memory_usage = 0;
+#endif
+    if (malloc_peak      // external memory profiler is available
+        || ocv_peak > 0  // or enabled OpenCV builtin allocation statistics
+    )
+    {
+        CV_LOG_INFO(NULL, "Memory usage total: " << (memory_usage + ocl_memory_usage));
+        ::testing::Test::RecordProperty("memory_usage",
+                cv::format("%llu", (unsigned long long)memory_usage));
+        ::testing::Test::RecordProperty("total_memory_usage",
+                cv::format("%llu", (unsigned long long)(memory_usage + ocl_memory_usage)));
+    }
+}
+
 void parseCustomOptions(int argc, char **argv)
 {
     const char * const command_line_keys =
@@ -735,7 +834,9 @@ void parseCustomOptions(int argc, char **argv)
         "{ skip_unstable      |false    |skip unstable tests }"
         "{ test_bigdata       |false    |run BigData tests (>=2Gb) }"
         "{ test_require_data  |false    |fail on missing non-required test data instead of skip}"
-        "{ h   help           |false    |print help info                          }";
+        CV_TEST_TAGS_PARAMS
+        "{ h   help           |false    |print help info                          }"
+    ;
 
     cv::CommandLineParser parser(argc, argv, command_line_keys);
     if (parser.get<bool>("help"))
@@ -759,8 +860,9 @@ void parseCustomOptions(int argc, char **argv)
     skipUnstableTests = parser.get<bool>("skip_unstable");
     runBigDataTests = parser.get<bool>("test_bigdata");
     checkTestData = parser.get<bool>("test_require_data");
-}
 
+    activateTestTags(parser);
+}
 
 static bool isDirectory(const std::string& path)
 {

modules/ts/src/ts_perf.cpp

@@ -1,5 +1,7 @@
 #include "precomp.hpp"
 
+#include "ts_tags.hpp"
+
 #include <map>
 #include <iostream>
 #include <fstream>
@@ -999,6 +1001,8 @@ void TestBase::Init(const std::vector<std::string> & availableImpls,
         "{   perf_cuda_info_only         |false    |print an information about system and an available CUDA devices and then exit.}"
 #endif
         "{ skip_unstable                 |false    |skip unstable tests }"
+
+        CV_TEST_TAGS_PARAMS
     ;
 
     cv::CommandLineParser args(argc, argv, command_line_keys);
@@ -1145,6 +1149,8 @@ void TestBase::Init(const std::vector<std::string> & availableImpls,
         ::testing::AddGlobalTestEnvironment(new PerfValidationEnvironment());
     }
 
+    activateTestTags(args);
+
     if (!args.check())
     {
         args.printErrors();
@@ -1869,14 +1875,15 @@ void TestBase::SetUp()
     currentIter = (unsigned int)-1;
     timeLimit = timeLimitDefault;
     times.clear();
+    metrics.terminationReason = performance_metrics::TERM_SKIP_TEST;
 }
 
 void TestBase::TearDown()
 {
     if (metrics.terminationReason == performance_metrics::TERM_SKIP_TEST)
     {
-        LOGI("\tTest was skipped");
-        GTEST_SUCCEED() << "Test was skipped";
+        //LOGI("\tTest was skipped");
+        //GTEST_SUCCEED() << "Test was skipped";
     }
     else
     {
@@ -1975,6 +1982,7 @@ std::string TestBase::getDataPath(const std::string& relativePath)
 
 void TestBase::RunPerfTestBody()
 {
+    metrics.clear();
     try
     {
 #ifdef CV_COLLECT_IMPL_DATA
@@ -1990,7 +1998,7 @@ void TestBase::RunPerfTestBody()
     catch(const SkipTestException&)
     {
         metrics.terminationReason = performance_metrics::TERM_SKIP_TEST;
-        return;
+        throw;
     }
     catch(const PerfSkipTestException&)
     {

modules/ts/src/ts_tags.cpp

@@ -0,0 +1,471 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#include "precomp.hpp"
+
+#include "ts_tags.hpp"
+
+namespace cvtest {
+
+static bool printTestTag = false;
+
+static std::vector<std::string> currentDirectTestTags, currentImpliedTestTags;
+static std::vector<const ::testing::TestInfo*> skipped_tests;
+
+static std::vector<std::string>& getTestTagsSkipList()
+{
+    static std::vector<std::string> testSkipWithTags;
+    static bool initialized = false;
+    if (!initialized)
+    {
+#if OPENCV_32BIT_CONFIGURATION
+        testSkipWithTags.push_back(CV_TEST_TAG_MEMORY_2GB);
+#else
+        testSkipWithTags.push_back(CV_TEST_TAG_MEMORY_6GB);
+#endif
+        testSkipWithTags.push_back(CV_TEST_TAG_VERYLONG);
+#if defined(_DEBUG)
+        testSkipWithTags.push_back(CV_TEST_TAG_DEBUG_VERYLONG);
+#endif
+        initialized = true;
+    }
+    return testSkipWithTags;
+}
+
+static std::vector<std::string>& getTestTagsForceList()
+{
+    static std::vector<std::string> getTestTagsForceList;
+    return getTestTagsForceList;
+}
+
+static std::vector<std::string>& getTestTagsRequiredList()
+{
+    static std::vector<std::string> getTestTagsRequiredList;
+    return getTestTagsRequiredList;
+}
+
+
+class TestTagsListener: public ::testing::EmptyTestEventListener
+{
+public:
+    void OnTestProgramStart(const ::testing::UnitTest& /*unit_test*/) CV_OVERRIDE
+    {
+        {
+            const std::vector<std::string>& tags = getTestTagsRequiredList();
+            std::ostringstream os, os_direct;
+            for (size_t i = 0; i < tags.size(); i++)
+            {
+                os << (i == 0 ? "'" : ", '") << tags[i] << "'";
+                os_direct << (i == 0 ? "" : ",") << tags[i];
+            }
+            std::string tags_str = os.str();
+            if (!tags.empty())
+                std::cout << "TEST: Run tests with tags: " << tags_str << std::endl;
+            ::testing::Test::RecordProperty("test_tags", os_direct.str());
+        }
+        {
+            const std::vector<std::string>& tags = getTestTagsSkipList();
+            std::ostringstream os, os_direct;
+            for (size_t i = 0; i < tags.size(); i++)
+            {
+                os << (i == 0 ? "'" : ", '") << tags[i] << "'";
+                os_direct << (i == 0 ? "" : ",") << tags[i];
+            }
+            std::string tags_str = os.str();
+            if (!tags.empty())
+                std::cout << "TEST: Skip tests with tags: " << tags_str << std::endl;
+            ::testing::Test::RecordProperty("test_tags_skip", os_direct.str());
+        }
+        {
+            const std::vector<std::string>& tags = getTestTagsForceList();
+            std::ostringstream os, os_direct;
+            for (size_t i = 0; i < tags.size(); i++)
+            {
+                os << (i == 0 ? "'" : ", '") << tags[i] << "'";
+                os_direct << (i == 0 ? "" : ",") << tags[i];
+            }
+            std::string tags_str = os.str();
+            if (!tags.empty())
+                std::cout << "TEST: Force tests with tags: " << tags_str << std::endl;
+            ::testing::Test::RecordProperty("test_tags_force", os_direct.str());
+        }
+    }
+
+    void OnTestStart(const ::testing::TestInfo& test_info) CV_OVERRIDE
+    {
+        currentDirectTestTags.clear();
+        currentImpliedTestTags.clear();
+
+        const char* value_param_ = test_info.value_param();
+        if (value_param_)
+        {
+            std::string value_param(value_param_);
+            if (value_param.find("CV_64F") != std::string::npos
+                || (value_param.find("64F") != std::string::npos
+                    && value_param.find(" 64F") != std::string::npos
+                    && value_param.find(",64F") != std::string::npos
+                    && value_param.find("(64F") != std::string::npos
+                )
+            )
+                applyTestTag_(CV_TEST_TAG_TYPE_64F);
+            if (value_param.find("1280x720") != std::string::npos)
+                applyTestTag_(CV_TEST_TAG_SIZE_HD);
+            if (value_param.find("1920x1080") != std::string::npos)
+                applyTestTag_(CV_TEST_TAG_SIZE_FULLHD);
+            if (value_param.find("3840x2160") != std::string::npos)
+                applyTestTag_(CV_TEST_TAG_SIZE_4K);
+        }
+    }
+
+    void OnTestEnd(const ::testing::TestInfo& /*test_info*/) CV_OVERRIDE
+    {
+        if (currentDirectTestTags.empty() && currentImpliedTestTags.empty())
+        {
+            if (printTestTag) std::cout << "[     TAGS ] No tags" << std::endl;
+            return;
+        }
+        std::ostringstream os;
+        std::ostringstream os_direct;
+        std::ostringstream os_implied;
+        {
+            const std::vector<std::string>& tags = currentDirectTestTags;
+            for (size_t i = 0; i < tags.size(); i++)
+            {
+                os << (i == 0 ? "" : ", ") << tags[i];
+                os_direct << (i == 0 ? "" : ",") << tags[i];
+            }
+        }
+        if (!currentImpliedTestTags.empty())
+        {
+            os << " (implied tags: ";
+            const std::vector<std::string>& tags = currentImpliedTestTags;
+            for (size_t i = 0; i < tags.size(); i++)
+            {
+                os << (i == 0 ? "" : ", ") << tags[i];
+                os_implied << (i == 0 ? "" : ",") << tags[i];
+            }
+            os << ")";
+        }
+        if (printTestTag) std::cout << "[     TAGS ] " << os.str() << std::endl;
+        ::testing::Test::RecordProperty("tags", os_direct.str());
+        ::testing::Test::RecordProperty("tags_implied", os_implied.str());
+    }
+
+    void OnTestIterationEnd(const ::testing::UnitTest& /*unit_test*/, int /*iteration*/) CV_OVERRIDE
+    {
+        if (!skipped_tests.empty())
+        {
+            std::cout << "[   SKIP   ] " << skipped_tests.size() << " tests via tags" << std::endl;
+        }
+        skipped_tests.clear();
+    }
+
+    void OnTestProgramEnd(const ::testing::UnitTest& /*unit_test*/) CV_OVERRIDE
+    {
+        /*if (!skipped_tests.empty())
+        {
+            for (size_t i = 0; i < skipped_tests.size(); i++)
+            {
+                const ::testing::TestInfo* test_info = skipped_tests[i];
+                if (!test_info) continue;
+                std::cout << "- " << test_info->test_case_name() << "." << test_info->name() << std::endl;
+            }
+        }*/
+    }
+};
+
+static bool isTestTagForced(const std::string& testTag)
+{
+    const std::vector<std::string>& forceTags = getTestTagsForceList();
+    for (size_t i = 0; i < forceTags.size(); ++i)
+    {
+        const std::string& forceTag = forceTags[i];
+        if (testTag == forceTag
+            || (testTag.size() >= forceTag.size()
+                && forceTag[forceTag.size() - 1] == '*'
+                && forceTag.substr(0, forceTag.size() - 1) == testTag.substr(0, forceTag.size() - 1)
+            )
+        )
+        {
+            return true;
+        }
+    }
+    return false;
+}
+
+static bool isTestTagSkipped(const std::string& testTag, CV_OUT std::string& skippedByTag)
+{
+    skippedByTag.clear();
+    const std::vector<std::string>& skipTags = getTestTagsSkipList();
+    for (size_t i = 0; i < skipTags.size(); ++i)
+    {
+        const std::string& skipTag = skipTags[i];
+        if (testTag == skipTag
+            || (testTag.size() >= skipTag.size()
+                && skipTag[skipTag.size() - 1] == '*'
+                && skipTag.substr(0, skipTag.size() - 1) == testTag.substr(0, skipTag.size() - 1)
+            )
+        )
+        {
+            skippedByTag = skipTag;
+            return true;
+        }
+    }
+    return false;
+}
+
+void checkTestTags()
+{
+    std::string skipTag;
+    const std::vector<std::string>& testTags = currentDirectTestTags;
+    {
+        const std::vector<std::string>& tags = getTestTagsRequiredList();
+        if (!tags.empty())
+        {
+            size_t found = 0;
+            for (size_t i = 0; i < tags.size(); ++i)
+            {
+                const std::string& tag = tags[i];
+                for (size_t j = 0; j < testTags.size(); ++j)
+                {
+                    const std::string& testTag = testTags[i];
+                    if (testTag == tag
+                        || (testTag.size() >= tag.size()
+                            && tag[tag.size() - 1] == '*'
+                            && tag.substr(0, tag.size() - 1) == testTag.substr(0, tag.size() - 1)
+                        )
+                    )
+                    {
+                        found++;
+                        break;
+                    }
+                }
+            }
+            if (found != tags.size())
+            {
+                skipped_tests.push_back(::testing::UnitTest::GetInstance()->current_test_info());
+                throw SkipTestException("Test tags don't pass required tags list (--test_tag parameter)");
+            }
+        }
+    }
+    for (size_t i = 0; i < testTags.size(); ++i)
+    {
+        const std::string& testTag = testTags[i];
+        if (isTestTagForced(testTag))
+            return;
+    }
+    for (size_t i = 0; i < testTags.size(); ++i)
+    {
+        const std::string& testTag = testTags[i];
+        if (isTestTagSkipped(testTag, skipTag))
+        {
+            skipped_tests.push_back(::testing::UnitTest::GetInstance()->current_test_info());
+            throw SkipTestException("Test with tag '" + testTag + "' is skipped ('" + skipTag + "' is in skip list)");
+        }
+    }
+    const std::vector<std::string>& testTagsImplied = currentImpliedTestTags;
+    for (size_t i = 0; i < testTagsImplied.size(); ++i)
+    {
+        const std::string& testTag = testTagsImplied[i];
+        if (isTestTagSkipped(testTag, skipTag))
+        {
+            skipped_tests.push_back(::testing::UnitTest::GetInstance()->current_test_info());
+            throw SkipTestException("Test with tag '" + testTag + "' is skipped ('" + skipTag + "' is in skip list)");
+        }
+    }
+}
+
+static bool applyTestTagImpl(const std::string& tag, bool direct = false)
+{
+    CV_Assert(!tag.empty());
+    std::vector<std::string>& testTags = direct ? currentDirectTestTags : currentImpliedTestTags;
+    for (size_t i = 0; i < testTags.size(); ++i)
+    {
+        const std::string& testTag = testTags[i];
+        if (tag == testTag)
+        {
+            return false;  // already exists, skip
+        }
+    }
+    testTags.push_back(tag);
+
+    // Tags implies logic
+    if (tag == CV_TEST_TAG_MEMORY_14GB)
+        applyTestTagImpl(CV_TEST_TAG_MEMORY_6GB);
+    if (tag == CV_TEST_TAG_MEMORY_6GB)
+        applyTestTagImpl(CV_TEST_TAG_MEMORY_2GB);
+    if (tag == CV_TEST_TAG_MEMORY_2GB)
+        applyTestTagImpl(CV_TEST_TAG_MEMORY_1GB);
+    if (tag == CV_TEST_TAG_MEMORY_1GB)
+        applyTestTagImpl(CV_TEST_TAG_MEMORY_512MB);
+    if (tag == CV_TEST_TAG_VERYLONG)
+    {
+        applyTestTagImpl(CV_TEST_TAG_DEBUG_VERYLONG);
+        applyTestTagImpl(CV_TEST_TAG_LONG);
+    }
+    else if (tag == CV_TEST_TAG_DEBUG_VERYLONG)
+    {
+        applyTestTagImpl(CV_TEST_TAG_DEBUG_LONG);
+    }
+    else if (tag == CV_TEST_TAG_LONG)
+    {
+        applyTestTagImpl(CV_TEST_TAG_DEBUG_LONG);
+    }
+
+    if (tag == CV_TEST_TAG_SIZE_4K)
+        applyTestTagImpl(CV_TEST_TAG_SIZE_FULLHD);
+    if (tag == CV_TEST_TAG_SIZE_FULLHD)
+        applyTestTagImpl(CV_TEST_TAG_SIZE_HD);
+
+    return true;
+}
+
+void applyTestTag(const std::string& tag)
+{
+    if (tag.empty()) return;
+    if (!applyTestTagImpl(tag, true))
+        return;
+    checkTestTags();
+}
+
+void applyTestTag_(const std::string& tag)
+{
+    if (tag.empty()) return;
+    if (!applyTestTagImpl(tag, true))
+        return;
+}
+
+static std::vector<std::string> parseStringList(const std::string& s)
+{
+    std::vector<std::string> result;
+    size_t start_pos = 0;
+    while (start_pos != std::string::npos)
+    {
+        while (start_pos < s.size() && s[start_pos] == ' ')
+            start_pos++;
+        const size_t pos_ = s.find(',', start_pos);
+        size_t pos = (pos_ == std::string::npos ? s.size() : pos_);
+        while (pos > start_pos && s[pos - 1] == ' ')
+            pos--;
+        if (pos > start_pos)
+        {
+            const std::string one_piece(s, start_pos, pos - start_pos);
+            result.push_back(one_piece);
+        }
+        start_pos = (pos_ == std::string::npos ? pos_ : pos_ + 1);
+    }
+    return result;
+
+}
+
+void activateTestTags(const cv::CommandLineParser& parser)
+{
+    std::string test_tag_skip = parser.get<std::string>("test_tag_skip");
+    if (!test_tag_skip.empty())
+    {
+        const std::vector<std::string> tag_list = parseStringList(test_tag_skip);
+        if (!tag_list.empty())
+        {
+            std::vector<std::string>& skipTags = getTestTagsSkipList();
+            for (size_t k = 0; k < tag_list.size(); ++k)
+            {
+                const std::string& tag = tag_list[k];
+                bool found = false;
+                for (size_t i = 0; i < skipTags.size(); ++i)
+                {
+                    if (tag == skipTags[i])
+                    {
+                        found = true;
+                        break;
+                    }
+                }
+                if (!found)
+                    skipTags.push_back(tag);
+            }
+        }
+    }
+
+    std::string test_tag_enable = parser.get<std::string>("test_tag_enable");
+    if (!test_tag_enable.empty())
+    {
+        const std::vector<std::string> tag_list = parseStringList(test_tag_enable);
+        if (!tag_list.empty())
+        {
+            std::vector<std::string>& skipTags = getTestTagsSkipList();
+            for (size_t k = 0; k < tag_list.size(); ++k)
+            {
+                const std::string& tag = tag_list[k];
+                bool found = false;
+                for (size_t i = 0; i < skipTags.size(); ++i)
+                {
+                    if (tag == skipTags[i])
+                    {
+                        skipTags.erase(skipTags.begin() + i);
+                        found = true;
+                    }
+                }
+                if (!found)
+                {
+                    std::cerr << "Can't re-enable tag '" << tag << "' - it is not in the skip list" << std::endl;
+                }
+            }
+        }
+    }
+
+    std::string test_tag_force = parser.get<std::string>("test_tag_force");
+    if (!test_tag_force.empty())
+    {
+        const std::vector<std::string> tag_list = parseStringList(test_tag_force);
+        if (!tag_list.empty())
+        {
+            std::vector<std::string>& forceTags = getTestTagsForceList();
+            for (size_t k = 0; k < tag_list.size(); ++k)
+            {
+                const std::string& tag = tag_list[k];
+                bool found = false;
+                for (size_t i = 0; i < forceTags.size(); ++i)
+                {
+                    if (tag == forceTags[i])
+                    {
+                        found = true;
+                        break;
+                    }
+                }
+                if (!found)
+                    forceTags.push_back(tag);
+            }
+        }
+    }
+
+    std::string test_tag = parser.get<std::string>("test_tag");
+    if (!test_tag.empty())
+    {
+        const std::vector<std::string> tag_list = parseStringList(test_tag);
+        if (!tag_list.empty())
+        {
+            std::vector<std::string>& requiredTags = getTestTagsRequiredList();
+            for (size_t k = 0; k < tag_list.size(); ++k)
+            {
+                const std::string& tag = tag_list[k];
+                bool found = false;
+                for (size_t i = 0; i < requiredTags.size(); ++i)
+                {
+                    if (tag == requiredTags[i])
+                    {
+                        found = true;
+                        break;
+                    }
+                }
+                if (!found)
+                    requiredTags.push_back(tag);
+            }
+        }
+    }
+
+    printTestTag = parser.get<bool>("test_tag_print");
+
+    ::testing::UnitTest::GetInstance()->listeners().Append(new TestTagsListener());
+}
+
+} // namespace

modules/ts/src/ts_tags.hpp

@@ -0,0 +1,26 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#ifndef OPENCV_TS_SRC_TAGS_HPP
+#define OPENCV_TS_SRC_TAGS_HPP
+
+// [all | test_tag] - (test_tag_skip - test_tag_enable) + test_tag_force
+
+#define CV_TEST_TAGS_PARAMS \
+    "{ test_tag           |         |run tests with specified 'tag' markers only (comma ',' separated list) }" \
+    "{ test_tag_skip      |         |skip tests with 'tag' markers (comma ',' separated list) }" \
+    "{ test_tag_enable    |         |don't skip tests with 'tag' markers (comma ',' separated list) }" \
+    "{ test_tag_force     |         |force running of tests with 'tag' markers (comma ',' separated list) }" \
+    "{ test_tag_print     | false   |print assigned tags for each test }" \
+
+// TODO
+//  "{ test_tag_file      |         |read test tags assignment }" \
+
+namespace cvtest {
+
+void activateTestTags(const cv::CommandLineParser& parser);
+
+} // namespace
+
+#endif // OPENCV_TS_SRC_TAGS_HPP