From 83fd385bbdc581d6aac631005fe2a5c652cff7b3 Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Thu, 4 Jul 2013 18:03:03 +0400 Subject: [PATCH 1/5] xls-report.py: don't eat all errors when reading sheet.conf --- modules/ts/misc/xls-report.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/ts/misc/xls-report.py b/modules/ts/misc/xls-report.py index e911314e92..4e2d8c63b2 100755 --- a/modules/ts/misc/xls-report.py +++ b/modules/ts/misc/xls-report.py @@ -67,6 +67,7 @@ from __future__ import division import ast +import errno import fnmatch import logging import numbers @@ -176,7 +177,8 @@ def main(): try: with open(os.path.join(sheet_path, 'sheet.conf')) as sheet_conf_file: sheet_conf = ast.literal_eval(sheet_conf_file.read()) - except Exception: + except IOError as ioe: + if ioe.errno != errno.ENOENT: raise sheet_conf = {} logging.debug('no sheet.conf for %s', sheet_path) From 4e939e2fce267caa944663d2578a93b807123a0f Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Thu, 4 Jul 2013 18:19:33 +0400 Subject: [PATCH 2/5] xls-report.py: added an option to include data from unrecognized XMLs --- modules/ts/misc/xls-report.py | 52 +++++++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/modules/ts/misc/xls-report.py b/modules/ts/misc/xls-report.py index 4e2d8c63b2..24891ff7c0 100755 --- a/modules/ts/misc/xls-report.py +++ b/modules/ts/misc/xls-report.py @@ -40,9 +40,15 @@ Corollary 2: an empty 'properties' dictionary matches every property set. 3) If a matching matcher is found, its 'name' string is presumed to be the name - of the configuration the XML file corresponds to. Otherwise, a warning is - printed. A warning is also printed if two different property sets match to the - same configuration name. + of the configuration the XML file corresponds to. A warning is printed if + two different property sets match to the same configuration name. + + 4) If a such a matcher isn't found, if --include-unmatched was specified, the + configuration name is assumed to be the relative path from the sheet's + directory to the XML file's containing directory. If the XML file isinstance + directly inside the sheet's directory, the configuration name is instead + a dump of all its properties. If --include-unmatched wasn't specified, + the XML file is ignored and a warning is printed. * 'configurations': [string] List of names for compile-time and runtime configurations of OpenCV. @@ -97,18 +103,25 @@ error_speedup_style = xlwt.easyxf('pattern: pattern solid, fore_color orange') header_style = xlwt.easyxf('font: bold true; alignment: horizontal centre, vertical top, wrap True') class Collector(object): - def __init__(self, config_match_func): + def __init__(self, config_match_func, include_unmatched): self.__config_cache = {} self.config_match_func = config_match_func + self.include_unmatched = include_unmatched self.tests = {} + self.extra_configurations = set() # Format a sorted sequence of pairs as if it was a dictionary. # We can't just use a dictionary instead, since we want to preserve the sorted order of the keys. @staticmethod - def __format_config_cache_key(pairs): - return '{' + ', '.join(repr(k) + ': ' + repr(v) for (k, v) in pairs) + '}' + def __format_config_cache_key(pairs, multiline=False): + return ( + ('{\n' if multiline else '{') + + (',\n' if multiline else ', ').join( + (' ' if multiline else '') + repr(k) + ': ' + repr(v) for (k, v) in pairs) + + ('\n}\n' if multiline else '}') + ) - def collect_from(self, xml_path): + def collect_from(self, xml_path, default_configuration): run = parseLogFile(xml_path) module = run.properties['module_name'] @@ -124,8 +137,17 @@ class Collector(object): configuration = self.config_match_func(properties) if configuration is None: - logging.warning('failed to match properties to a configuration: %s', - Collector.__format_config_cache_key(props_key)) + if self.include_unmatched: + if default_configuration is not None: + configuration = default_configuration + else: + configuration = Collector.__format_config_cache_key(props_key, multiline=True) + + self.extra_configurations.add(configuration) + else: + logging.warning('failed to match properties to a configuration: %s', + Collector.__format_config_cache_key(props_key)) + else: same_config_props = [it[0] for it in self.__config_cache.iteritems() if it[1] == configuration] if len(same_config_props) > 0: @@ -160,6 +182,8 @@ def main(): arg_parser.add_argument('sheet_dirs', nargs='+', metavar='DIR', help='directory containing perf test logs') arg_parser.add_argument('-o', '--output', metavar='XLS', default='report.xls', help='name of output file') arg_parser.add_argument('-c', '--config', metavar='CONF', help='global configuration file') + arg_parser.add_argument('--include-unmatched', action='store_true', + help='include results from XML files that were not recognized by configuration matchers') args = arg_parser.parse_args() @@ -187,12 +211,18 @@ def main(): config_names = sheet_conf.get('configurations', []) config_matchers = sheet_conf.get('configuration_matchers', []) - collector = Collector(make_match_func(config_matchers)) + collector = Collector(make_match_func(config_matchers), args.include_unmatched) for root, _, filenames in os.walk(sheet_path): logging.info('looking in %s', root) for filename in fnmatch.filter(filenames, '*.xml'): - collector.collect_from(os.path.join(root, filename)) + if os.path.normpath(sheet_path) == os.path.normpath(root): + default_conf = None + else: + default_conf = os.path.relpath(root, sheet_path) + collector.collect_from(os.path.join(root, filename), default_conf) + + config_names.extend(sorted(collector.extra_configurations - set(config_names))) sheet = wb.add_sheet(sheet_conf.get('sheet_name', os.path.basename(os.path.abspath(sheet_path)))) From ea054e28885e1190fb81df3fa791a54142553172 Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Thu, 4 Jul 2013 18:44:56 +0400 Subject: [PATCH 3/5] xls-report.py: made most columns a little wider --- modules/ts/misc/xls-report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ts/misc/xls-report.py b/modules/ts/misc/xls-report.py index 24891ff7c0..39e587a465 100755 --- a/modules/ts/misc/xls-report.py +++ b/modules/ts/misc/xls-report.py @@ -235,7 +235,7 @@ def main(): sheet_comparisons = sheet_conf.get('comparisons', []) for i, w in enumerate([2000, 15000, 2500, 2000, 15000] - + (len(config_names) + 1 + len(sheet_comparisons)) * [3000]): + + (len(config_names) + 1 + len(sheet_comparisons)) * [4000]): sheet.col(i).width = w for i, caption in enumerate(['Module', 'Test', 'Image\nsize', 'Data\ntype', 'Parameters'] From 0732905cff6ebad5afb86303355221b8817ec182 Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Thu, 4 Jul 2013 18:52:10 +0400 Subject: [PATCH 4/5] xls-report.py: write tests in sorted order --- modules/ts/misc/xls-report.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/ts/misc/xls-report.py b/modules/ts/misc/xls-report.py index 39e587a465..0aeb1c3c0b 100755 --- a/modules/ts/misc/xls-report.py +++ b/modules/ts/misc/xls-report.py @@ -81,7 +81,6 @@ import os, os.path import re from argparse import ArgumentParser -from collections import OrderedDict from glob import glob from itertools import ifilter @@ -160,7 +159,7 @@ class Collector(object): if configuration is None: return - module_tests = self.tests.setdefault(module, OrderedDict()) + module_tests = self.tests.setdefault(module, {}) for test in run.tests: test_results = module_tests.setdefault((test.shortName(), test.param()), {}) @@ -250,7 +249,7 @@ def main(): for module, color in module_colors.iteritems()} for module, tests in sorted(collector.tests.iteritems()): - for ((test, param), configs) in tests.iteritems(): + for ((test, param), configs) in sorted(tests.iteritems()): sheet.write(row, 0, module, module_styles.get(module, xlwt.Style.default_style)) sheet.write(row, 1, test) From 2b441720265f2e1f530394ac072169a7960cd96b Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Thu, 4 Jul 2013 19:59:26 +0400 Subject: [PATCH 5/5] xls-report.py: make behaviour more intelligent in case of repeated results --- modules/ts/misc/xls-report.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/modules/ts/misc/xls-report.py b/modules/ts/misc/xls-report.py index 0aeb1c3c0b..131f3fab59 100755 --- a/modules/ts/misc/xls-report.py +++ b/modules/ts/misc/xls-report.py @@ -163,7 +163,13 @@ class Collector(object): for test in run.tests: test_results = module_tests.setdefault((test.shortName(), test.param()), {}) - test_results[configuration] = test.get("gmean") if test.status == 'run' else test.status + new_result = test.get("gmean") if test.status == 'run' else test.status + test_results[configuration] = min( + test_results.get(configuration), new_result, + key=lambda r: (1, r) if isinstance(r, numbers.Number) else + (2,) if r is not None else + (3,) + ) # prefer lower result; prefer numbers to errors and errors to nothing def make_match_func(matchers): def match_func(properties):