Commit d8e9b876 authored by thomie's avatar thomie

Testsuite: cleanup printing of summary

Just use a simple list of tuples, instead of a nested map.

-90 lines of code.
parent 58f0086b
...@@ -320,8 +320,8 @@ if config.list_broken: ...@@ -320,8 +320,8 @@ if config.list_broken:
print(' '.join(map (lambda bdn: '#' + str(bdn[0]) + '(' + bdn[1] + '/' + bdn[2] + ')', brokens))) print(' '.join(map (lambda bdn: '#' + str(bdn[0]) + '(' + bdn[1] + '/' + bdn[2] + ')', brokens)))
print('') print('')
if t.n_framework_failures != 0: if t.framework_failures:
print('WARNING:', str(t.n_framework_failures), 'framework failures!') print('WARNING:', len(framework_failures), 'framework failures!')
print('') print('')
else: else:
# Now run all the tests # Now run all the tests
......
...@@ -131,23 +131,18 @@ class TestRun: ...@@ -131,23 +131,18 @@ class TestRun:
self.start_time = None self.start_time = None
self.total_tests = 0 self.total_tests = 0
self.total_test_cases = 0 self.total_test_cases = 0
self.n_framework_failures = 0
self.framework_failures = {}
self.n_tests_skipped = 0 self.n_tests_skipped = 0
self.tests_skipped = {}
self.n_expected_passes = 0 self.n_expected_passes = 0
self.expected_passes = {}
self.n_expected_failures = 0 self.n_expected_failures = 0
self.expected_failures = {}
self.n_missing_libs = 0 self.missing_libs = []
self.missing_libs = {} self.framework_failures = []
self.n_unexpected_passes = 0
self.unexpected_passes = {} self.unexpected_passes = []
self.n_unexpected_failures = 0 self.unexpected_failures = []
self.unexpected_failures = {} self.unexpected_stat_failures = []
self.n_unexpected_stat_failures = 0
self.unexpected_stat_failures = {}
global t global t
t = TestRun() t = TestRun()
......
...@@ -675,7 +675,7 @@ do_not_copy = ('.hi', '.o', '.dyn_hi', '.dyn_o') # 12112 ...@@ -675,7 +675,7 @@ do_not_copy = ('.hi', '.o', '.dyn_hi', '.dyn_o') # 12112
def test_common_work (name, opts, func, args): def test_common_work (name, opts, func, args):
try: try:
t.total_tests = t.total_tests+1 t.total_tests += 1
setLocalTestOpts(opts) setLocalTestOpts(opts)
package_conf_cache_file_start_timestamp = get_package_cache_timestamp() package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
...@@ -696,7 +696,7 @@ def test_common_work (name, opts, func, args): ...@@ -696,7 +696,7 @@ def test_common_work (name, opts, func, args):
# A test itself can request extra ways by setting opts.extra_ways # A test itself can request extra ways by setting opts.extra_ways
all_ways = all_ways + [way for way in opts.extra_ways if way not in all_ways] all_ways = all_ways + [way for way in opts.extra_ways if way not in all_ways]
t.total_test_cases = t.total_test_cases + len(all_ways) t.total_test_cases += len(all_ways)
ok_way = lambda way: \ ok_way = lambda way: \
not getTestOpts().skip \ not getTestOpts().skip \
...@@ -761,9 +761,7 @@ def test_common_work (name, opts, func, args): ...@@ -761,9 +761,7 @@ def test_common_work (name, opts, func, args):
framework_fail(name, way, str(e)) framework_fail(name, way, str(e))
traceback.print_exc() traceback.print_exc()
for way in all_ways: t.n_tests_skipped += len(set(all_ways) - set(do_ways))
if way not in do_ways:
skiptest (name,way)
if config.cleanup and do_ways: if config.cleanup and do_ways:
cleanup() cleanup()
...@@ -782,8 +780,10 @@ def do_test(name, way, func, args, files): ...@@ -782,8 +780,10 @@ def do_test(name, way, func, args, files):
full_name = name + '(' + way + ')' full_name = name + '(' + way + ')'
if_verbose(2, "=====> {0} {1} of {2} {3}".format( if_verbose(2, "=====> {0} {1} of {2} {3}".format(
full_name, t.total_tests, len(allTestNames), full_name, t.total_tests, len(allTestNames),
[t.n_unexpected_passes, t.n_unexpected_failures, t.n_framework_failures])) [len(t.unexpected_passes),
len(t.unexpected_failures),
len(t.framework_failures)]))
# Clean up prior to the test, so that we can't spuriously conclude # Clean up prior to the test, so that we can't spuriously conclude
# that it passed on the basis of old run outputs. # that it passed on the basis of old run outputs.
...@@ -847,86 +847,38 @@ def do_test(name, way, func, args, files): ...@@ -847,86 +847,38 @@ def do_test(name, way, func, args, files):
except: except:
passFail = 'No passFail found' passFail = 'No passFail found'
directory = re.sub('^\\.[/\\\\]', '', opts.testdir)
if passFail == 'pass': if passFail == 'pass':
if _expect_pass(way): if _expect_pass(way):
t.n_expected_passes = t.n_expected_passes + 1 t.n_expected_passes += 1
if name in t.expected_passes:
t.expected_passes[name].append(way)
else:
t.expected_passes[name] = [way]
else: else:
if_verbose(1, '*** unexpected pass for %s' % full_name) if_verbose(1, '*** unexpected pass for %s' % full_name)
t.n_unexpected_passes = t.n_unexpected_passes + 1 t.unexpected_passes.append((directory, name, 'unexpected', way))
addPassingTestInfo(t.unexpected_passes, opts.testdir, name, way)
elif passFail == 'fail': elif passFail == 'fail':
if _expect_pass(way): if _expect_pass(way):
reason = result['reason'] reason = result['reason']
tag = result.get('tag') tag = result.get('tag')
if tag == 'stat': if tag == 'stat':
if_verbose(1, '*** unexpected stat test failure for %s' % full_name) if_verbose(1, '*** unexpected stat test failure for %s' % full_name)
t.n_unexpected_stat_failures = t.n_unexpected_stat_failures + 1 t.unexpected_stat_failures.append((directory, name, reason, way))
addFailingTestInfo(t.unexpected_stat_failures, opts.testdir, name, reason, way)
else: else:
if_verbose(1, '*** unexpected failure for %s' % full_name) if_verbose(1, '*** unexpected failure for %s' % full_name)
t.n_unexpected_failures = t.n_unexpected_failures + 1 t.unexpected_failures.append((directory, name, reason, way))
addFailingTestInfo(t.unexpected_failures, opts.testdir, name, reason, way)
else: else:
if opts.expect == 'missing-lib': if opts.expect == 'missing-lib':
t.n_missing_libs = t.n_missing_libs + 1 t.missing_libs.append((directory, name, 'missing-lib', way))
if name in t.missing_libs:
t.missing_libs[name].append(way)
else:
t.missing_libs[name] = [way]
else: else:
t.n_expected_failures = t.n_expected_failures + 1 t.n_expected_failures += 1
if name in t.expected_failures:
t.expected_failures[name].append(way)
else:
t.expected_failures[name] = [way]
else: else:
framework_fail(name, way, 'bad result ' + passFail) framework_fail(name, way, 'bad result ' + passFail)
def addPassingTestInfo (testInfos, directory, name, way): def framework_fail(name, way, reason):
directory = re.sub('^\\.[/\\\\]', '', directory) opts = getTestOpts()
directory = re.sub('^\\.[/\\\\]', '', opts.testdir)
if not directory in testInfos:
testInfos[directory] = {}
if not name in testInfos[directory]:
testInfos[directory][name] = []
testInfos[directory][name].append(way)
def addFailingTestInfo (testInfos, directory, name, reason, way):
directory = re.sub('^\\.[/\\\\]', '', directory)
if not directory in testInfos:
testInfos[directory] = {}
if not name in testInfos[directory]:
testInfos[directory][name] = {}
if not reason in testInfos[directory][name]:
testInfos[directory][name][reason] = []
testInfos[directory][name][reason].append(way)
def skiptest (name, way):
# print 'Skipping test \"', name, '\"'
t.n_tests_skipped = t.n_tests_skipped + 1
if name in t.tests_skipped:
t.tests_skipped[name].append(way)
else:
t.tests_skipped[name] = [way]
def framework_fail( name, way, reason ):
full_name = name + '(' + way + ')' full_name = name + '(' + way + ')'
if_verbose(1, '*** framework failure for %s %s ' % (full_name, reason)) if_verbose(1, '*** framework failure for %s %s ' % (full_name, reason))
t.n_framework_failures = t.n_framework_failures + 1 t.framework_failures.append((directory, name, way, reason))
if name in t.framework_failures:
t.framework_failures[name].append(way)
else:
t.framework_failures[name] = [way]
def badResult(result): def badResult(result):
try: try:
...@@ -1907,89 +1859,55 @@ def summary(t, file, short=False): ...@@ -1907,89 +1859,55 @@ def summary(t, file, short=False):
+ repr(t.n_tests_skipped).rjust(8) + repr(t.n_tests_skipped).rjust(8)
+ ' were skipped\n' + ' were skipped\n'
+ '\n' + '\n'
+ repr(t.n_missing_libs).rjust(8) + repr(len(t.missing_libs)).rjust(8)
+ ' had missing libraries\n' + ' had missing libraries\n'
+ repr(t.n_expected_passes).rjust(8) + repr(t.n_expected_passes).rjust(8)
+ ' expected passes\n' + ' expected passes\n'
+ repr(t.n_expected_failures).rjust(8) + repr(t.n_expected_failures).rjust(8)
+ ' expected failures\n' + ' expected failures\n'
+ '\n' + '\n'
+ repr(t.n_framework_failures).rjust(8) + repr(len(t.framework_failures)).rjust(8)
+ ' caused framework failures\n' + ' caused framework failures\n'
+ repr(t.n_unexpected_passes).rjust(8) + repr(len(t.unexpected_passes)).rjust(8)
+ ' unexpected passes\n' + ' unexpected passes\n'
+ repr(t.n_unexpected_failures).rjust(8) + repr(len(t.unexpected_failures)).rjust(8)
+ ' unexpected failures\n' + ' unexpected failures\n'
+ repr(t.n_unexpected_stat_failures).rjust(8) + repr(len(t.unexpected_stat_failures)).rjust(8)
+ ' unexpected stat failures\n' + ' unexpected stat failures\n'
+ '\n') + '\n')
if t.n_unexpected_passes > 0: if t.unexpected_passes:
file.write('Unexpected passes:\n') file.write('Unexpected passes:\n')
printPassingTestInfosSummary(file, t.unexpected_passes) printTestInfosSummary(file, t.unexpected_passes)
if t.n_unexpected_failures > 0: if t.unexpected_failures:
file.write('Unexpected failures:\n') file.write('Unexpected failures:\n')
printFailingTestInfosSummary(file, t.unexpected_failures) printTestInfosSummary(file, t.unexpected_failures)
if t.n_unexpected_stat_failures > 0: if t.unexpected_stat_failures:
file.write('Unexpected stat failures:\n') file.write('Unexpected stat failures:\n')
printFailingTestInfosSummary(file, t.unexpected_stat_failures) printTestInfosSummary(file, t.unexpected_stat_failures)
if t.n_framework_failures > 0: if t.framework_failures:
file.write('Test framework failures:\n') file.write('Framework failures:\n')
printFrameworkFailureSummary(file, t.framework_failures) printTestInfosSummary(file, t.framework_failures)
if stopping(): if stopping():
file.write('WARNING: Testsuite run was terminated early\n') file.write('WARNING: Testsuite run was terminated early\n')
def printUnexpectedTests(file, testInfoss): def printUnexpectedTests(file, testInfoss):
unexpected = [] unexpected = {name for testInfos in testInfoss
for testInfos in testInfoss: for (_, name, _, _) in testInfos}
directories = testInfos.keys() if unexpected:
for directory in directories:
tests = list(testInfos[directory].keys())
unexpected += tests
if unexpected != []:
file.write('Unexpected results from:\n') file.write('Unexpected results from:\n')
file.write('TEST="' + ' '.join(unexpected) + '"\n') file.write('TEST="' + ' '.join(unexpected) + '"\n')
file.write('\n') file.write('\n')
def printPassingTestInfosSummary(file, testInfos): def printTestInfosSummary(file, testInfos):
directories = list(testInfos.keys()) maxDirLen = max(len(directory) for (directory, _, _, _) in testInfos)
directories.sort() for (directory, name, reason, way) in testInfos:
maxDirLen = max(len(x) for x in directories) directory = directory.ljust(maxDirLen)
for directory in directories: file.write(' {directory} {name} [{reason}] ({way})\n'.format(**locals()))
tests = list(testInfos[directory].keys())
tests.sort()
for test in tests:
file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
' (' + ','.join(testInfos[directory][test]) + ')\n')
file.write('\n')
def printFailingTestInfosSummary(file, testInfos):
directories = list(testInfos.keys())
directories.sort()
maxDirLen = max(len(d) for d in directories)
for directory in directories:
tests = list(testInfos[directory].keys())
tests.sort()
for test in tests:
reasons = testInfos[directory][test].keys()
for reason in reasons:
file.write(' ' + directory.ljust(maxDirLen + 2) + test + \
' [' + reason + ']' + \
' (' + ','.join(testInfos[directory][test][reason]) + ')\n')
file.write('\n')
def printFrameworkFailureSummary(file, testInfos):
names = list(testInfos.keys())
names.sort()
maxNameLen = max(len(n) for n in names)
for name in names:
ways = testInfos[name]
file.write(' ' + name.ljust(maxNameLen + 2) + \
' (' + ','.join(ways) + ')\n')
file.write('\n') file.write('\n')
def modify_lines(s, f): def modify_lines(s, f):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment