Commit 6e5dfcd2 authored by Ben Gamari's avatar Ben Gamari 🐢 Committed by Marge Bot

testsuite: Rework tracking of fragile tests

Breaks fragile tests into two groups, allowing us to easily preserve
stdout/stderr of failing fragile tests.
parent 2e031806
......@@ -19,7 +19,8 @@ def junit(t: TestRun) -> ET.ElementTree:
for res_type, group in [('stat failure', t.unexpected_stat_failures),
('unexpected failure', t.unexpected_failures),
('unexpected pass', t.unexpected_passes),
('fragile', t.fragile_results)]:
('fragile failure', t.fragile_failures),
('fragile pass', t.fragile_passes)]:
for tr in group:
testcase = ET.SubElement(testsuite, 'testcase',
classname = tr.way,
......
......@@ -236,7 +236,8 @@ class TestRun:
self.unexpected_stat_failures = [] # type: List[TestResult]
# Results from tests that have been marked as fragile
self.fragile_results = [] # type: List[TestResult]
self.fragile_passes = [] # type: List[TestResult]
self.fragile_failures = [] # type: List[TestResult]
# List of all metrics measured in this test run.
# [(change, PerfStat)] where change is one of the MetricChange
......
......@@ -1008,7 +1008,12 @@ def do_test(name: TestName,
if way in opts.fragile_ways:
if_verbose(1, '*** fragile test %s resulted in %s' % (full_name, passFail))
t.fragile_results.append(TestResult(directory, name, 'fragile %s' % passFail, way))
if passFail == 'pass':
t.fragile_passes.append(TestResult(directory, name, 'fragile', way))
else:
t.fragile_failures.append(TestResult(directory, name, 'fragile', way,
stdout=result.stdout,
stderr=result.stderr))
elif passFail == 'pass':
if _expect_pass(way):
t.expected_passes.append(TestResult(directory, name, "", way))
......@@ -2363,7 +2368,7 @@ def summary(t: TestRun, file: TextIO, short=False, color=False) -> None:
+ ' unexpected failures\n'
+ repr(len(t.unexpected_stat_failures)).rjust(8)
+ ' unexpected stat failures\n'
+ repr(len(t.fragile_results)).rjust(8)
+ repr(len(t.fragile_failures) + len(t.fragile_passes)).rjust(8)
+ ' fragile tests\n'
+ '\n')
......@@ -2387,9 +2392,13 @@ def summary(t: TestRun, file: TextIO, short=False, color=False) -> None:
file.write('Framework warnings:\n')
printTestInfosSummary(file, t.framework_warnings)
if t.fragile_results:
file.write('Fragile tests:\n')
printTestInfosSummary(file, t.fragile_results)
if t.fragile_passes:
file.write('Fragile test passes:\n')
printTestInfosSummary(file, t.fragile_passes)
if t.fragile_failures:
file.write('Fragile test failures:\n')
printTestInfosSummary(file, t.fragile_failures)
if stopping():
file.write('WARNING: Testsuite run was terminated early\n')
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment