Commit 782cacf5 authored by thomie's avatar thomie

Testsuite: framework failure improvements (#11165)

* add framework failures to unexpected results list
* report errors in .T files as framework failures (show in summary)
* don't report missing tests when framework failures in .T files
parent d8e9b876
......@@ -298,20 +298,26 @@ def cleanup_and_exit(exitcode):
exit(exitcode)
# First collect all the tests to be run
t_files_ok = True
for file in t_files:
if_verbose(2, '====> Scanning %s' % file)
newTestDir(tempdir, os.path.dirname(file))
try:
exec(open(file).read())
except Exception:
print('*** framework failure: found an error while executing ', file, ':')
t.n_framework_failures = t.n_framework_failures + 1
except Exception as e:
traceback.print_exc()
framework_fail(file, '', str(e))
t_files_ok = False
if config.only:
# See Note [Mutating config.only]
sys.stderr.write("ERROR: tests not found: {0}\n".format(list(config.only)))
cleanup_and_exit(1)
for name in config.only:
if t_files_ok:
# See Note [Mutating config.only]
framework_fail(name, '', 'test not found')
else:
# Let user fix .T file errors before reporting on unfound tests.
# The reson the test can not be found is likely because of those
# .T file errors.
pass
if config.list_broken:
global brokens
......
......@@ -1841,7 +1841,9 @@ def findTFiles(roots):
def summary(t, file, short=False):
file.write('\n')
printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures, t.unexpected_stat_failures])
printUnexpectedTests(file,
[t.unexpected_passes, t.unexpected_failures,
t.unexpected_stat_failures, t.framework_failures])
if short:
# Only print the list of unexpected tests above.
......@@ -1897,7 +1899,8 @@ def summary(t, file, short=False):
def printUnexpectedTests(file, testInfoss):
unexpected = {name for testInfos in testInfoss
for (_, name, _, _) in testInfos}
for (_, name, _, _) in testInfos
if not name.endswith('.T')}
if unexpected:
file.write('Unexpected results from:\n')
file.write('TEST="' + ' '.join(unexpected) + '"\n')
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment