Commit 1e246965 authored by Joachim Breitner's avatar Joachim Breitner
Browse files

Test driver: Implement different verbosities

Select verbosity with "make VERBOSE=n". Options so far:
 n=0: No per-test output
 n=1: Only failing test results
 n=2: As above, plus progress information (names of all tests)
 n=3: As aobve, plus commands called.
Default currently is n=3, although n=2 might be a nicer default.
parent 33dfd925
......@@ -13,6 +13,9 @@ Commands to run testsuite:
* Run a specific test: `make TEST=tc054`
* Test a specific 'way': `make WAY=optllvm`
* Test a specifc stage of GHC: `make stage=1`
* Set verbosity: `make VERBOSE=n`
where n=0: No per-test ouput, n=1: Only failures,
n=2: Progress output, n=3: Include commands called (default)
* Pass in extra GHC options: `make EXTRA_HC_OPTS=-fvectorize`
You can also change directory to a specific test folder to run that
......
......@@ -44,6 +44,7 @@ long_options = [
"skipway=", # skip this way
"threads=", # threads to run simultaneously
"check-files-written", # check files aren't written by multiple tests
"verbose=", # verbose (0,1,2 so far)
]
opts, args = getopt.getopt(sys.argv[1:], "e:", long_options)
......@@ -92,6 +93,13 @@ for opt,arg in opts:
if opt == '--check-files-written':
config.check_files_written = True
if opt == '--verbose':
if arg not in ["0","1","2","3"]:
sys.stderr.write("ERROR: requested verbosity %s not supported, use 0,1,2 or 3" % arg)
sys.exit(1)
config.verbose = int(arg)
if config.use_threads == 1:
# Trac #1558 says threads don't work in python 2.4.4, but do
# in 2.5.2. Probably >= 2.5 is sufficient, but let's be
......@@ -242,7 +250,7 @@ sys.stdout = os.fdopen(sys.__stdout__.fileno(), "w", 0)
# First collect all the tests to be run
for file in t_files:
print '====> Scanning', file
if_verbose(2, '====> Scanning %s' % file)
newTestDir(os.path.dirname(file))
try:
execfile(file)
......
......@@ -43,7 +43,7 @@ class TestConfig:
self.wordsize = ''
# Verbosity level
self.verbose = 1
self.verbose = 3
# run the "fast" version of the test suite
self.fast = 0
......
......@@ -712,10 +712,11 @@ def do_test(name, way, func, args):
full_name = name + '(' + way + ')'
try:
print '=====>', full_name, t.total_tests, 'of', len(allTestNames), \
str([t.n_unexpected_passes, \
t.n_unexpected_failures, \
t.n_framework_failures])
if_verbose(2, "=====> %s %d of %d %s " % \
(full_name, t.total_tests, len(allTestNames), \
[t.n_unexpected_passes, \
t.n_unexpected_failures, \
t.n_framework_failures]))
if config.use_threads:
t.lock.release()
......@@ -754,13 +755,13 @@ def do_test(name, way, func, args):
else:
t.expected_passes[name] = [way]
else:
print '*** unexpected pass for', full_name
if_verbose(1, '*** unexpected pass for %s' % full_name)
t.n_unexpected_passes = t.n_unexpected_passes + 1
addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
elif passFail == 'fail':
if getTestOpts().expect == 'pass' \
and way not in getTestOpts().expect_fail_for:
print '*** unexpected failure for', full_name
if_verbose(1, '*** unexpected failure for %s' % full_name)
t.n_unexpected_failures = t.n_unexpected_failures + 1
reason = result['reason']
addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
......@@ -820,7 +821,7 @@ def skiptest (name, way):
def framework_fail( name, way, reason ):
full_name = name + '(' + way + ')'
print '*** framework failure for', full_name, reason
if_verbose(1, '*** framework failure for %s %s ' %s (full_name, reason))
t.n_framework_failures = t.n_framework_failures + 1
if name in t.framework_failures:
t.framework_failures[name].append(way)
......@@ -1595,7 +1596,7 @@ def compare_outputs( kind, normaliser, expected_file, actual_file ):
if expected_str == actual_str:
return 1
else:
print 'Actual ' + kind + ' output differs from expected:'
if_verbose(1, 'Actual ' + kind + ' output differs from expected:')
if expected_file_for_diff == '/dev/null':
expected_normalised_file = '/dev/null'
......@@ -1614,17 +1615,18 @@ def compare_outputs( kind, normaliser, expected_file, actual_file ):
# (including newlines) so the diff would be hard to read.
# This does mean that the diff might contain changes that
# would be normalised away.
r = os.system( 'diff -uw ' + expected_file_for_diff + \
' ' + actual_file )
if (config.verbose >= 1):
r = os.system( 'diff -uw ' + expected_file_for_diff + \
' ' + actual_file )
# If for some reason there were no non-whitespace differences,
# then do a full diff
if r == 0:
r = os.system( 'diff -u ' + expected_file_for_diff + \
' ' + actual_file )
# If for some reason there were no non-whitespace differences,
# then do a full diff
if r == 0:
r = os.system( 'diff -u ' + expected_file_for_diff + \
' ' + actual_file )
if config.accept:
print 'Accepting new output.'
if_verbose(1, 'Accepting new output.')
write_file(expected_file, actual_raw)
return 1
else:
......@@ -1766,7 +1768,7 @@ def rawSystemWithTimeout(cmd_and_args):
# Then, when using the native Python, os.system will invoke the cmd shell
def runCmd( cmd ):
if_verbose( 1, cmd )
if_verbose( 3, cmd )
r = 0
if config.os == 'mingw32':
# On MinGW, we will always have timeout
......@@ -1779,7 +1781,7 @@ def runCmd( cmd ):
return r << 8
def runCmdFor( name, cmd, timeout_multiplier=1.0 ):
if_verbose( 1, cmd )
if_verbose( 3, cmd )
r = 0
if config.os == 'mingw32':
# On MinGW, we will always have timeout
......
......@@ -159,6 +159,10 @@ ifneq "$(THREADS)" ""
RUNTEST_OPTS += --threads=$(THREADS)
endif
ifneq "$(VERBOSE)" ""
RUNTEST_OPTS += --verbose=$(VERBOSE)
endif
ifneq "$(CLEAN_ONLY)" ""
RUNTEST_OPTS += -e clean_only=True
else
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment