Commit 1084d375 authored by thomie's avatar thomie
Browse files

Testsuite: use ignore_stderr/stdout instead of ignore_output

The problem with ignore_output is that it hides errors for WAY=ghci.
GHCi always returns with exit code 0 (unless it is broken itself).

For example: ghci015 must have been failing with compile errors for
years, but we didn't notice because all output was ignored.

Therefore, replace all uses of ignore_output with either ignore_stderr
or ignore_stdout. In some cases I opted for adding the expected output.

Update submodule hpc and stm.

Reviewed by: simonmar

Differential Revision:
parent 8f7194fa
......@@ -147,18 +147,10 @@ test('topHandler01', when(opsys('mingw32'), skip), compile_and_run, [''])
[when(opsys('mingw32'), skip),
# Irritatingly, the test driver calls the programs via a shell, and
# depending on the shell, they can add their own "helpful" commentary,
# pretty printing the name of the signal that killed the process. So we
# ignore the stdout here, we only care about the exit code (which itself
# is messed up because of the shell, using 128+sig encoding)
], compile_and_run, [''])
[when(opsys('mingw32'), skip),
# As above, shells, grrr.
], compile_and_run, [''])
Subproject commit b52ab0cc013beb1440607a7e4521a45fd6e96ce8
Subproject commit 956887d4a15de3e68aae82b14bfa1630c8149649
Subproject commit ee756000fc654a105ff3f8a319b904f2df33c65b
Subproject commit fe8899331e6ca7bdf80d57cf77dd597023ae4718
......@@ -65,6 +65,7 @@ extra_src_files = {
'T11223_link_order_a_b_succeed': ['bar.c', 'foo.c', 'foo2.hs'],
'T11223_link_order_b_a_2_succeed': ['bar.c', 'foo.c', 'foo3.hs'],
'T11223_link_order_b_a_succeed': ['bar.c', 'foo.c', 'foo2.hs'],
'T11223_simple_duplicate': ['bar.c', 'foo.c', 'foo.hs'],
'T11223_simple_duplicate_lib': ['bar.c', 'foo.c', 'foo.hs'],
'T11223_simple_link': ['foo.c', 'foo.hs'],
'T11223_simple_link_lib': ['foo.c', 'foo.hs'],
......@@ -176,7 +176,8 @@ class TestOptions:
self.stdin = ''
# don't compare output
self.ignore_output = 0
self.ignore_stdout = False
self.ignore_stderr = False
# We sometimes want to modify the compiler_always_flags, so
# they are copied from config.compiler_always_flags when we
......@@ -246,6 +247,9 @@ class TestOptions:
# output, and a normaliser function given other test options
self.check_stdout = None
# Check .hp file when profiling libraries are available?
self.check_hp = True
# Extra normalisation for compiler error messages
self.extra_errmsg_normaliser = lambda x: x
......@@ -172,8 +172,11 @@ def req_smp( name, opts ):
if not config.have_smp:
opts.expect = 'fail'
def ignore_output( name, opts ):
opts.ignore_output = 1
def ignore_stdout(name, opts):
opts.ignore_stdout = True
def ignore_stderr(name, opts):
opts.ignore_stderr = True
def combined_output( name, opts ):
opts.combined_output = True
......@@ -484,6 +487,9 @@ def check_stdout( f ):
def _check_stdout( name, opts, f ):
opts.check_stdout = f
def no_check_hp(name, opts):
opts.check_hp = False
# ----
def normalise_slashes( name, opts ):
......@@ -902,7 +908,7 @@ def failBecause(reason, tag=None):
# The expected exit code can be changed via exit_code() as normal, and
# the expected stdout/stderr are stored in <testname>.stdout and
# <testname>.stderr. The output of the command can be ignored
# altogether by using run_command_ignore_output instead of
# altogether by using the setup function ignore_stdout instead of
# run_command.
def run_command( name, way, cmd ):
......@@ -1216,21 +1222,19 @@ def simple_run(name, way, prog, extra_run_opts):
return failBecause('bad exit code')
check_hp = '-h' in my_rts_flags
if not (opts.ignore_stderr or stderr_ok(name, way) or opts.combined_output):
return failBecause('bad stderr')
if not (opts.ignore_stdout or stdout_ok(name, way)):
return failBecause('bad stdout')
check_hp = '-h' in my_rts_flags and opts.check_hp
check_prof = '-p' in my_rts_flags
if not opts.ignore_output:
bad_stderr = not opts.combined_output and not check_stderr_ok(name, way)
bad_stdout = not check_stdout_ok(name, way)
if bad_stderr:
return failBecause('bad stderr')
if bad_stdout:
return failBecause('bad stdout')
# exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
return failBecause('bad heap profile')
if check_prof and not check_prof_ok(name, way):
return failBecause('bad profile')
# exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
return failBecause('bad heap profile')
if check_prof and not check_prof_ok(name, way):
return failBecause('bad profile')
return checkStats(name, way, stats_file, opts.stats_range_fields)
......@@ -1307,11 +1311,12 @@ def interpreter_run(name, way, extra_hc_opts, top_mod):
# ToDo: if the sub-shell was killed by ^C, then exit
if getTestOpts().ignore_output or (check_stderr_ok(name, way) and
check_stdout_ok(name, way)):
return passed()
if not (opts.ignore_stderr or stderr_ok(name, way)):
return failBecause('bad stderr')
elif not (opts.ignore_stdout or stdout_ok(name, way)):
return failBecause('bad stdout')
return failBecause('bad stdout or stderr')
return passed()
def split_file(in_fn, delimiter, out1_fn, out2_fn):
# See Note [Universal newlines].
......@@ -1345,7 +1350,7 @@ def get_compiler_flags():
return flags
def check_stdout_ok(name, way):
def stdout_ok(name, way):
actual_stdout_file = add_suffix(name, 'run.stdout')
expected_stdout_file = find_expected_file(name, 'stdout')
......@@ -1363,7 +1368,7 @@ def dump_stdout( name ):
print(read_no_crs(in_testdir(name, 'run.stdout')))
def check_stderr_ok(name, way):
def stderr_ok(name, way):
actual_stderr_file = add_suffix(name, 'run.stderr')
expected_stderr_file = find_expected_file(name, 'stderr')
......@@ -243,8 +243,7 @@ ghcpkg02:
$(GHC_PKG_ghcpkg02) init $(PACKAGE_CONF_ghcpkg02)
set -e; \
for i in `'$(GHC_PKG)' list --global --simple-output -v0`; do \
echo Updating $$i; \
'$(GHC_PKG)' describe --global $$i | $(GHC_PKG_ghcpkg02) update --global --force -; \
'$(GHC_PKG)' describe --global $$i | $(GHC_PKG_ghcpkg02) -v0 update --global --force -; \
......@@ -6,19 +6,31 @@ test('ghcpkg01',
['$MAKE -s --no-print-directory ghcpkg01'])
# Use ignore_stderr to prevent (when HADDOCK_DOCS=NO):
# warning: haddock-interfaces .. doesn't exist or isn't a file
extra_clean(['package.conf.ghcpkg02', 'package.conf.ghcpkg02.old'])],
['$MAKE -s --no-print-directory ghcpkg02'])
['$MAKE -s --no-print-directory ghcpkg03'])
def normalise_package_order(s):
# Package order is not deterministic?
return re.sub('testpkg- newtestpkg-2.0',
'newtestpkg-2.0 testpkg-',
......@@ -6,9 +6,9 @@ include $(TOP)/mk/
SETUP=../Setup -v0
# This test is for overlapping/shadowing packages with Cabal. We
# 1. install p-1.0 to the global DB
# 2. install q-1.0 (depending on p-1.0) to a temporary DB
# 3. install a differnet p-1.0 to the temp DB
# 1. install p-1.0
# 2. install q-1.0 (depending on p-1.0)
# 3. install a different p-1.0
# 4. attempt to configure and buidl r, which depends on p-1.0 and q-1.0
# step 4 will elicit a warning from Cabal's configure step that the
......@@ -30,7 +30,7 @@ cabal03: clean
cd p && $(SETUP) configure $(CABAL_MINIMAL_BUILD) --with-ghc='$(TEST_HC)' --ghc-options='$(TEST_HC_OPTS)' --package-db=../tmp.d --ghc-pkg-option=--force --ipid p-withopt
cd p && $(SETUP) build
cd p && $(SETUP) register
cd r && ! ../Setup configure $(CABAL_MINIMAL_BUILD) --with-ghc='$(TEST_HC)' --ghc-options='$(TEST_HC_OPTS)' --package-db=../tmp.d --ghc-pkg-option=--force
cd r && ! ../Setup configure -v0 $(CABAL_MINIMAL_BUILD) --with-ghc='$(TEST_HC)' --ghc-options='$(TEST_HC_OPTS)' --package-db=../tmp.d --ghc-pkg-option=--force
ifneq "$(CLEANUP)" ""
$(MAKE) -s --no-print-directory clean
......@@ -4,6 +4,6 @@ else:
cleanup = 'CLEANUP=0'
['$MAKE -s --no-print-directory cabal03 ' + cleanup])
Setup: The following installed packages are broken because other packages they
depend on are missing. These broken packages must be rebuilt before they can
be used.
package q-1.0 is broken due to missing package p-noopt
......@@ -4,6 +4,6 @@ else:
cleanup = 'CLEANUP=0'
['$MAKE -s --no-print-directory cabal05 ' + cleanup])
T.hs:3:1: error:
Ambiguous interface for ‘Conflict’:
it is bound as p- by a reexport in package p-
it is bound as P by a reexport in package p-
......@@ -4,6 +4,6 @@ else:
cleanup = 'CLEANUP=0'
['$MAKE -s --no-print-directory cabal09 ' + cleanup])
Failed to load interface for `A':
ghcpkg04.hs:1:1: error:
Ambiguous interface for ‘A’:
it was found in multiple packages: testpkg- newtestpkg-2.0
......@@ -239,9 +239,9 @@ test('conc064',
compile_and_run, [''])
test('conc065', ignore_output, compile_and_run, [''])
test('conc066', ignore_output, compile_and_run, [''])
test('conc067', ignore_output, compile_and_run, [''])
test('conc065', ignore_stdout, compile_and_run, [''])
test('conc066', ignore_stdout, compile_and_run, [''])
test('conc067', ignore_stdout, compile_and_run, [''])
# omit threaded2, the behaviour of this test is non-deterministic with more
# than one CPU.
......@@ -2,8 +2,7 @@ test ('EnumFromToP'
, [ extra_clean(['EnumFromToP.o', 'EnumFromToP.hi'])
, reqlib('dph-lifted-vseg')
, reqlib('dph-prim-par')
, ignore_output
, expect_broken(7736)
, only_ways(['normal', 'threaded1', 'threaded2']) ]
, compile_fail
, compile
, [ '-O0 -package dph-lifted-vseg'])
......@@ -457,8 +457,8 @@ static001:
$(RM) T3674*.o T3674*.hi T3674_pre
"$(TEST_HC)" $(TEST_HC_OPTS) --make T3674_pre.hs
"$(TEST_HC)" $(TEST_HC_OPTS) --make T3674.hs
"$(TEST_HC)" $(TEST_HC_OPTS) --make -v0 T3674_pre.hs
"$(TEST_HC)" $(TEST_HC_OPTS) --make -v0 T3674.hs
.PHONY: rtsopts001
......@@ -614,9 +614,9 @@ T10182:
.PHONY: T10320
$(RM) -rf T10320 T10320.dump-rule-rewrites T10320.hi T10320.o
"$(TEST_HC)" $(TEST_HC_OPTS) -ddump-to-file -ddump-rule-rewrites -fenable-rewrite-rules T10320.hs
"$(TEST_HC)" $(TEST_HC_OPTS) -v0 -ddump-to-file -ddump-rule-rewrites -fenable-rewrite-rules T10320.hs
[ -s T10320.dump-rule-rewrites ]
"$(TEST_HC)" $(TEST_HC_OPTS) -fforce-recomp -ddump-to-file -ddump-rule-rewrites T10320.hs
"$(TEST_HC)" $(TEST_HC_OPTS) -v0 -fforce-recomp -ddump-to-file -ddump-rule-rewrites T10320.hs
[ -f T10320.dump-rule-rewrites ] && [ ! -s T10320.dump-rule-rewrites ]
.PHONY: T12135
......@@ -333,7 +333,7 @@ test('T5313',
test('T2464', normal, compile, [''])
extra_clean(['T3674_pre.hi', 'T3674_pre.o', 'T3674_pre', 'T3674_pre.exe'])],
['$MAKE -s --no-print-directory T3674'])
......@@ -480,8 +480,8 @@ test('T11429c', normal, compile_fail, ['-Wunrecognised-warning-flags -Werror -Wf
test('T11763', normal, compile_and_run, ['-fno-version-macros'])
[ ignore_output
, extra_clean(['T10320', 'T10320.o', 'T10320.hi'])],
extra_clean(['T10320', 'T10320.o', 'T10320.hi'])],
['$MAKE -s --no-print-directory T10320'])
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment