Commit 1084d375 authored by thomie's avatar thomie

Testsuite: use ignore_stderr/stdout instead of ignore_output

The problem with ignore_output is that it hides errors for WAY=ghci.
GHCi always returns with exit code 0 (unless it is broken itself).

For example: ghci015 must have been failing with compile errors for
years, but we didn't notice because all output was ignored.

Therefore, replace all uses of ignore_output with either ignore_stderr
or ignore_stdout. In some cases I opted for adding the expected output.

Update submodule hpc and stm.

Reviewed by: simonmar

Differential Revision:
parent 8f7194fa
......@@ -147,18 +147,10 @@ test('topHandler01', when(opsys('mingw32'), skip), compile_and_run, [''])
[when(opsys('mingw32'), skip),
# Irritatingly, the test driver calls the programs via a shell, and
# depending on the shell, they can add their own "helpful" commentary,
# pretty printing the name of the signal that killed the process. So we
# ignore the stdout here, we only care about the exit code (which itself
# is messed up because of the shell, using 128+sig encoding)
], compile_and_run, [''])
[when(opsys('mingw32'), skip),
# As above, shells, grrr.
], compile_and_run, [''])
Subproject commit b52ab0cc013beb1440607a7e4521a45fd6e96ce8
Subproject commit 956887d4a15de3e68aae82b14bfa1630c8149649
Subproject commit ee756000fc654a105ff3f8a319b904f2df33c65b
Subproject commit fe8899331e6ca7bdf80d57cf77dd597023ae4718
......@@ -65,6 +65,7 @@ extra_src_files = {
'T11223_link_order_a_b_succeed': ['bar.c', 'foo.c', 'foo2.hs'],
'T11223_link_order_b_a_2_succeed': ['bar.c', 'foo.c', 'foo3.hs'],
'T11223_link_order_b_a_succeed': ['bar.c', 'foo.c', 'foo2.hs'],
'T11223_simple_duplicate': ['bar.c', 'foo.c', 'foo.hs'],
'T11223_simple_duplicate_lib': ['bar.c', 'foo.c', 'foo.hs'],
'T11223_simple_link': ['foo.c', 'foo.hs'],
'T11223_simple_link_lib': ['foo.c', 'foo.hs'],
......@@ -176,7 +176,8 @@ class TestOptions:
self.stdin = ''
# don't compare output
self.ignore_output = 0
self.ignore_stdout = False
self.ignore_stderr = False
# We sometimes want to modify the compiler_always_flags, so
# they are copied from config.compiler_always_flags when we
......@@ -246,6 +247,9 @@ class TestOptions:
# output, and a normaliser function given other test options
self.check_stdout = None
# Check .hp file when profiling libraries are available?
self.check_hp = True
# Extra normalisation for compiler error messages
self.extra_errmsg_normaliser = lambda x: x
......@@ -172,8 +172,11 @@ def req_smp( name, opts ):
if not config.have_smp:
opts.expect = 'fail'
def ignore_output( name, opts ):
opts.ignore_output = 1
def ignore_stdout(name, opts):
opts.ignore_stdout = True
def ignore_stderr(name, opts):
opts.ignore_stderr = True
def combined_output( name, opts ):
opts.combined_output = True
......@@ -484,6 +487,9 @@ def check_stdout( f ):
def _check_stdout( name, opts, f ):
opts.check_stdout = f
def no_check_hp(name, opts):
opts.check_hp = False
# ----
def normalise_slashes( name, opts ):
......@@ -902,7 +908,7 @@ def failBecause(reason, tag=None):
# The expected exit code can be changed via exit_code() as normal, and
# the expected stdout/stderr are stored in <testname>.stdout and
# <testname>.stderr. The output of the command can be ignored
# altogether by using run_command_ignore_output instead of
# altogether by using the setup function ignore_stdout instead of
# run_command.
def run_command( name, way, cmd ):
......@@ -1216,21 +1222,19 @@ def simple_run(name, way, prog, extra_run_opts):
return failBecause('bad exit code')
check_hp = '-h' in my_rts_flags
if not (opts.ignore_stderr or stderr_ok(name, way) or opts.combined_output):
return failBecause('bad stderr')
if not (opts.ignore_stdout or stdout_ok(name, way)):
return failBecause('bad stdout')
check_hp = '-h' in my_rts_flags and opts.check_hp
check_prof = '-p' in my_rts_flags
if not opts.ignore_output:
bad_stderr = not opts.combined_output and not check_stderr_ok(name, way)
bad_stdout = not check_stdout_ok(name, way)
if bad_stderr:
return failBecause('bad stderr')
if bad_stdout:
return failBecause('bad stdout')
# exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
return failBecause('bad heap profile')
if check_prof and not check_prof_ok(name, way):
return failBecause('bad profile')
# exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
return failBecause('bad heap profile')
if check_prof and not check_prof_ok(name, way):
return failBecause('bad profile')
return checkStats(name, way, stats_file, opts.stats_range_fields)
......@@ -1307,11 +1311,12 @@ def interpreter_run(name, way, extra_hc_opts, top_mod):
# ToDo: if the sub-shell was killed by ^C, then exit
if getTestOpts().ignore_output or (check_stderr_ok(name, way) and
check_stdout_ok(name, way)):
return passed()
if not (opts.ignore_stderr or stderr_ok(name, way)):
return failBecause('bad stderr')
elif not (opts.ignore_stdout or stdout_ok(name, way)):
return failBecause('bad stdout')
return failBecause('bad stdout or stderr')
return passed()
def split_file(in_fn, delimiter, out1_fn, out2_fn):
# See Note [Universal newlines].
......@@ -1345,7 +1350,7 @@ def get_compiler_flags():
return flags
def check_stdout_ok(name, way):
def stdout_ok(name, way):
actual_stdout_file = add_suffix(name, 'run.stdout')
expected_stdout_file = find_expected_file(name, 'stdout')
......@@ -1363,7 +1368,7 @@ def dump_stdout( name ):
print(read_no_crs(in_testdir(name, 'run.stdout')))
def check_stderr_ok(name, way):
def stderr_ok(name, way):
actual_stderr_file = add_suffix(name, 'run.stderr')
expected_stderr_file = find_expected_file(name, 'stderr')
......@@ -243,8 +243,7 @@ ghcpkg02:
$(GHC_PKG_ghcpkg02) init $(PACKAGE_CONF_ghcpkg02)
set -e; \
for i in `'$(GHC_PKG)' list --global --simple-output -v0`; do \
echo Updating $$i; \
'$(GHC_PKG)' describe --global $$i | $(GHC_PKG_ghcpkg02) update --global --force -; \
'$(GHC_PKG)' describe --global $$i | $(GHC_PKG_ghcpkg02) -v0 update --global --force -; \
......@@ -6,19 +6,31 @@ test('ghcpkg01',
['$MAKE -s --no-print-directory ghcpkg01'])
# Use ignore_stderr to prevent (when HADDOCK_DOCS=NO):
# warning: haddock-interfaces .. doesn't exist or isn't a file
extra_clean(['package.conf.ghcpkg02', 'package.conf.ghcpkg02.old'])],
['$MAKE -s --no-print-directory ghcpkg02'])
['$MAKE -s --no-print-directory ghcpkg03'])
def normalise_package_order(s):
# Package order is not deterministic?
return re.sub('testpkg- newtestpkg-2.0',
'newtestpkg-2.0 testpkg-',
......@@ -6,9 +6,9 @@ include $(TOP)/mk/
SETUP=../Setup -v0
# This test is for overlapping/shadowing packages with Cabal. We
# 1. install p-1.0 to the global DB
# 2. install q-1.0 (depending on p-1.0) to a temporary DB
# 3. install a differnet p-1.0 to the temp DB
# 1. install p-1.0
# 2. install q-1.0 (depending on p-1.0)
# 3. install a different p-1.0
# 4. attempt to configure and buidl r, which depends on p-1.0 and q-1.0
# step 4 will elicit a warning from Cabal's configure step that the
......@@ -30,7 +30,7 @@ cabal03: clean
cd p && $(SETUP) configure $(CABAL_MINIMAL_BUILD) --with-ghc='$(TEST_HC)' --ghc-options='$(TEST_HC_OPTS)' --package-db=../tmp.d --ghc-pkg-option=--force --ipid p-withopt
cd p && $(SETUP) build
cd p && $(SETUP) register
cd r && ! ../Setup configure $(CABAL_MINIMAL_BUILD) --with-ghc='$(TEST_HC)' --ghc-options='$(TEST_HC_OPTS)' --package-db=../tmp.d --ghc-pkg-option=--force
cd r && ! ../Setup configure -v0 $(CABAL_MINIMAL_BUILD) --with-ghc='$(TEST_HC)' --ghc-options='$(TEST_HC_OPTS)' --package-db=../tmp.d --ghc-pkg-option=--force
ifneq "$(CLEANUP)" ""
$(MAKE) -s --no-print-directory clean
......@@ -4,6 +4,6 @@ else:
cleanup = 'CLEANUP=0'
['$MAKE -s --no-print-directory cabal03 ' + cleanup])
Setup: The following installed packages are broken because other packages they
depend on are missing. These broken packages must be rebuilt before they can
be used.
package q-1.0 is broken due to missing package p-noopt
......@@ -4,6 +4,6 @@ else:
cleanup = 'CLEANUP=0'
['$MAKE -s --no-print-directory cabal05 ' + cleanup])
T.hs:3:1: error:
Ambiguous interface for ‘Conflict’:
it is bound as p- by a reexport in package p-
it is bound as P by a reexport in package p-
......@@ -4,6 +4,6 @@ else:
cleanup = 'CLEANUP=0'
['$MAKE -s --no-print-directory cabal09 ' + cleanup])
Failed to load interface for `A':
ghcpkg04.hs:1:1: error:
Ambiguous interface for ‘A’:
it was found in multiple packages: testpkg- newtestpkg-2.0
......@@ -239,9 +239,9 @@ test('conc064',
compile_and_run, [''])
test('conc065', ignore_output, compile_and_run, [''])
test('conc066', ignore_output, compile_and_run, [''])
test('conc067', ignore_output, compile_and_run, [''])
test('conc065', ignore_stdout, compile_and_run, [''])
test('conc066', ignore_stdout, compile_and_run, [''])
test('conc067', ignore_stdout, compile_and_run, [''])
# omit threaded2, the behaviour of this test is non-deterministic with more
# than one CPU.
......@@ -2,8 +2,7 @@ test ('EnumFromToP'
, [ extra_clean(['EnumFromToP.o', 'EnumFromToP.hi'])
, reqlib('dph-lifted-vseg')
, reqlib('dph-prim-par')
, ignore_output
, expect_broken(7736)
, only_ways(['normal', 'threaded1', 'threaded2']) ]
, compile_fail
, compile
, [ '-O0 -package dph-lifted-vseg'])
......@@ -457,8 +457,8 @@ static001:
$(RM) T3674*.o T3674*.hi T3674_pre
"$(TEST_HC)" $(TEST_HC_OPTS) --make T3674_pre.hs
"$(TEST_HC)" $(TEST_HC_OPTS) --make T3674.hs
"$(TEST_HC)" $(TEST_HC_OPTS) --make -v0 T3674_pre.hs
"$(TEST_HC)" $(TEST_HC_OPTS) --make -v0 T3674.hs
.PHONY: rtsopts001
......@@ -614,9 +614,9 @@ T10182:
.PHONY: T10320
$(RM) -rf T10320 T10320.dump-rule-rewrites T10320.hi T10320.o
"$(TEST_HC)" $(TEST_HC_OPTS) -ddump-to-file -ddump-rule-rewrites -fenable-rewrite-rules T10320.hs
"$(TEST_HC)" $(TEST_HC_OPTS) -v0 -ddump-to-file -ddump-rule-rewrites -fenable-rewrite-rules T10320.hs
[ -s T10320.dump-rule-rewrites ]
"$(TEST_HC)" $(TEST_HC_OPTS) -fforce-recomp -ddump-to-file -ddump-rule-rewrites T10320.hs
"$(TEST_HC)" $(TEST_HC_OPTS) -v0 -fforce-recomp -ddump-to-file -ddump-rule-rewrites T10320.hs
[ -f T10320.dump-rule-rewrites ] && [ ! -s T10320.dump-rule-rewrites ]
.PHONY: T12135
......@@ -333,7 +333,7 @@ test('T5313',
test('T2464', normal, compile, [''])
extra_clean(['T3674_pre.hi', 'T3674_pre.o', 'T3674_pre', 'T3674_pre.exe'])],
['$MAKE -s --no-print-directory T3674'])
......@@ -480,8 +480,8 @@ test('T11429c', normal, compile_fail, ['-Wunrecognised-warning-flags -Werror -Wf
test('T11763', normal, compile_and_run, ['-fno-version-macros'])
[ ignore_output
, extra_clean(['T10320', 'T10320.o', 'T10320.hi'])],
extra_clean(['T10320', 'T10320.o', 'T10320.hi'])],
['$MAKE -s --no-print-directory T10320'])
......@@ -3,22 +3,22 @@ include $(TOP)/mk/
include $(TOP)/mk/
'$(TEST_HC)' $(TEST_HC_OPTS) -ignore-dot-ghci -e "return ("
-'$(TEST_HC)' $(TEST_HC_OPTS) -ignore-dot-ghci -e "return ("
'$(TEST_HC)' $(TEST_HC_OPTS) -ignore-dot-ghci -e "import This.Module.Does.Not.Exist"
-'$(TEST_HC)' $(TEST_HC_OPTS) -ignore-dot-ghci -e "import This.Module.Does.Not.Exist"
'$(TEST_HC)' $(TEST_HC_OPTS) -ignore-dot-ghci -e "import Data.List (bogusIdentifier)"
-'$(TEST_HC)' $(TEST_HC_OPTS) -ignore-dot-ghci -e "import Data.List (bogusIdentifier)"
'$(TEST_HC)' $(TEST_HC_OPTS) -ignore-dot-ghci -e "import Prelude (+)" # syntax error
-'$(TEST_HC)' $(TEST_HC_OPTS) -ignore-dot-ghci -e "import Prelude (+)" # syntax error
'$(TEST_HC)' $(TEST_HC_OPTS) -ignore-dot-ghci -e "class ["
-'$(TEST_HC)' $(TEST_HC_OPTS) -ignore-dot-ghci -e "class ["
'$(TEST_HC)' $(TEST_HC_OPTS) -ignore-dot-ghci -e "type A = A"
-'$(TEST_HC)' $(TEST_HC_OPTS) -ignore-dot-ghci -e "type A = A"
'$(TEST_HC)' $(TEST_HC_OPTS) -ignore-dot-ghci -x hs T9930
-'$(TEST_HC)' $(TEST_HC_OPTS) -v0 -x hs T9930
<interactive>:0:9: error:
parse error (possibly incorrect indentation or mismatched brackets)
<no location info>: error:
Could not find module ‘This.Module.Does.Not.Exist’
It is not a module in the current program, or in any known package.
<interactive>:1:19: error:
Module ‘Data.List’ does not export ‘bogusIdentifier’
<interactive>:0:17: error: parse error on input ‘+’
ghc-stage2: default output name would overwrite the input file; must specify -o explicitly
Usage: For basic information, try the `--help' option.
test('T7962', [exit_code(2), req_interp, ignore_output], run_command,
test('T7962', req_interp, run_command,
['$MAKE --no-print-directory -s T7962'])
test('T9905fail1', [exit_code(2), req_interp, ignore_output], run_command,
test('T9905fail1', req_interp, run_command,
['$MAKE --no-print-directory -s T9905fail1'])
test('T9905fail2', [exit_code(2), req_interp, ignore_output], run_command,
test('T9905fail2', req_interp, run_command,
['$MAKE --no-print-directory -s T9905fail2'])
test('T9905fail3', [exit_code(2), req_interp, ignore_output], run_command,
test('T9905fail3', req_interp, run_command,
['$MAKE --no-print-directory -s T9905fail3'])
test('ghc-e-fail1', [exit_code(2), req_interp, ignore_output], run_command,
test('ghc-e-fail1', req_interp, run_command,
['$MAKE --no-print-directory -s ghc-e-fail1'])
test('ghc-e-fail2', [exit_code(2), req_interp, ignore_output], run_command,
test('ghc-e-fail2', req_interp, run_command,
['$MAKE --no-print-directory -s ghc-e-fail2'])
# Don't run on Windows, as executable is written to T9930.exe
# and no failure is induced.
test('T9930fail', [exit_code(2), ignore_output, when(opsys('mingw32'), skip)], run_command,
test('T9930fail', when(opsys('mingw32'), skip), run_command,
['$MAKE --no-print-directory -s T9930fail'])
<interactive>:0:8: error:
parse error (possibly incorrect indentation or mismatched brackets)
<interactive>:0:1: error:
Cycle in type synonym declarations:
<interactive>:0:1-10: type A = A
......@@ -28,10 +28,12 @@ test('ghci012', normal, ghci_script, ['ghci012.script'])
test('ghci013', normal, ghci_script, ['ghci013.script'])
test('ghci014', reqlib('QuickCheck'), ghci_script, ['ghci014.script'])
# Ignore output from ghci015, the output is non-deterministic
# Ignore stderr from ghci015, the output is non-deterministic
# (has occasional "thread blocked indefinitely" exceptions), and this
# is part of the test.
test('ghci015', ignore_output, ghci_script, ['ghci015.script'])
# But don't silence both stderr and stdout, as then the test would trivially
# pass when ghci015.hs has errors.
test('ghci015', [reqlib('stm'), ignore_stderr], ghci_script, ['ghci015.script'])
test('ghci016', expect_broken(552), ghci_script, ['ghci016.script'])
test('ghci017', [reqlib('haskell98'), extra_run_opts('-hide-package haskell98')], ghci_script, ['ghci017.script'])
......@@ -245,7 +247,7 @@ test('T10576a', expect_broken(10576), ghci_script, ['T10576a.script'])
test('T10576b', expect_broken(10576), ghci_script, ['T10576b.script'])
test('T11051a', normal, ghci_script, ['T11051a.script'])
test('T11051b', normal, ghci_script, ['T11051b.script'])
test('T11266', check_stdout(lambda *args: 1), ghci_script, ['T11266.script'])
test('T11266', ignore_stdout, ghci_script, ['T11266.script'])
test('T11389', req_interp, run_command, ['$MAKE -s --no-print-directory T11389'])
test('T11524a', normal, ghci_script, ['T11524a.script'])
......@@ -25,8 +25,8 @@ runTest loop = do
forked loop args@(tc1, tc2, tmv, hisTId) = catch ((loop args) >>= setTMV . Just) hndlr `finally` setTMV Nothing
setTMV x = atomically (tryPutTMVar tmv x >> return ())
hndlr (AsyncException ThreadKilled) = return ()
hndlr e = throwTo hisTId e
hndlr ThreadKilled = return ()
hndlr e = throwTo hisTId e
goodLoop args@(tc1, tc2, tmv, hisTId) = do
x <- atomically (readTChan tc1)
test('T10138', ignore_output, run_command,
test('T10138', ignore_stdout, run_command,
# Using --hpcdir with an absolute path should work (exit code 0).
['{hpc} report T10138.keepme.tix --hpcdir="`pwd`/.keepme.hpc.T10138"'])
......@@ -69,7 +69,7 @@ test('lazy-bs-alloc',
# 2016-04-06: 429760 (x86/Linux) no idea what happened
# Use `+RTS -G1` for more stable residency measurements. Note [residency].
# Only 64-bit as we don't have a good 32-bit test environment at the moment
......@@ -265,7 +265,7 @@ test('T5536',
# 17/1/13: 892399040 (x86_64/Linux)
# (new demand analyser)
......@@ -25,7 +25,7 @@ test('T11223_simple_link_lib',
# I'm ignoring the output since for this particular invocation normalise_errmsg
# isn't being called and I can't figure out why not.
[when(ghc_dynamic(), skip), ignore_output, exit_code(2), normalise_errmsg_fun(normalise_duplicate_errmsg)],
[when(ghc_dynamic(), skip), ignore_stderr, ignore_stdout, exit_code(2), normalise_errmsg_fun(normalise_duplicate_errmsg)],
['$MAKE -s --no-print-directory t_11223_simple_duplicate'])
module Main where
main :: IO ()
main = return ()
T9839_01: flag -T given an argument when none was expected: -T-s
T9839_02: flag -Pa given an argument when none was expected: -Pax
T9839_03: flag -P given an argument when none was expected: -Px
T9839_05: flag -x given an argument when none was expected: -xcx
T9839_06: flag -x given an argument when none was expected: -xtx
......@@ -23,7 +23,7 @@ test('derefnull',
# Apparently the output can be different on different
# Linux setups, so just ignore it. As long as we get
# the right exit code we're OK.
when(opsys('linux'), ignore_output),
when(opsys('linux'), ignore_stderr),
# SIGBUS on OX X (PPC and x86 only; amd64 gives SEGV)
when(platform('i386-apple-darwin'), exit_code(138)),
when(platform('powerpc-apple-darwin'), exit_code(138)),
......@@ -39,7 +39,7 @@ test('divbyzero',
# Apparently the output can be different on different
# Linux setups, so just ignore it. As long as we get
# the right exit code we're OK.
when(opsys('linux'), ignore_output),
when(opsys('linux'), ignore_stderr),
# PowerPC 64 bit and most likely PowerPC 32 bit processors
# do not generate an exception (interrupt) for integer
# division by zero but the result is undefined.
......@@ -57,7 +57,7 @@ test('outofmem', when(opsys('darwin'), skip),
run_command, ['$MAKE -s --no-print-directory outofmem'])
test('outofmem2', normal, run_command, ['$MAKE -s --no-print-directory outofmem2'])
test('T2047', [ignore_output, extra_run_opts('+RTS -c -RTS')],
test('T2047', [ignore_stdout, extra_run_opts('+RTS -c -RTS')],
compile_and_run, ['-package containers'])
# Blackhole-detection test.
......@@ -223,9 +223,10 @@ test('T7815', [ multi_cpu_race,
only_ways(['threaded1', 'threaded2']) ], compile_and_run, [''] )
# ignore_output because it contains a unique:
# ignore_stderr because it contains a unique:
# ffishutdown: Main_dul: interrupted
test('ffishutdown', [ ignore_output, only_ways(['threaded1','threaded2']) ], compile_and_run, [''])
test('ffishutdown', [ignore_stderr, only_ways(['threaded1','threaded2'])],
compile_and_run, [''])
[extra_clean(['T7919A.o','T7919A.hi', 'T7919A.dyn_o','T7919A.dyn_hi']),
......@@ -240,10 +241,10 @@ test('linker_unload',
['$MAKE -s --no-print-directory linker_unload'])
test('T8209', [ only_ways(threaded_ways), ignore_output ],
test('T8209', [ only_ways(threaded_ways), ignore_stdout ],
compile_and_run, [''])
test('T8242', [ only_ways(threaded_ways), ignore_output ],
test('T8242', [ only_ways(threaded_ways), ignore_stdout ],
compile_and_run, [''])
test('T8124', [ only_ways(threaded_ways), omit_ways(['ghci']),
......@@ -286,51 +287,62 @@ test('overflow2', [ exit_code(251) ], compile_and_run, [''])
test('overflow3', [ exit_code(251) ], compile_and_run, [''])
[ extra_clean(['linker_error1.o','linker_error1']), ignore_output ],
[ extra_clean(['linker_error1.o','linker_error1']), ignore_stderr ],
['$MAKE -s --no-print-directory linker_error1'])
[ extra_clean(['linker_error2.o','linker_error2_c.o', 'linker_error2']),
ignore_output ],
ignore_stderr ],
['$MAKE -s --no-print-directory linker_error2'])
[ extra_clean(['linker_error3.o','linker_error3_c.o', 'linker_error3']),