Commit 1084d375 authored by thomie's avatar thomie
Browse files

Testsuite: use ignore_stderr/stdout instead of ignore_output

The problem with ignore_output is that it hides errors for WAY=ghci.
GHCi always returns with exit code 0 (unless it is broken itself).

For example: ghci015 must have been failing with compile errors for
years, but we didn't notice because all output was ignored.

Therefore, replace all uses of ignore_output with either ignore_stderr
or ignore_stdout. In some cases I opted for adding the expected output.

Update submodule hpc and stm.

Reviewed by: simonmar

Differential Revision: https://phabricator.haskell.org/D2367
parent 8f7194fa
...@@ -147,18 +147,10 @@ test('topHandler01', when(opsys('mingw32'), skip), compile_and_run, ['']) ...@@ -147,18 +147,10 @@ test('topHandler01', when(opsys('mingw32'), skip), compile_and_run, [''])
test('topHandler02', test('topHandler02',
[when(opsys('mingw32'), skip), [when(opsys('mingw32'), skip),
omit_ways(['ghci']), omit_ways(['ghci']),
# Irritatingly, the test driver calls the programs via a shell, and
# depending on the shell, they can add their own "helpful" commentary,
# pretty printing the name of the signal that killed the process. So we
# ignore the stdout here, we only care about the exit code (which itself
# is messed up because of the shell, using 128+sig encoding)
ignore_output,
signal_exit_code(2) signal_exit_code(2)
], compile_and_run, ['']) ], compile_and_run, [''])
test('topHandler03', test('topHandler03',
[when(opsys('mingw32'), skip), [when(opsys('mingw32'), skip),
# As above, shells, grrr.
ignore_output,
signal_exit_code(15) signal_exit_code(15)
], compile_and_run, ['']) ], compile_and_run, [''])
test('topHandler04', test('topHandler04',
......
Subproject commit b52ab0cc013beb1440607a7e4521a45fd6e96ce8 Subproject commit 956887d4a15de3e68aae82b14bfa1630c8149649
Subproject commit ee756000fc654a105ff3f8a319b904f2df33c65b Subproject commit fe8899331e6ca7bdf80d57cf77dd597023ae4718
...@@ -65,6 +65,7 @@ extra_src_files = { ...@@ -65,6 +65,7 @@ extra_src_files = {
'T11223_link_order_a_b_succeed': ['bar.c', 'foo.c', 'foo2.hs'], 'T11223_link_order_a_b_succeed': ['bar.c', 'foo.c', 'foo2.hs'],
'T11223_link_order_b_a_2_succeed': ['bar.c', 'foo.c', 'foo3.hs'], 'T11223_link_order_b_a_2_succeed': ['bar.c', 'foo.c', 'foo3.hs'],
'T11223_link_order_b_a_succeed': ['bar.c', 'foo.c', 'foo2.hs'], 'T11223_link_order_b_a_succeed': ['bar.c', 'foo.c', 'foo2.hs'],
'T11223_simple_duplicate': ['bar.c', 'foo.c', 'foo.hs'],
'T11223_simple_duplicate_lib': ['bar.c', 'foo.c', 'foo.hs'], 'T11223_simple_duplicate_lib': ['bar.c', 'foo.c', 'foo.hs'],
'T11223_simple_link': ['foo.c', 'foo.hs'], 'T11223_simple_link': ['foo.c', 'foo.hs'],
'T11223_simple_link_lib': ['foo.c', 'foo.hs'], 'T11223_simple_link_lib': ['foo.c', 'foo.hs'],
......
...@@ -176,7 +176,8 @@ class TestOptions: ...@@ -176,7 +176,8 @@ class TestOptions:
self.stdin = '' self.stdin = ''
# don't compare output # don't compare output
self.ignore_output = 0 self.ignore_stdout = False
self.ignore_stderr = False
# We sometimes want to modify the compiler_always_flags, so # We sometimes want to modify the compiler_always_flags, so
# they are copied from config.compiler_always_flags when we # they are copied from config.compiler_always_flags when we
...@@ -246,6 +247,9 @@ class TestOptions: ...@@ -246,6 +247,9 @@ class TestOptions:
# output, and a normaliser function given other test options # output, and a normaliser function given other test options
self.check_stdout = None self.check_stdout = None
# Check .hp file when profiling libraries are available?
self.check_hp = True
# Extra normalisation for compiler error messages # Extra normalisation for compiler error messages
self.extra_errmsg_normaliser = lambda x: x self.extra_errmsg_normaliser = lambda x: x
......
...@@ -172,8 +172,11 @@ def req_smp( name, opts ): ...@@ -172,8 +172,11 @@ def req_smp( name, opts ):
if not config.have_smp: if not config.have_smp:
opts.expect = 'fail' opts.expect = 'fail'
def ignore_output( name, opts ): def ignore_stdout(name, opts):
opts.ignore_output = 1 opts.ignore_stdout = True
def ignore_stderr(name, opts):
opts.ignore_stderr = True
def combined_output( name, opts ): def combined_output( name, opts ):
opts.combined_output = True opts.combined_output = True
...@@ -484,6 +487,9 @@ def check_stdout( f ): ...@@ -484,6 +487,9 @@ def check_stdout( f ):
def _check_stdout( name, opts, f ): def _check_stdout( name, opts, f ):
opts.check_stdout = f opts.check_stdout = f
def no_check_hp(name, opts):
opts.check_hp = False
# ---- # ----
def normalise_slashes( name, opts ): def normalise_slashes( name, opts ):
...@@ -902,7 +908,7 @@ def failBecause(reason, tag=None): ...@@ -902,7 +908,7 @@ def failBecause(reason, tag=None):
# The expected exit code can be changed via exit_code() as normal, and # The expected exit code can be changed via exit_code() as normal, and
# the expected stdout/stderr are stored in <testname>.stdout and # the expected stdout/stderr are stored in <testname>.stdout and
# <testname>.stderr. The output of the command can be ignored # <testname>.stderr. The output of the command can be ignored
# altogether by using run_command_ignore_output instead of # altogether by using the setup function ignore_stdout instead of
# run_command. # run_command.
def run_command( name, way, cmd ): def run_command( name, way, cmd ):
...@@ -1216,21 +1222,19 @@ def simple_run(name, way, prog, extra_run_opts): ...@@ -1216,21 +1222,19 @@ def simple_run(name, way, prog, extra_run_opts):
dump_stderr(name) dump_stderr(name)
return failBecause('bad exit code') return failBecause('bad exit code')
check_hp = '-h' in my_rts_flags if not (opts.ignore_stderr or stderr_ok(name, way) or opts.combined_output):
return failBecause('bad stderr')
if not (opts.ignore_stdout or stdout_ok(name, way)):
return failBecause('bad stdout')
check_hp = '-h' in my_rts_flags and opts.check_hp
check_prof = '-p' in my_rts_flags check_prof = '-p' in my_rts_flags
if not opts.ignore_output: # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
bad_stderr = not opts.combined_output and not check_stderr_ok(name, way) if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
bad_stdout = not check_stdout_ok(name, way) return failBecause('bad heap profile')
if bad_stderr: if check_prof and not check_prof_ok(name, way):
return failBecause('bad stderr') return failBecause('bad profile')
if bad_stdout:
return failBecause('bad stdout')
# exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
return failBecause('bad heap profile')
if check_prof and not check_prof_ok(name, way):
return failBecause('bad profile')
return checkStats(name, way, stats_file, opts.stats_range_fields) return checkStats(name, way, stats_file, opts.stats_range_fields)
...@@ -1307,11 +1311,12 @@ def interpreter_run(name, way, extra_hc_opts, top_mod): ...@@ -1307,11 +1311,12 @@ def interpreter_run(name, way, extra_hc_opts, top_mod):
# ToDo: if the sub-shell was killed by ^C, then exit # ToDo: if the sub-shell was killed by ^C, then exit
if getTestOpts().ignore_output or (check_stderr_ok(name, way) and if not (opts.ignore_stderr or stderr_ok(name, way)):
check_stdout_ok(name, way)): return failBecause('bad stderr')
return passed() elif not (opts.ignore_stdout or stdout_ok(name, way)):
return failBecause('bad stdout')
else: else:
return failBecause('bad stdout or stderr') return passed()
def split_file(in_fn, delimiter, out1_fn, out2_fn): def split_file(in_fn, delimiter, out1_fn, out2_fn):
# See Note [Universal newlines]. # See Note [Universal newlines].
...@@ -1345,7 +1350,7 @@ def get_compiler_flags(): ...@@ -1345,7 +1350,7 @@ def get_compiler_flags():
return flags return flags
def check_stdout_ok(name, way): def stdout_ok(name, way):
actual_stdout_file = add_suffix(name, 'run.stdout') actual_stdout_file = add_suffix(name, 'run.stdout')
expected_stdout_file = find_expected_file(name, 'stdout') expected_stdout_file = find_expected_file(name, 'stdout')
...@@ -1363,7 +1368,7 @@ def dump_stdout( name ): ...@@ -1363,7 +1368,7 @@ def dump_stdout( name ):
print('Stdout:') print('Stdout:')
print(read_no_crs(in_testdir(name, 'run.stdout'))) print(read_no_crs(in_testdir(name, 'run.stdout')))
def check_stderr_ok(name, way): def stderr_ok(name, way):
actual_stderr_file = add_suffix(name, 'run.stderr') actual_stderr_file = add_suffix(name, 'run.stderr')
expected_stderr_file = find_expected_file(name, 'stderr') expected_stderr_file = find_expected_file(name, 'stderr')
......
...@@ -243,8 +243,7 @@ ghcpkg02: ...@@ -243,8 +243,7 @@ ghcpkg02:
$(GHC_PKG_ghcpkg02) init $(PACKAGE_CONF_ghcpkg02) $(GHC_PKG_ghcpkg02) init $(PACKAGE_CONF_ghcpkg02)
set -e; \ set -e; \
for i in `'$(GHC_PKG)' list --global --simple-output -v0`; do \ for i in `'$(GHC_PKG)' list --global --simple-output -v0`; do \
echo Updating $$i; \ '$(GHC_PKG)' describe --global $$i | $(GHC_PKG_ghcpkg02) -v0 update --global --force -; \
'$(GHC_PKG)' describe --global $$i | $(GHC_PKG_ghcpkg02) update --global --force -; \
done done
PKGCONF07=local07.package.conf PKGCONF07=local07.package.conf
......
...@@ -6,19 +6,31 @@ test('ghcpkg01', ...@@ -6,19 +6,31 @@ test('ghcpkg01',
'local01.package.conf.old']), 'local01.package.conf.old']),
run_command, run_command,
['$MAKE -s --no-print-directory ghcpkg01']) ['$MAKE -s --no-print-directory ghcpkg01'])
# Use ignore_stderr to prevent (when HADDOCK_DOCS=NO):
# warning: haddock-interfaces .. doesn't exist or isn't a file
test('ghcpkg02', test('ghcpkg02',
[ignore_output, [ignore_stderr,
extra_clean(['package.conf.ghcpkg02', 'package.conf.ghcpkg02.old'])], extra_clean(['package.conf.ghcpkg02', 'package.conf.ghcpkg02.old'])],
run_command, run_command,
['$MAKE -s --no-print-directory ghcpkg02']) ['$MAKE -s --no-print-directory ghcpkg02'])
test('ghcpkg03', test('ghcpkg03',
[extra_clean(['local03.package.conf', [extra_clean(['local03.package.conf',
'local03.package.conf.old']), 'local03.package.conf.old']),
normalise_errmsg_fun(normaliseDynlibNames)], normalise_errmsg_fun(normaliseDynlibNames)],
run_command, run_command,
['$MAKE -s --no-print-directory ghcpkg03']) ['$MAKE -s --no-print-directory ghcpkg03'])
def normalise_package_order(s):
# Package order is not deterministic?
return re.sub('testpkg-1.2.3.4 newtestpkg-2.0',
'newtestpkg-2.0 testpkg-1.2.3.4',
s)
test('ghcpkg04', test('ghcpkg04',
[ignore_output, [
normalise_errmsg_fun(normalise_package_order),
extra_clean(['local04.package.conf', extra_clean(['local04.package.conf',
'local04.package.conf.old'])], 'local04.package.conf.old'])],
run_command, run_command,
......
...@@ -6,9 +6,9 @@ include $(TOP)/mk/test.mk ...@@ -6,9 +6,9 @@ include $(TOP)/mk/test.mk
SETUP=../Setup -v0 SETUP=../Setup -v0
# This test is for overlapping/shadowing packages with Cabal. We # This test is for overlapping/shadowing packages with Cabal. We
# 1. install p-1.0 to the global DB # 1. install p-1.0
# 2. install q-1.0 (depending on p-1.0) to a temporary DB # 2. install q-1.0 (depending on p-1.0)
# 3. install a differnet p-1.0 to the temp DB # 3. install a different p-1.0
# 4. attempt to configure and buidl r, which depends on p-1.0 and q-1.0 # 4. attempt to configure and buidl r, which depends on p-1.0 and q-1.0
# #
# step 4 will elicit a warning from Cabal's configure step that the # step 4 will elicit a warning from Cabal's configure step that the
...@@ -30,7 +30,7 @@ cabal03: clean ...@@ -30,7 +30,7 @@ cabal03: clean
cd p && $(SETUP) configure $(CABAL_MINIMAL_BUILD) --with-ghc='$(TEST_HC)' --ghc-options='$(TEST_HC_OPTS)' --package-db=../tmp.d --ghc-pkg-option=--force --ipid p-withopt cd p && $(SETUP) configure $(CABAL_MINIMAL_BUILD) --with-ghc='$(TEST_HC)' --ghc-options='$(TEST_HC_OPTS)' --package-db=../tmp.d --ghc-pkg-option=--force --ipid p-withopt
cd p && $(SETUP) build cd p && $(SETUP) build
cd p && $(SETUP) register cd p && $(SETUP) register
cd r && ! ../Setup configure $(CABAL_MINIMAL_BUILD) --with-ghc='$(TEST_HC)' --ghc-options='$(TEST_HC_OPTS)' --package-db=../tmp.d --ghc-pkg-option=--force cd r && ! ../Setup configure -v0 $(CABAL_MINIMAL_BUILD) --with-ghc='$(TEST_HC)' --ghc-options='$(TEST_HC_OPTS)' --package-db=../tmp.d --ghc-pkg-option=--force
ifneq "$(CLEANUP)" "" ifneq "$(CLEANUP)" ""
$(MAKE) -s --no-print-directory clean $(MAKE) -s --no-print-directory clean
endif endif
......
...@@ -4,6 +4,6 @@ else: ...@@ -4,6 +4,6 @@ else:
cleanup = 'CLEANUP=0' cleanup = 'CLEANUP=0'
test('cabal03', test('cabal03',
ignore_output, normal,
run_command, run_command,
['$MAKE -s --no-print-directory cabal03 ' + cleanup]) ['$MAKE -s --no-print-directory cabal03 ' + cleanup])
Setup: The following installed packages are broken because other packages they
depend on are missing. These broken packages must be rebuilt before they can
be used.
package q-1.0 is broken due to missing package p-noopt
...@@ -4,6 +4,6 @@ else: ...@@ -4,6 +4,6 @@ else:
cleanup = 'CLEANUP=0' cleanup = 'CLEANUP=0'
test('cabal05', test('cabal05',
ignore_output, normal,
run_command, run_command,
['$MAKE -s --no-print-directory cabal05 ' + cleanup]) ['$MAKE -s --no-print-directory cabal05 ' + cleanup])
T.hs:3:1: error:
Ambiguous interface for ‘Conflict’:
it is bound as p-0.1.0.0:P2 by a reexport in package p-0.1.0.0
it is bound as P by a reexport in package p-0.1.0.0
...@@ -4,6 +4,6 @@ else: ...@@ -4,6 +4,6 @@ else:
cleanup = 'CLEANUP=0' cleanup = 'CLEANUP=0'
test('cabal09', test('cabal09',
ignore_output, normal,
run_command, run_command,
['$MAKE -s --no-print-directory cabal09 ' + cleanup]) ['$MAKE -s --no-print-directory cabal09 ' + cleanup])
ghcpkg04.hs:1:0: ghcpkg04.hs:1:1: error:
Failed to load interface for `A': Ambiguous interface for ‘A’:
it was found in multiple packages: testpkg-1.2.3.4 newtestpkg-2.0 it was found in multiple packages: testpkg-1.2.3.4 newtestpkg-2.0
...@@ -239,9 +239,9 @@ test('conc064', ...@@ -239,9 +239,9 @@ test('conc064',
exit_code(1), exit_code(1),
compile_and_run, ['']) compile_and_run, [''])
test('conc065', ignore_output, compile_and_run, ['']) test('conc065', ignore_stdout, compile_and_run, [''])
test('conc066', ignore_output, compile_and_run, ['']) test('conc066', ignore_stdout, compile_and_run, [''])
test('conc067', ignore_output, compile_and_run, ['']) test('conc067', ignore_stdout, compile_and_run, [''])
# omit threaded2, the behaviour of this test is non-deterministic with more # omit threaded2, the behaviour of this test is non-deterministic with more
# than one CPU. # than one CPU.
......
...@@ -2,8 +2,7 @@ test ('EnumFromToP' ...@@ -2,8 +2,7 @@ test ('EnumFromToP'
, [ extra_clean(['EnumFromToP.o', 'EnumFromToP.hi']) , [ extra_clean(['EnumFromToP.o', 'EnumFromToP.hi'])
, reqlib('dph-lifted-vseg') , reqlib('dph-lifted-vseg')
, reqlib('dph-prim-par') , reqlib('dph-prim-par')
, ignore_output
, expect_broken(7736) , expect_broken(7736)
, only_ways(['normal', 'threaded1', 'threaded2']) ] , only_ways(['normal', 'threaded1', 'threaded2']) ]
, compile_fail , compile
, [ '-O0 -package dph-lifted-vseg']) , [ '-O0 -package dph-lifted-vseg'])
...@@ -457,8 +457,8 @@ static001: ...@@ -457,8 +457,8 @@ static001:
T3674: T3674:
$(RM) T3674*.o T3674*.hi T3674_pre $(RM) T3674*.o T3674*.hi T3674_pre
"$(TEST_HC)" $(TEST_HC_OPTS) --make T3674_pre.hs "$(TEST_HC)" $(TEST_HC_OPTS) --make -v0 T3674_pre.hs
"$(TEST_HC)" $(TEST_HC_OPTS) --make T3674.hs "$(TEST_HC)" $(TEST_HC_OPTS) --make -v0 T3674.hs
.PHONY: rtsopts001 .PHONY: rtsopts001
rtsopts001: rtsopts001:
...@@ -614,9 +614,9 @@ T10182: ...@@ -614,9 +614,9 @@ T10182:
.PHONY: T10320 .PHONY: T10320
T10320: T10320:
$(RM) -rf T10320 T10320.dump-rule-rewrites T10320.hi T10320.o $(RM) -rf T10320 T10320.dump-rule-rewrites T10320.hi T10320.o
"$(TEST_HC)" $(TEST_HC_OPTS) -ddump-to-file -ddump-rule-rewrites -fenable-rewrite-rules T10320.hs "$(TEST_HC)" $(TEST_HC_OPTS) -v0 -ddump-to-file -ddump-rule-rewrites -fenable-rewrite-rules T10320.hs
[ -s T10320.dump-rule-rewrites ] [ -s T10320.dump-rule-rewrites ]
"$(TEST_HC)" $(TEST_HC_OPTS) -fforce-recomp -ddump-to-file -ddump-rule-rewrites T10320.hs "$(TEST_HC)" $(TEST_HC_OPTS) -v0 -fforce-recomp -ddump-to-file -ddump-rule-rewrites T10320.hs
[ -f T10320.dump-rule-rewrites ] && [ ! -s T10320.dump-rule-rewrites ] [ -f T10320.dump-rule-rewrites ] && [ ! -s T10320.dump-rule-rewrites ]
.PHONY: T12135 .PHONY: T12135
......
...@@ -333,7 +333,7 @@ test('T5313', ...@@ -333,7 +333,7 @@ test('T5313',
test('T2464', normal, compile, ['']) test('T2464', normal, compile, [''])
test('T3674', test('T3674',
[ignore_output, [
extra_clean(['T3674_pre.hi', 'T3674_pre.o', 'T3674_pre', 'T3674_pre.exe'])], extra_clean(['T3674_pre.hi', 'T3674_pre.o', 'T3674_pre', 'T3674_pre.exe'])],
run_command, run_command,
['$MAKE -s --no-print-directory T3674']) ['$MAKE -s --no-print-directory T3674'])
...@@ -480,8 +480,8 @@ test('T11429c', normal, compile_fail, ['-Wunrecognised-warning-flags -Werror -Wf ...@@ -480,8 +480,8 @@ test('T11429c', normal, compile_fail, ['-Wunrecognised-warning-flags -Werror -Wf
test('T11763', normal, compile_and_run, ['-fno-version-macros']) test('T11763', normal, compile_and_run, ['-fno-version-macros'])
test('T10320', test('T10320',
[ ignore_output [
, extra_clean(['T10320', 'T10320.o', 'T10320.hi'])], extra_clean(['T10320', 'T10320.o', 'T10320.hi'])],
run_command, run_command,
['$MAKE -s --no-print-directory T10320']) ['$MAKE -s --no-print-directory T10320'])
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment