ignore_output should be a test option

testsuite/driver/testglobals.py

@@ -124,6 +124,9 @@ class TestOptions:
        # the stdin file that this test will use (empty for <name>.stdin)
        self.stdin = ''
 
+       # don't compare output
+       self.ignore_output = 0
+
        # compile this test to .hc only
        self.compile_to_hc = 0
 

testsuite/driver/testlib.py

@@ -77,6 +77,9 @@ def expect_broken( bug ):
 def _expect_broken( opts, bug ):
     opts.expect = 'fail';
 
+def ignore_output( opts ):
+    opts.ignore_output = 1
+
 # -----
 
 def expect_fail_for( ways ):
@@ -361,7 +364,7 @@ def do_test(name, way, func, args):
         
         if getTestOpts().expect != 'pass' and getTestOpts().expect != 'fail' or \
            result != 'pass' and result != 'fail':
-            framework_fail(full_name)
+            framework_fail(name, way)
 
         if result == 'pass':
             if getTestOpts().expect == 'pass' \
@@ -396,7 +399,7 @@ def do_test(name, way, func, args):
     except:
         print '*** framework failure for', full_name, ':'
         traceback.print_exc()
-        framework_fail(full_name)
+        framework_fail(name, way)
 
 def skiptest (name, way):
     # print 'Skipping test \"', name, '\"'
@@ -406,9 +409,12 @@ def skiptest (name, way):
     else:
         t.tests_skipped[name] = [way]
 
-def framework_fail( name ):
+def framework_fail( name, way ):
     t.n_framework_failures = t.n_framework_failures + 1
-    t.framework_failures.append(name)
+    if name in t.framework_failures:
+        t.framework_failures[name].append(way)
+    else:
+        t.framework_failures[name] = [way]
 
 # -----------------------------------------------------------------------------
 # Generic command tests
@@ -498,56 +504,29 @@ def do_compile( name, way, should_fail, top_mod, extra_hc_opts ):
 # -----------------------------------------------------------------------------
 # Compile-and-run tests
 
-def compile_and_run( name, way, extra_hc_opts ):
+def compile_and_run__( name, way, extra_hc_opts, top_mod ):
     # print 'Compile and run, extra args = ', extra_hc_opts
     pretest_cleanup(name)
 
     if way == 'ghci': # interpreted...
-        return interpreter_run( name, way, extra_hc_opts, 0, '' )
+        return interpreter_run( name, way, extra_hc_opts, 0, top_mod )
     elif way == 'extcore' or way == 'optextcore' :
-        return extcore_run( name, way, extra_hc_opts, 0, '' )
+        return extcore_run( name, way, extra_hc_opts, 0, top_mod )
     else: # compiled...
-        result = simple_build( name, way, extra_hc_opts, 0, '', 1 )
-
+        result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1 )
+        
         if result != 0:
             return 'fail'
 
         # we don't check the compiler's stderr for a compile-and-run test
-        return simple_run( name, way, './'+name, getTestOpts().extra_run_opts, 0 )
+        return simple_run( name, way, './'+name, getTestOpts().extra_run_opts,
+                           ignore_output )
 
+def compile_and_run( name, way, extra_hc_opts ):
+    return compile_and_run__( name, way, extra_hc_opts, '' )
 
 def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
-    pretest_cleanup(name)
-
-    if way == 'ghci': # interpreted...
-        return interpreter_run( name, way, extra_hc_opts, 0, top_mod )
-    elif way == 'extcore' or way == 'optextcore' :
-        return extcore_run( name, way, extra_hc_opts, 0, top_mod )
-    else: # compiled...
-        result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1 )
-
-    if result != 0:
-        return 'fail'
-
-    # we don't check the compiler's stderr for a compile-and-run test
-    return simple_run( name, way, './'+name, getTestOpts().extra_run_opts, 0 )
-
-def multimod_compile_and_run_ignore_output( name, way, top_mod, extra_hc_opts ):
-    pretest_cleanup(name)
-
-    if way == 'ghci': # interpreted...
-        # not supported: exit code is too difficult to check.
-	return 'pass'
-    elif way == 'extcore' or way == 'optextcore' :
-        return extcore_run( name, way, extra_hc_opts, 0, top_mod )
-    else: # compiled...
-        result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1 )
-
-    if result != 0:
-        return 'fail'
-
-    # we don't check the compiler's stderr for a compile-and-run test
-    return simple_run( name, way, './'+name, getTestOpts().extra_run_opts, 1 )
+    return compile_and_run__( name, way, extra_hc_opts, top_mod )
 
 # -----------------------------------------------------------------------------
 # Build a single-module program

testsuite/tests/ghc-regress/cabal/all.T

 setTestOpts(compose(alone,only_compiler_types(['ghc'])))
 
-test('ghcpkg01', skip_if_fast, run_command_ignore_output, ['$MAKE ghcpkg01'])
+test('ghcpkg01', compose(ignore_output, skip_if_fast), run_command, ['$MAKE ghcpkg01'])
 clean(['local.package.conf', 'local.package.conf.old'])
-test('ghcpkg02', skip_if_fast, run_command_ignore_output, ['$MAKE ghcpkg02'])
+test('ghcpkg02', compose(ignore_output, skip_if_fast), run_command, ['$MAKE ghcpkg02'])
 clean(['package.conf.copy', 'package.conf.copy.old'])
-test('ghcpkg03', normal, run_command_ignore_output, ['$MAKE ghcpkg03'])
+test('ghcpkg03', ignore_output, run_command, ['$MAKE ghcpkg03'])
 clean(['local.package.conf', 'local.package.conf.old'])
-test('ghcpkg04', normal, run_command, ['$MAKE --no-print-directory ghcpkg04'])
+test('ghcpkg04', ignore_output, run_command, ['$MAKE --no-print-directory ghcpkg04'])
 clean(['local.package.conf', 'local.package.conf.old'])
 
 # Test that we *can* compile a module that also belongs to a package

testsuite/tests/ghc-regress/cabal/cabal01/all.T

 setTestOpts(only_compiler_types(['ghc']))
 
-test('cabal01', normal, run_command_ignore_output, ['$MAKE cabal01'])
+test('cabal01', ignore_output, run_command, ['$MAKE cabal01'])
 
 if default_testopts.cleanup != '':
     runCmd('$MAKE -C ' + in_testdir('') + ' clean')

testsuite/tests/ghc-regress/cabal/cabal02/all.T

-test('cabal02', skip_if_fast, run_command_ignore_output, ['$MAKE cabal02'])
+test('cabal02', compose(ignore_output, skip_if_fast), run_command, ['$MAKE cabal02'])

testsuite/tests/ghc-regress/driver/all.T

@@ -14,101 +14,101 @@ def cleanall():
         os.spawnlp(os.P_WAIT, 'rm', 'rm', '-rf', in_testdir('obj'))
         os.spawnlp(os.P_WAIT, 'rm', 'rm', '-rf', in_testdir('stub'))
 
-test('driver011', normal, run_command_ignore_output, ['$MAKE test011'])
+test('driver011', ignore_output, run_command, ['$MAKE test011'])
 cleanall()
-test('driver012', normal, run_command_ignore_output, ['$MAKE test012'])
+test('driver012', ignore_output, run_command, ['$MAKE test012'])
 cleanall()
-test('driver013', normal, run_command_ignore_output, ['$MAKE test013'])
+test('driver013', ignore_output, run_command, ['$MAKE test013'])
 cleanall()
-test('driver014', normal, run_command_ignore_output, ['$MAKE test014'])
+test('driver014', ignore_output, run_command, ['$MAKE test014'])
 cleanall()
-test('driver015', normal, run_command_ignore_output, ['$MAKE test015'])
+test('driver015', ignore_output, run_command, ['$MAKE test015'])
 cleanall()
-test('driver016', normal, run_command_ignore_output, ['$MAKE test016'])
+test('driver016', ignore_output, run_command, ['$MAKE test016'])
 cleanall()
-test('driver017', normal, run_command_ignore_output, ['$MAKE test017'])
+test('driver017', ignore_output, run_command, ['$MAKE test017'])
 cleanall()
-test('driver018', normal, run_command_ignore_output, ['$MAKE test018'])
+test('driver018', ignore_output, run_command, ['$MAKE test018'])
 cleanall()
-test('driver021', normal, run_command_ignore_output, ['$MAKE test021'])
+test('driver021', ignore_output, run_command, ['$MAKE test021'])
 cleanall()
-test('driver022', normal, run_command_ignore_output, ['$MAKE test022'])
+test('driver022', ignore_output, run_command, ['$MAKE test022'])
 cleanall()
-test('driver023', normal, run_command_ignore_output, ['$MAKE test023'])
+test('driver023', ignore_output, run_command, ['$MAKE test023'])
 cleanall()
-test('driver024', normal, run_command_ignore_output, ['$MAKE test024'])
+test('driver024', ignore_output, run_command, ['$MAKE test024'])
 cleanall()
-test('driver024a', normal, run_command_ignore_output, ['$MAKE test024a'])
+test('driver024a', ignore_output, run_command, ['$MAKE test024a'])
 cleanall()
-test('driver025', normal, run_command_ignore_output, ['$MAKE test025'])
+test('driver025', ignore_output, run_command, ['$MAKE test025'])
 cleanall()
-test('driver026', normal, run_command_ignore_output, ['$MAKE test026'])
+test('driver026', ignore_output, run_command, ['$MAKE test026'])
 cleanall()
-test('driver027', normal, run_command_ignore_output, ['$MAKE test027'])
+test('driver027', ignore_output, run_command, ['$MAKE test027'])
 cleanall()
-test('driver028', normal, run_command_ignore_output, ['$MAKE test028'])
+test('driver028', ignore_output, run_command, ['$MAKE test028'])
 cleanall()
-test('driver031', normal, run_command_ignore_output, ['$MAKE test031'])
+test('driver031', ignore_output, run_command, ['$MAKE test031'])
 cleanall()
-test('driver032', normal, run_command_ignore_output, ['$MAKE test032'])
+test('driver032', ignore_output, run_command, ['$MAKE test032'])
 cleanall()
-test('driver033', normal, run_command_ignore_output, ['$MAKE test033'])
+test('driver033', ignore_output, run_command, ['$MAKE test033'])
 cleanall()
-test('driver034', normal, run_command_ignore_output, ['$MAKE test034'])
+test('driver034', ignore_output, run_command, ['$MAKE test034'])
 cleanall()
-test('driver035', normal, run_command_ignore_output, ['$MAKE test035'])
+test('driver035', ignore_output, run_command, ['$MAKE test035'])
 cleanall()
-test('driver041', normal, run_command_ignore_output, ['$MAKE test041'])
+test('driver041', ignore_output, run_command, ['$MAKE test041'])
 cleanall()
-test('driver042', normal, run_command_ignore_output, ['$MAKE test042'])
+test('driver042', ignore_output, run_command, ['$MAKE test042'])
 cleanall()
-test('driver043', normal, run_command_ignore_output, ['$MAKE test043'])
+test('driver043', ignore_output, run_command, ['$MAKE test043'])
 cleanall()
-test('driver044', normal, run_command_ignore_output, ['$MAKE test044'])
+test('driver044', ignore_output, run_command, ['$MAKE test044'])
 cleanall()
-test('driver045', normal, run_command_ignore_output, ['$MAKE test045'])
+test('driver045', ignore_output, run_command, ['$MAKE test045'])
 cleanall()
-test('driver051', normal, run_command_ignore_output, ['$MAKE test051'])
+test('driver051', ignore_output, run_command, ['$MAKE test051'])
 cleanall()
-test('driver052', normal, run_command_ignore_output, ['$MAKE test052'])
+test('driver052', ignore_output, run_command, ['$MAKE test052'])
 cleanall()
-test('driver053', normal, run_command_ignore_output, ['$MAKE test053'])
+test('driver053', ignore_output, run_command, ['$MAKE test053'])
 cleanall()
 
-test('driver060', normal, run_command_ignore_output, ['$MAKE test060'])
+test('driver060', ignore_output, run_command, ['$MAKE test060'])
 cleanall()
-test('driver061', normal, run_command_ignore_output, ['$MAKE test061'])
+test('driver061', ignore_output, run_command, ['$MAKE test061'])
 cleanall()
 
-test('driver062.1', normal, run_command_ignore_output, ['$MAKE test062.1'])
+test('driver062.1', ignore_output, run_command, ['$MAKE test062.1'])
 cleanall()
-test('driver062.2', normal, run_command_ignore_output, ['$MAKE test062.2'])
+test('driver062.2', ignore_output, run_command, ['$MAKE test062.2'])
 cleanall()
-test('driver062.3', normal, run_command_ignore_output, ['$MAKE test062.3'])
+test('driver062.3', ignore_output, run_command, ['$MAKE test062.3'])
 cleanall()
 
 test('driver063', normal, run_command, ['$MAKE -s --no-print-directory test063'])
 cleanall()
-test('driver064', normal, run_command_ignore_output, ['$MAKE test064'])
+test('driver064', ignore_output, run_command, ['$MAKE test064'])
 cleanall()
-test('driver065', normal, run_command_ignore_output, ['$MAKE test065'])
+test('driver065', ignore_output, run_command, ['$MAKE test065'])
 cleanall()
-test('driver066', normal, run_command_ignore_output, ['$MAKE test066'])
+test('driver066', ignore_output, run_command, ['$MAKE test066'])
 cleanall()
-test('driver067', normal, run_command_ignore_output, ['$MAKE test067'])
+test('driver067', ignore_output, run_command, ['$MAKE test067'])
 cleanall()
-test('driver068', normal, run_command_ignore_output, ['$MAKE test068'])
+test('driver068', ignore_output, run_command, ['$MAKE test068'])
 cleanall()
-test('driver069', normal, run_command_ignore_output, ['$MAKE test069'])
+test('driver069', ignore_output, run_command, ['$MAKE test069'])
 cleanall()
-test('driver070', normal, run_command_ignore_output, ['$MAKE test070'])
+test('driver070', ignore_output, run_command, ['$MAKE test070'])
 cleanall()
-test('driver071', normal, run_command_ignore_output, ['$MAKE test071'])
+test('driver071', ignore_output, run_command, ['$MAKE test071'])
 cleanall()
 
-test('driver080', normal, run_command_ignore_output, ['$MAKE test080'])
+test('driver080', ignore_output, run_command, ['$MAKE test080'])
 cleanall()
 
-test('driver200', normal, run_command_ignore_output, ['$MAKE test200'])
+test('driver200', ignore_output, run_command, ['$MAKE test200'])
 cleanall()
 

testsuite/tests/ghc-regress/gadt/all.T

@@ -19,7 +19,7 @@ test('gadt14', normal, compile, [''])
 test('gadt15', normal, compile, [''])
 test('gadt16', normal, compile, [''])
 
-test('gadt17', normal, run_command_ignore_output, ['$MAKE gadt17'])
+test('gadt17', ignore_output, run_command, ['$MAKE gadt17'])
 clean(['Gadt17_help.hi', 'Gadt17_help.o'])
 
 test('gadt18', normal, compile, [''])

testsuite/tests/ghc-regress/ghci/prog004/prog004.T

@@ -5,5 +5,5 @@ def f(opts):
 	opts.skip = 1
 setTestOpts(f)
 
-test('ghciprog004', normal, run_command_ignore_output, ['$MAKE ghciprog004'])
+test('ghciprog004', ignore_output, run_command, ['$MAKE ghciprog004'])
 clean(['ctest.c', 'ctest.o'])

testsuite/tests/ghc-regress/typecheck/should_compile/all.T

@@ -183,7 +183,7 @@ test('tc171', normal, compile, [''])
 test('tc172', normal, compile, [''])
 
 # The point about this test is that it compiles Tc173a and Tc173b *separately*
-test('tc173', normal, run_command_ignore_output, ['$MAKE tc173'])
+test('tc173', ignore_output, run_command, ['$MAKE tc173'])
 clean(['Tc173a.hi', 'Tc173a.o', 'Tc173b.hi', 'Tc173b.o'])
 
 test('tc174', only_compiler_types(['ghc']), compile, [''])