Commit e5063a04 authored by simonmar's avatar simonmar

[project @ 2002-07-31 14:24:18 by simonmar]

Revamp the testsuite framework.  The previous framework was an
experiment that got a little out of control - a whole new language
with an interpreter written in Haskell was rather heavyweight and left
us with a maintenance problem.

So the new test driver is written in Python.  The downside is that you
need Python to run the testsuite, but we don't think that's too big a
problem since it only affects developers and Python installs pretty
easily onto everything these days.

Highlights:

  - 790 lines of Python, vs. 5300 lines of Haskell + 720 lines of
    <strange made-up language>.

  - the framework supports running tests in various "ways", which should
    catch more bugs.  By default, each test is run in three ways:
    normal, -O, and -O -fasm.  Additionally, if profiling libraries
    have been built, another way (-O -prof -auto-all) is added.  I plan
    to also add a 'GHCi' way.

    Running tests multiple ways has already shown up some new bugs!

  - documentation is in the README file and is somewhat improved.

  - the framework is rather less GHC-specific, and could without much
    difficulty be coaxed into using other compilers.  Most of the
    GHC-specificness is in a separate configuration file (config/ghc).

Things may need a while to settle down.  Expect some unexpected
failures.
parent 0d4aee25
This diff is collapsed.
# Testsuite configuration setup for GHC
#
# This file is Python source
#
config.compiler_type = 'ghc'
config.compiler = 'ghc'
config.compiler_always_flags = ['-no-recomp', '-dcore-lint']
config.compile_ways = ['normal', 'opt', 'optasm']
config.run_ways = ['normal', 'opt', 'optasm']
config.way_flags = { 'normal' : [],
'opt' : ['-O'],
'optasm' : ['-O -fasm'],
'prof' : ['-O -prof -auto-all'],
'unreg' : ['-unreg']
}
-----------------------------------------------------------------------
--- Stuff to do with multiple-source-file tests. We assume ---
--- that the name of the test is to be used as the basename ---
--- for everything. ---
-----------------------------------------------------------------------
-- global variables:
$stdin = ""
$expect = "pass"
$normalise_errmsg = False
$normalise_output = False
---------------------------------------------------------------
--- UTILITY FNs ---
---------------------------------------------------------------
include ($confdir ++ "/" ++ $conffilename)
include ($confdir ++ "/../std-macros.T")
-- (eg) "fooble" --> "testdir/fooble"
def testdirify ( $basename )
{
return $testdir ++ "/" ++ $basename
}
---------------------------------------------------------------
--- COMPILATION ---
---------------------------------------------------------------
-- Clean up prior to the test, so that we can't spuriously conclude
-- that it passed on the basis of old run outputs.
def pretest_cleanup()
{
rm_nofail(qualify("comp.stderr"))
rm_nofail(qualify("run.stderr"))
rm_nofail(qualify("run.stdout"))
-- simple_build_Main zaps the following:
-- objects
-- executable
-- not interested in the return code
}
-- Guess flags suitable for the compiler.
def guess_compiler_flags()
{
if $tool contains "ghc"
then
return "-no-recomp --make -dcore-lint" ++
" -i" ++ $testdir
else
-- Problem here is that nhc and hbc don't understand --make,
-- and we rely on it.
-- if $tool contains "nhc"
-- then
-- return "-an-nhc-specific-flag"
-- else
-- if $tool contains "hbc"
-- then
-- return ""
-- else
framefail ("Can't guess what kind of Haskell compiler " ++
"you're testing: $tool = " ++ $tool)
-- fi
-- fi
fi
}
-- Build Main, and return the compiler result code. Compilation
-- output goes into testname.comp.stderr. Source is assumed to
-- be in Main.hs or Main.lhs, and modules reachable from it.
def simple_build_prog_WRK ( $_main, $_extra_args )
{
$flags = guess_compiler_flags()
$errname = qualify("comp.stderr")
$exename = qualify("") -- ie, the exe name == the test name
rm_or_fail($errname)
rm_or_fail($exename)
rm_nofail(testdirify("*.o"))
$cmd = "\"" ++ $tool ++ "\" " ++ $flags ++ " " ++ $_extra_args ++ " "
++ (if defined $extra_hc_flags
then $extra_hc_flags
else "")
++ " -o " ++ $exename ++ " "
++ $_main ++ " >" ++ $errname ++ " 2>&1"
$res = run $cmd
return $res
}
---------------------------------------------------------------
--- CONDUCTING A COMPLETE TEST ---
---------------------------------------------------------------
-- Compile and run (should_run) style test
def multimod-compile( $mod, $extra_compile_args )
{
pretest_cleanup()
$main = if $mod == "" then "Main" else $mod
$res = simple_build_prog_WRK( $main, $extra_compile_args )
if $res /= "0" then
say_fail_because_compiler_barfd ( $res )
return False
else
return True
fi
}
def multimod-run-test ( $extra_run_args,
$allowable_nonzero_exit_code )
{
$exit_code =
if $allowable_nonzero_exit_code /= "" then
$allowable_nonzero_exit_code
else "0"
return simple_run_pgm( $extra_run_args, $exit_code )
}
---------------------------------------------------------------
--- TOP-LEVEL FNS ---
---------------------------------------------------------------
--------------------------------------------------------------
-- top-level
-- Compile and run (should_run) style test
def mtc ( $mod, $extra_compile_args )
{
$test_passed = multimod-compile( $mod, $extra_compile_args )
if ($expect == "pass") then
expect pass
else
expect fail
fi
pass when $test_passed
fail when otherwise
}
def mtr ( $extra_compile_args,
$extra_run_args,
$allowable_nonzero_exit_code )
{
$test_passed
= multimod-compile( "Main", $extra_compile_args )
&& multimod-run-test( $extra_run_args,
$allowable_nonzero_exit_code )
if ($expect == "pass") then
expect pass
else
expect fail
fi
pass when $test_passed
fail when otherwise
}
-----------------------------------------------------------------------
--- end multimod-test.T ---
-----------------------------------------------------------------------
include ($confdir ++ "/../singlefile-macros.T")
expect pass
pretest_cleanup()
$res = simple_compile_Main()
pass when contents("comp.stdout") == ""
fail when otherwise
include ($confdir ++ "/../singlefile-macros.T")
expect pass
pretest_cleanup()
$res = simple_compile_Main()
pass when
$tool contains "ghc"
&& contents("comp.stdout") contains "Could not deduce"
-- put a pass clause here for NHC
fail when otherwise
include ($confdir ++ "/../singlefile-macros.T")
expect pass
pretest_cleanup()
simple_build_Main()
$res = simple_run_main_no_stdin()
pass when contents("run.stdout") == "True\n"
fail when otherwise
\ No newline at end of file
$diff = "diff -C 2"
$rm = "rm -f"
$cp = "cp"
-- -----------------------------------------------------------------------------
-- generic useful stuff
-- Gotta do some pretty basic stuff :)
def not ( $_bool )
{
if $_bool == "True" then return False
else if $_bool == "False" then return True
else framefail ("not(): invalid input: " ++ $_bool )
fi fi
}
def runCmd( $cmd )
{
if defined $verbose then
print $cmd
fi
$res = run $cmd
return $res
}
def runCmdDontFail( $cmd )
{
$res = runCmd($cmd)
if $res /= "0"
then framefail ("unexpected cmd failure: " ++ $cmd)
fi
}
-- Delete a file and abort if that doesn't work.
def rm_or_fail ( $_files )
{
$cmd = $rm ++ " " ++ $_files
$res = runCmd($cmd)
if $res /= "0" then framefail ("rm_or_fail: can't rm: " ++ $_files) fi
}
-- Delete a file but keep going antidisirregardless of the outcome.
def rm_nofail ( $_files )
{
$cmd = $rm ++ " " ++ $_files
$res = runCmd($cmd)
}
-----------------------------------------------------------------------------
-- (eg) "run.stdout" --> "testdir/testname.run.stdout"
def qualify ( $_filename_frag )
{
if $_filename_frag == ""
then
return $testdir ++ "/" ++ $testname
else
return $testdir ++ "/" ++ $testname ++ "." ++ $_filename_frag
fi
}
-- "foo" -> qualify("foo-platform") if it exists, or qualify("foo") otherwise
def platform_qualify ( $_filename_frag )
{
$name = qualify($_filename_frag)
if exists($name ++ "-" ++ $platform)
then return $name ++ "-" ++ $platform
else return $name
fi
}
def testnameWith ( $_filename_frag )
{
if $_filename_frag == ""
then
return $testname
else
return $testname ++ "." ++ $_filename_frag
fi
}
-- Clean up prior to the test, so that we can't spuriously conclude
-- that it passed on the basis of old run outputs.
def pretest_cleanup()
{
rm_nofail(qualify("comp.stderr"))
rm_nofail(qualify("run.stderr"))
rm_nofail(qualify("run.stdout"))
-- simple_build_Main zaps the following:
-- rm_nofail(qualify("o"))
-- rm_nofail(qualify(""))
-- not interested in the return code
}
-- Pipe an error message through normalise_errmsg.
def normalise_errmsg ( $errmsg )
{
$unpathify = $confdir ++ "/../../utils/normalise_errmsg/normalise_errmsg"
$normd = $errmsg | $unpathify
return $normd
}
-- returns True if both files are identical.
def same ( $_file1, $_file2 )
{
if defined $verbose
then print "vanilla-test: comparing " ++ $_file1
++ " and " ++ $_file2
fi
$cts1 = contents($_file1)
$cts2 = contents($_file2)
$same = $cts1 == $cts2
if not($same) then
say_fail_because_noteq($_file1, $_file2)
fi
return $same
}
-- returns True if both files are identical when normalised (unpathified)
def same_normalised ( $_file1, $_file2 )
{
if defined $verbose
then print "vanilla-test: comparing " ++ $_file1
++ " and " ++ $_file2
fi
$cts1 = normalise_errmsg(contents($_file1))
$cts2 = normalise_errmsg(contents($_file2))
$same = $cts1 == $cts2
if not($same) then
say_fail_because_noteq($_file1, $_file2)
fi
return $same
}
-- Give hints as to why a test is failing.
def say_fail_because_noteq ( $filename1, $filename2 )
{
print "--- FAIL because the following files differ:"
print "--- " ++ $filename1
print "--- " ++ $filename2
if defined $accept then
print "--- (accepting new output)"
runCmdDontFail($cp ++ " " ++ $filename2 ++ " " ++ $filename1)
fi
if defined $verbose then
$ignore = runCmd($diff ++ " " ++ $filename1 ++ " " ++ $filename2)
fi
}
def say_fail_because_nonempty ( $filename1 )
{
print "--- FAIL because the following file is non-empty:"
print "--- " ++ $filename1
print "--- contents:"
print (contents $filename1)
print ("--- end of " ++ $filename1)
}
def say_fail_because_exit_code_wrong ( $prg, $exit_code, $should_be )
{
print "--- FAIL because $prog had the wrong exit code (" ++
$exit_code ++ ", should be " ++ $should_be ++ ")"
}
def say_fail_because_compiler_barfd ( $res )
{
print "--- FAIL because the compiler returned non-zero exit code = " ++ $res
$comp_stderr = qualify("comp.stderr")
if exists($comp_stderr)
then print "--- Error messages:"
print contents(qualify("comp.stderr"))
fi
}
---------------------------------------------------------------
--- RUNNING, AND ASSESSING RUN RESULTS ---
---------------------------------------------------------------
-- Run testname. If testname.stdin exists, route input from that, else
-- from /dev/null. Route output to testname.run.stdout and
-- testname.run.stderr. Returns the exit code of the run.
def simple_run_pgm( $extra_args, $exit_code )
{
-- figure out what to use for stdin
$devnull = "/dev/null"
if $stdin /= "" then
$use_stdin = $stdin
else
$stdin = testnameWith("stdin")
$stdin_path = $testdir ++ "/" ++ $stdin
$use_stdin = if exists($stdin_path) then $stdin else $devnull
fi
$run_stdout = testnameWith("run.stdout")
$run_stderr = testnameWith("run.stderr")
rm_or_fail($run_stdout)
rm_or_fail($run_stderr)
$cmd = "cd " ++ $testdir ++ " && "
++ "./" ++ $testname ++ " " ++ $extra_args
++ " < " ++ $use_stdin
++ " > " ++ $run_stdout
++ " 2> " ++ $run_stderr
-- run the command
$res = runCmd($cmd)
-- check the exit code
if $res /= $exit_code then
say_fail_because_exit_code_wrong($testname, $res, $exit_code)
return False
fi
-- check the stdout and stderr outputs
$test_passed = check_stdout_ok() && check_stderr_ok()
return $test_passed
}
-- Check that the run.stdout file matches either the .stdout-TARGETPLATFORM
-- (if it exists) or the .stdout otherwise.
def check_stdout_ok()
{
$r_stdout = qualify("run.stdout")
$s_stdout = platform_qualify("stdout")
if not ( exists($s_stdout) )
then if ((contents $r_stdout) == "")
then return True
else say_fail_because_nonempty($r_stdout)
return False
fi
fi
if $normalise_output
then return same_normalised($s_stdout, $r_stdout)
else return same($s_stdout, $r_stdout)
fi
}
-- Check that the run.stderr matches either the .stderr-TARGETPLATFORM
-- (if it exists) or the .stderr otherwise. Normalise the stderr if
-- $normalise_errmsg is set
def check_stderr_ok()
{
$r_stderr = qualify("run.stderr")
$s_stderr = platform_qualify("stderr")
if not ( exists($s_stderr) )
then if ((contents $r_stderr) == "")
then return True
else say_fail_because_nonempty($r_stderr)
return False
fi
fi
-- if $normalise_errmsg
-- then
return same_normalised($s_stderr, $r_stderr)
-- else return same($s_stderr, $r_stderr)
-- fi
}
-----------------------------------------------------------------------
--- Stuff to do with simple single-source-file tests. We assume ---
--- that the name of the test is to be used as the basename ---
--- for everything. ---
-----------------------------------------------------------------------
-- global variables:
$stdin = ""
$expect = "pass"
$normalise_errmsg = False
$normalise_output = False
---------------------------------------------------------------
-- Define the following things on the command line:
--
-- $platform TARGETPLATFORM from config.mk
-- $verbose print command lines
-- $accept accept any changed output
---------------------------------------------------------------
--- UTILITY FNs ---
---------------------------------------------------------------
include ($confdir ++ "/" ++ $conffilename)
include ($confdir ++ "/../std-macros.T")
---------------------------------------------------------------
--- COMPILATION ---
---------------------------------------------------------------
-- Guess flags suitable for the compiler.
def guess_compiler_flags()
{
if $tool contains "ghc"
then
return "-no-recomp -dcore-lint"
else
if $tool contains "nhc"
then
return "-an-nhc-specific-flag"
else
if $tool contains "hbc"
then
return ""
else
framefail ("Can't guess what kind of Haskell compiler " ++
"you're testing: $tool = " ++ $tool)
fi
fi
fi
}
-- Build Main, and return the compiler result code. Compilation
-- output goes into testname.comp.stderr.
def simple_build_Main_WRK ( $_extra_args, $compile_only )
{
$flags = guess_compiler_flags()
$errname = testnameWith("comp.stderr")
$srcname = testnameWith("hs")
rm_or_fail($errname)
rm_or_fail($testname)
$cmd = "cd " ++ $testdir ++ " && \"" ++
$tool ++ "\" "
++ (if defined $compile_to_hc && $compile_to_hc
then "-C "
else if $compile_only
then "-c "
else "-o " ++ $testname ++ " ")
++ $srcname ++ " "
++ $flags ++ " " ++ $_extra_args ++ " "
++ (if defined $extra_hc_flags
then $extra_hc_flags ++ " "
else "")
++ ">" ++ $errname ++ " 2>&1"
$res = runCmd($cmd)
return $res
}
---------------------------------------------------------------
--- CONDUCTING A COMPLETE TEST ---
---------------------------------------------------------------
-- Compile and run (should_run) style test
def vanilla-run-test-actions ( $extra_compile_args,
$extra_run_args,
$allowable_nonzero_exit_code )
{
pretest_cleanup()
$res = simple_build_Main_WRK( $extra_compile_args, False )
-- If the compiler barf'd, fail.
if $res /= "0"
then say_fail_because_compiler_barfd ( $res )
return False
fi
$exit_code =
if $allowable_nonzero_exit_code /= "" then
$allowable_nonzero_exit_code
else "0"
return simple_run_pgm( $extra_run_args, $exit_code )
}
-- Compile only (should_compile) style test. Deemed to have
-- succeeded if the compiler returned zero AND (testname.comp.stderr
-- matches testname.stderr, if it exists, or is empty).
def vanilla-compok-test-actions ( $extra_compile_args )
{
pretest_cleanup()
$res = simple_build_Main_WRK ( $extra_compile_args, True )
-- If the compiler barf'd, fail.
if $res /= "0"
then say_fail_because_compiler_barfd ( $res )
return False
fi
-- If there's an expected .stderr, presumably containing
-- warnings, ensure the compiler produced the same.
$actual_stderr = qualify("comp.stderr")
$expected_stderr = platform_qualify("stderr")
if exists($expected_stderr)
then $stderr_a = normalise_errmsg(contents($actual_stderr))
$stderr_e = normalise_errmsg(contents($expected_stderr))
if $stderr_e /= $stderr_a
then say_fail_because_noteq($expected_stderr, $actual_stderr)
return False
else return True
fi
fi
-- There's no expected stderr, so just insist that the compiler
-- produced nothing on stderr.
if (contents $actual_stderr) /= ""
then say_fail_because_nonempty($actual_stderr)
return False
fi
-- Must have succeeded.
return True
}
-- Compile with expected fail (should_fail) style test. Deemed to have
-- succeeded if the compiler returned nonzero AND testname.comp.stderr
-- equals testname.stderr.
def vanilla-compfail-test-actions ( $extra_compile_args )
{
pretest_cleanup()
$expected_stderr = platform_qualify("stderr")