Commit ff6b4956 authored by simonmar's avatar simonmar

[project @ 2006-01-18 16:31:10 by simonmar]

Add a fast version of the testsuite

The idea is to have a way to run as much of the testsuite as possible
in a short time, so that we'll run it more often (such as just before
checking in a change, for example).  'make fast' tries for good
coverage without using too many cycles.  Currently it takes about 4
minutes on a fast machine with an optimised GHC build; I think this
might still be a little on the slow side.

When you say 'make fast' in testsuite/tests/ghc-regress, we run each
test only one way, and all of the long-running tests are omitted.
Also, to get the runtime down further, I arbitrarily omitted many of
the should_run tests (because these tend to take a lot longer than
should_compile or should_fail tests).  I tried to keep a
representative few in each category.
parent adf1df0b
......@@ -9,6 +9,12 @@ To run the test suite against a GHC build in the same source tree:
cd fptools/testsuite/tests/ghc-regress
make
To run a fast version of the testsuite, which should complete in under
5 minutes on a fast machine with an optimised GHC build:
cd fptools/testsuite/tests/ghc-regress
make fast
To run the test suite against a different GHC, say ghc-5.04:
cd fptools/testsuite/tests/ghc-regress
......
......@@ -48,6 +48,9 @@ class TestConfig:
# Verbosity level
self.verbose = 1
# run the "fast" version of the test suite
self.fast = 0
# Compiler type (ghc, hugs, nhc, etc.)
self.compiler_type = ''
......@@ -289,6 +292,12 @@ def skip_if_no_ghci(opts):
if not ('ghci' in config.run_ways):
opts.skip = 1
# ----
def skip_if_fast(opts):
if config.fast:
opts.skip = 1
# ----
# Function for composing two opt-fns together
......@@ -322,6 +331,12 @@ def getTestDir():
def test( name, setup, func, args ):
t.total_tests = t.total_tests + 1
# Reset the test-local options to the options for this "set"
resetTestOpts()
# Set our test-local options
setup(testopts)
if func == compile or func == multimod_compile:
ways = config.compile_ways
elif func == compile_and_run or func == multimod_compile_and_run:
......@@ -335,7 +350,15 @@ def test( name, setup, func, args ):
ways = ['normal']
for way in ways:
do_test( name, way, setup, func, args )
if testopts.skip \
or (config.only != [] and name not in config.only) \
or (testopts.only_ways != [] and way not in testopts.only_ways) \
or way in testopts.omit_ways:
skiptest(name)
else:
do_test( name, way, func, args )
if config.fast:
break # just do the first way in fast mode
clean(map (lambda suff: name + suff,
['', '.genscript', '.run.stderr', '.run.stdout',
......@@ -356,24 +379,11 @@ def clean_full_paths(names):
if os.access(name, os.F_OK) :
os.remove(name)
def do_test(name, way, setup, func, args):
def do_test(name, way, func, args):
full_name = name + '(' + way + ')'
t.total_test_cases = t.total_test_cases + 1
try:
# Reset the test-local options to the options for this "set"
resetTestOpts()
# Set our test-local options
setup(testopts)
if testopts.skip \
or (config.only != [] and name not in config.only) \
or (testopts.only_ways != [] and way not in testopts.only_ways) \
or way in testopts.omit_ways:
skiptest(name)
return
print '=====>', full_name
result = apply(func, [name,way] + args)
......
......@@ -85,9 +85,21 @@ RUNTEST_OPTS += \
-e config.timeout_prog=\"$(TOP)/timeout/timeout\" \
$(EXTRA_RUNTEST_OPTS)
ifeq "$(fast)" "YES"
setfast = -e config.fast=1
else
setfast =
endif
ifeq "$(accept)" "YES"
setaccept = -e config.accept=1
else
setaccept =
endif
TESTS =
TEST =
WAY =
WAY =
all :: test
......@@ -95,14 +107,15 @@ test:
$(PYTHON) $(RUNTESTS) $(RUNTEST_OPTS) \
$(patsubst %, --only=%, $(TEST)) \
$(patsubst %, --only=%, $(TESTS)) \
$(patsubst %, --way=%, $(WAY))
$(patsubst %, --way=%, $(WAY)) \
$(setfast)
$(setaccept)
verbose: test
accept:
$(PYTHON) $(RUNTESTS) $(RUNTEST_OPTS) \
$(patsubst %, --only=%, $(TEST)) \
$(patsubst %, --only=%, $(TESTS)) \
$(patsubst %, --way=%, $(WAY)) \
-e config.accept=1
$(MAKE) accept=YES
fast:
$(MAKE) fast=YES
......@@ -4,20 +4,20 @@
# extra run flags
# expected process return value, if not zero
test('arr001', normal, compile_and_run, [''])
test('arr002', normal, compile_and_run, [''])
test('arr003', exit_code(1), compile_and_run, [''])
test('arr004', exit_code(1), compile_and_run, [''])
test('arr005', normal, compile_and_run, [''])
test('arr006', normal, compile_and_run, [''])
test('arr007', exit_code(1), compile_and_run, [''])
test('arr008', exit_code(1), compile_and_run, [''])
test('arr009', normal, compile_and_run, [''])
test('arr010', normal, compile_and_run, [''])
test('arr011', normal, compile_and_run, [''])
test('arr012', normal, compile_and_run, [''])
test('arr013', normal, compile_and_run, [''])
test('arr014', normal, compile_and_run, [''])
test('arr015', normal, compile_and_run, [''])
test('arr001', skip_if_fast, compile_and_run, [''])
test('arr002', skip_if_fast, compile_and_run, [''])
test('arr003', compose(skip_if_fast,exit_code(1)), compile_and_run, [''])
test('arr004', compose(skip_if_fast,exit_code(1)), compile_and_run, [''])
test('arr005', skip_if_fast, compile_and_run, [''])
test('arr006', skip_if_fast, compile_and_run, [''])
test('arr007', compose(skip_if_fast,exit_code(1)), compile_and_run, [''])
test('arr008', compose(skip_if_fast,exit_code(1)), compile_and_run, [''])
test('arr009', skip_if_fast, compile_and_run, [''])
test('arr010', skip_if_fast, compile_and_run, [''])
test('arr011', skip_if_fast, compile_and_run, [''])
test('arr012', skip_if_fast, compile_and_run, [''])
test('arr013', skip_if_fast, compile_and_run, [''])
test('arr014', skip_if_fast, compile_and_run, [''])
test('arr015', skip_if_fast, compile_and_run, [''])
test('arr016', normal, compile_and_run, ['-fglasgow-exts'])
test('arr017', normal, compile_and_run, [''])
test('arr017', skip_if_fast, compile_and_run, [''])
setTestOpts(only_compiler_types(['ghc']))
test('arrowrun001', normal, compile_and_run, [''])
test('arrowrun002', normal, compile_and_run, [''])
test('arrowrun002', skip_if_fast, compile_and_run, [''])
test('arrowrun003', normal, compile_and_run, [''])
test('arrowrun004', normal, compile_and_run, [''])
test('arrowrun004', skip_if_fast, compile_and_run, [''])
setTestOpts(only_compiler_types(['ghc']))
test('ghcpkg01', normal, run_command_ignore_output, ['$MAKE ghcpkg01'])
test('ghcpkg02', normal, run_command_ignore_output, ['$MAKE ghcpkg02'])
test('ghcpkg01', skip_if_fast, run_command_ignore_output, ['$MAKE ghcpkg01'])
test('ghcpkg02', skip_if_fast, run_command_ignore_output, ['$MAKE ghcpkg02'])
test('ghcpkg03', normal, run_command_ignore_output, ['$MAKE ghcpkg03'])
test('ghcpkg04', normal, run_command, ['$MAKE --no-print-directory ghcpkg04'])
......
......@@ -49,7 +49,7 @@ elif config.platform.startswith('i386-'):
else:
ways = expect_fail_for(['extcore','optextcore'])
test('ffi009', ways ,compile_and_run, [''])
test('ffi009', compose(skip_if_fast,ways) ,compile_and_run, [''])
# Doesn't work with External Core due to __labels
test('ffi010', expect_fail_for(['extcore','optextcore']), compile_and_run, [''])
......
......@@ -25,7 +25,7 @@ test('cg024', normal, compile_and_run, [''])
test('cg025', compose(extra_run_opts('cg025.hs'),exit_code(1)), \
compile_and_run, [''])
test('cg026', only_compiler_types(['ghc']), compile_and_run, [''])
test('cg026', compose(skip_if_fast,only_compiler_types(['ghc'])), compile_and_run, [''])
test('cg027', normal, compile_and_run, [''])
test('cg028', normal, compile_and_run, [''])
test('cg031', only_compiler_types(['ghc']), compile_and_run, ['-fglasgow-exts'])
......
# Test for bug #1285326, results in "internal error: scavenge_one:
# strange object 47" with GHC 6.4, fixed in 6.4.1.
test('concprog001', normal, multimod_compile_and_run, ['Mult',''])
test('concprog001', skip_if_fast, multimod_compile_and_run, ['Mult',''])
clean_o_hi()
# Args to compile_and_run are:
# extra compile flags
# extra run flags
# expected process return value, if not zero
# -----------------------------------------------------------------------------
# These tests we do even for 'make fast'
def f( opts ):
opts.extra_hc_opts = '-fglasgow-exts'
setTestOpts(f)
test('conc003', normal, compile_and_run, [''])
test('conc006', normal, compile_and_run, [''])
test('conc027', normal, compile_and_run, [''])
test('conc049', only_compiler_types(['ghc']), compile_and_run, ['-package stm'])
test('conc051', normal, compile_and_run, [''])
# -----------------------------------------------------------------------------
# These tests we only do for a full run
def f( opts ):
opts.extra_hc_opts = '-fglasgow-exts'
if config.fast:
opts.skip = 1
setTestOpts(f)
test('conc001', normal, compile_and_run, [''])
test('conc002', normal, compile_and_run, [''])
test('conc003', normal, compile_and_run, [''])
# Omit GHCi way - it blows up to 0.5G. Something to do with the threaded RTS?
test('conc004', omit_ways(['ghci']), compile_and_run, [''])
test('conc006', normal, compile_and_run, [''])
test('conc007', compose(only_compiler_types(['ghc']),
extra_run_opts('+RTS -H128M -RTS')),
compile_and_run, [''])
......@@ -39,11 +51,10 @@ test('conc020', only_compiler_types(['ghc']), compile_and_run, [''])
test('conc021', compose(omit_ways(['ghci']), exit_code(1)),
compile_and_run, [''])
test('conc022', only_compiler_types(['ghc']), compile_and_run, [''])
test('conc023', only_compiler_types(['ghc']), compile_and_run, [''])
test('conc023', compose(skip_if_fast,only_compiler_types(['ghc'])), compile_and_run, [''])
test('conc024', only_compiler_types(['ghc']), compile_and_run, [''])
test('conc025', normal, compile_and_run, [''])
test('conc026', only_compiler_types(['ghc']), compile_and_run, [''])
test('conc027', normal, compile_and_run, [''])
test('conc028', normal, compile_and_run, [''])
test('conc029', normal, compile_and_run, [''])
test('conc030', compose(only_compiler_types(['ghc']),
......@@ -72,7 +83,8 @@ test('conc035', only_compiler_types(['ghc']), compile_and_run, [''])
# it is waiting for a ^C interrupt.
# Omit for threaded: deadlock detection isn't working properly.
test('conc036', compose(omit_ways(['ghci','threaded']),
only_compiler_types(['ghc'])), compile_and_run, [''])
compose(skip_if_fast,
only_compiler_types(['ghc']))), compile_and_run, [''])
test('conc037', only_ways(['threaded','smp']), compile_and_run, [''])
test('conc038', only_ways(['threaded','smp']), compile_and_run, [''])
......@@ -103,8 +115,6 @@ test('conc047', compose(only_compiler_types(['ghc']),
test('conc048', compose(only_compiler_types(['ghc']),
omit_ways(['threaded','ghci'])), compile_and_run, [''])
test('conc049', only_compiler_types(['ghc']), compile_and_run, ['-package stm'])
# skip this test right now - for some reason it goes on for ever in the
# 'opt' ways. (ToDo)
# test('conc050', compose(only_compiler_types(['ghc']), extra_run_opts('10000')), compile_and_run, ['-package stm'])
......@@ -117,3 +127,4 @@ test('conc053', only_ways(['threaded','smp']), compile_and_run, ['-package stm']
test('conc054', normal, compile_and_run, ['-package stm'])
test('conc055', exit_code(1), compile_and_run, ['-package stm'])
test('conc056', only_ways(['threaded']), compile_and_run, ['-package stm -package network'])
......@@ -13,6 +13,6 @@ test('dsrun007', exit_code(1), compile_and_run, [''])
test('dsrun008', exit_code(1), compile_and_run, [''])
test('dsrun009', normal, compile_and_run, [''])
test('dsrun010', normal, compile_and_run, [''])
test('dsrun011', normal, compile_and_run, [''])
test('dsrun012', normal, compile_and_run, [''])
test('dsrun011', skip_if_fast, compile_and_run, [''])
test('dsrun012', skip_if_fast, compile_and_run, [''])
test('dsrun013', normal, compile_and_run, [''])
......@@ -3,21 +3,21 @@
# extra run flags
# expected process return value, if not zero
test('drvrun001', normal, compile_and_run, [''])
test('drvrun002', normal, compile_and_run, [''])
test('drvrun003', normal, compile_and_run, [''])
test('drvrun004', normal, compile_and_run, [''])
test('drvrun005', normal, compile_and_run, [''])
test('drvrun001', skip_if_fast, compile_and_run, [''])
test('drvrun002', skip_if_fast, compile_and_run, [''])
test('drvrun003', skip_if_fast, compile_and_run, [''])
test('drvrun004', skip_if_fast, compile_and_run, [''])
test('drvrun005', skip_if_fast, compile_and_run, [''])
test('drvrun006', normal, compile_and_run, [''])
test('drvrun007', normal, compile_and_run, [''])
test('drvrun008', normal, compile_and_run, ['-funbox-strict-fields'])
test('drvrun009', normal, compile_and_run, [''])
test('drvrun010', normal, compile_and_run, [''])
test('drvrun011', normal, compile_and_run, [''])
test('drvrun012', normal, compile_and_run, [''])
test('drvrun013', normal, compile_and_run, [''])
test('drvrun014', normal, compile_and_run, [''])
test('drvrun015', normal, compile_and_run, [''])
test('drvrun016', normal, compile_and_run, ['-funbox-strict-fields'])
test('drvrun017', only_compiler_types(['ghc']), compile_and_run, [''])
test('drvrun018', normal, compile_and_run, [''])
test('drvrun007', skip_if_fast, compile_and_run, [''])
test('drvrun008', skip_if_fast, compile_and_run, ['-funbox-strict-fields'])
test('drvrun009', skip_if_fast, compile_and_run, [''])
test('drvrun010', skip_if_fast, compile_and_run, [''])
test('drvrun011', skip_if_fast, compile_and_run, [''])
test('drvrun012', skip_if_fast, compile_and_run, [''])
test('drvrun013', skip_if_fast, compile_and_run, [''])
test('drvrun014', skip_if_fast, compile_and_run, [''])
test('drvrun015', skip_if_fast, compile_and_run, [''])
test('drvrun016', skip_if_fast, compile_and_run, ['-funbox-strict-fields'])
test('drvrun017', compose(skip_if_fast, only_compiler_types(['ghc'])), compile_and_run, [''])
test('drvrun018', skip_if_fast, compile_and_run, [''])
test('recomp001', normal, run_command, ['$MAKE -s --no-print-directory recomp001'])
test('recomp001', skip_if_fast, run_command, ['$MAKE -s --no-print-directory recomp001'])
setTestOpts(only_compiler_types(['ghc']))
setTestOpts(only_ways(['normal']));
# In fast mode, we omit all the compile_and_run tests except a couple
test('gadt1', normal, compile, [''])
test('gadt2', normal, compile_and_run, [''])
test('gadt2', skip_if_fast, compile_and_run, [''])
test('gadt3', normal, compile, [''])
test('gadt4', normal, compile_and_run, [''])
test('gadt5', normal, compile_and_run, [''])
test('gadt4', skip_if_fast, compile_and_run, [''])
test('gadt5', skip_if_fast, compile_and_run, [''])
test('gadt6', normal, compile, [''])
test('gadt7', normal, compile_fail, [''])
test('gadt8', normal, compile, [''])
......@@ -16,7 +18,7 @@ test('gadt13', normal, compile_fail, [''])
test('gadt14', normal, compile, [''])
test('red-black', normal, compile, [''])
test('type-rep', normal, compile_and_run, [''])
test('type-rep', skip_if_fast, compile_and_run, [''])
test('equal', normal, compile, [''])
test('nbe', normal, compile, [''])
test('while', normal, compile_and_run, [''])
......@@ -26,9 +28,9 @@ test('lazypatok', normal, compile_fail, [''])
test('tc', normal, compile_and_run, [''])
test('arrow', normal, compile, [''])
test('tdpe', normal, compile, [''])
test('Nilsson', normal, compile, [''])
test('Nilsson', skip_if_fast, compile, [''])
test('records', normal, compile_and_run, [''])
test('records', skip_if_fast, compile_and_run, [''])
test('records-fail1', normal, compile_fail, [''])
# New ones from Dimitrios
......
......@@ -18,7 +18,7 @@ test('hlist', normal, compile_and_run, [''])
test('hopat', normal, compile_and_run, [''])
test('foldTree', normal, compile_and_run, [''])
test('newtype', normal, compile_and_run, [''])
test('perm', normal, compile_and_run, [''])
test('perm', skip_if_fast, compile_and_run, [''])
test('typecase1', normal, compile_and_run, [''])
test('typecase2', normal, compile_and_run, [''])
test('typeableY', normal, compile_and_run, [''])
setTestOpts(only_compiler_types(['ghc']))
test( 'bits',
normal,
skip_if_fast,
multimod_compile_and_run,
['Main','']
)
......
setTestOpts(only_compiler_types(['ghc']))
test( 'genUpTo',
normal,
skip_if_fast,
multimod_compile_and_run,
['Main','']
)
......
setTestOpts(only_compiler_types(['ghc']))
test( 'geq',
normal,
skip_if_fast,
multimod_compile_and_run,
['Main','']
)
......
setTestOpts(only_compiler_types(['ghc']))
test( 'gshow',
normal,
test( 'gshow',
skip_if_fast,
multimod_compile_and_run,
['Main','']
)
......
setTestOpts(only_compiler_types(['ghc']))
test( 'gzip',
normal,
skip_if_fast,
multimod_compile_and_run,
['Main','']
)
......
setTestOpts(only_compiler_types(['ghc']))
test( 'paradise',
normal,
skip_if_fast,
multimod_compile_and_run,
['Main','']
)
......
setTestOpts(only_compiler_types(['ghc']))
test( 'reify',
normal,
skip_if_fast,
multimod_compile_and_run,
['Main','']
)
......
setTestOpts(only_compiler_types(['ghc']))
test( 'strings',
normal,
skip_if_fast,
multimod_compile_and_run,
['Main','']
)
......
setTestOpts(only_compiler_types(['ghc']))
test( 'tree',
normal,
skip_if_fast,
multimod_compile_and_run,
['Main','']
)
......
setTestOpts(only_compiler_types(['ghc']))
test( 'xmlish',
normal,
skip_if_fast,
multimod_compile_and_run,
['Main','']
)
......
......@@ -78,6 +78,7 @@ test('readwrite002', compose(omit_ways(['ghci']), set_stdin('readwrite002.hs')),
clean(['readwrite002.inout'])
test('hGetBuf001', compose(only_compiler_types(['ghc']),
expect_fail_if_windows), compile_and_run, ['-package unix'])
compose(skip_if_fast,
expect_fail_if_windows)), compile_and_run, ['-package unix'])
test('hDuplicateTo001', normal, compile_and_run, [''])
test('ioref001', extra_run_opts('+RTS -K32m -RTS'), \
test('ioref001', compose(skip_if_fast,extra_run_opts('+RTS -K32m -RTS')), \
compile_and_run, ['+RTS -K32m -RTS'])
# need to add -K32m to the compiler opts, so that GHCi gets it too
......@@ -3,4 +3,4 @@ test('net001', only_compiler_types(['ghc']), compile_and_run, ['-package network
test('net002', compose(only_compiler_types(['ghc']), extra_run_opts('3')),
compile_and_run, ['-package network'])
test('uri001', normal, compile_and_run, ['-package network -package HUnit'])
test('uri001', skip_if_fast, compile_and_run, ['-package network -package HUnit'])
......@@ -11,25 +11,25 @@ test('text001', normal, compile_and_run, [''])
test('tup001', normal, compile_and_run, [''])
test('addr001', normal, compile_and_run, [''])
test('dynamic001', normal, compile_and_run, [''])
test('dynamic002', normal, compile_and_run, [''])
test('enum01', normal, compile_and_run, ['-cpp'])
test('enum02', normal, compile_and_run, ['-cpp'])
test('enum03', normal, compile_and_run, ['-cpp'])
test('dynamic001', skip_if_fast, compile_and_run, [''])
test('dynamic002', skip_if_fast, compile_and_run, [''])
test('enum01', skip_if_fast, compile_and_run, ['-cpp'])
test('enum02', skip_if_fast, compile_and_run, ['-cpp'])
test('enum03', skip_if_fast, compile_and_run, ['-cpp'])
test('enum04', expect_fail, compile_and_run, [ ''])
test('exceptions001', normal, compile_and_run, [''])
test('exceptions002', normal, compile_and_run, [''])
test('list001' , normal, compile_and_run, [''])
test('list001' , skip_if_fast, compile_and_run, [''])
test('memo001', extra_run_opts('+RTS -A10k -RTS'), \
test('memo001', compose(skip_if_fast,extra_run_opts('+RTS -A10k -RTS')), \
multimod_compile_and_run, ['memo001',''])
test('memo002', extra_run_opts('20'), \
test('memo002', compose(skip_if_fast,extra_run_opts('20')), \
multimod_compile_and_run, ['memo002',''])
test('packedstring001', normal, compile_and_run, [''])
test('stableptr001', extra_run_opts('+RTS -K8m -RTS'), compile_and_run, [''])
test('stableptr001', compose(skip_if_fast,extra_run_opts('+RTS -K8m -RTS')), compile_and_run, [''])
test('stableptr003', normal, compile_and_run, [''])
test('stableptr004', extra_run_opts('+RTS -K4m -RTS'), compile_and_run, [''])
test('stableptr005', normal, compile_and_run, [''])
......
......@@ -3,13 +3,15 @@
# extra run flags
# expected process return value, if not zero
test('arith001', normal, compile_and_run, [''])
test('arith002', normal, compile_and_run, [''])
# In fast mode, we do arith003 and arith011 only.
test('arith001', skip_if_fast, compile_and_run, [''])
test('arith002', skip_if_fast, compile_and_run, [''])
test('arith003', normal, compile_and_run, [''])
test('arith004', normal, compile_and_run, [''])
test('arith005', normal, compile_and_run, [''])
test('arith006', normal, compile_and_run, [''])
test('arith007', normal, compile_and_run, [''])
test('arith004', skip_if_fast, compile_and_run, [''])
test('arith005', skip_if_fast, compile_and_run, [''])
test('arith006', skip_if_fast, compile_and_run, [''])
test('arith007', skip_if_fast, compile_and_run, [''])
# This test generates slightly different results with the NCG on x86,
# because of the extra precision when floating point computations are
......@@ -19,10 +21,10 @@ if config.platform.startswith('i386-'):
else:
ways = normal
test('arith008', ways, compile_and_run, [''])
test('arith008', compose(skip_if_fast,ways), compile_and_run, [''])
test('arith009', normal, compile_and_run, [''])
test('arith010', normal, compile_and_run, [''])
test('arith009', skip_if_fast, compile_and_run, [''])
test('arith010', skip_if_fast, compile_and_run, [''])
test('arith011', normal, compile_and_run, [''])
if config.platform.startswith('i386-'):
......@@ -30,13 +32,13 @@ if config.platform.startswith('i386-'):
else:
ways = normal
test('arith012', ways, compile_and_run, [''])