Commit c43c8e2c authored by thomie's avatar thomie

Testsuite: by default run all tests for a single way

`make test` now runs all tests for a single way only. Use `make slowtest` to
get the previous behaviour (i.e. run all tests for all ways).

The intention is to use this new `make test` setting for Phabricator, as
a reasonable compromise between `make fasttest` (what it previously
used) and a fullblown `make slowtest` (which runs all tests for all
ways).

See Note [validate and testsuite speed] in toplevel Makefile.

Differential Revision: https://phabricator.haskell.org/D1178
parent 32a9eada
......@@ -53,4 +53,6 @@ script:
# * But use VERBOSE=2 (the default, but not when using --quiet) otherwise
# the testsuite might not print output for over 10 minutes (more likely so
# when DEBUG_STAGE2=NO), causing Travis to again kill our job.
# * Use --fast, to stay within the time limits set by Travis.
# See Note [validate and testsuite speed] in toplevel Makefile.
- THREADS=3 SKIP_PERF_TESTS=YES VERBOSE=2 ./validate --fast --quiet
......@@ -83,6 +83,7 @@ REALGOALS=$(filter-out \
help \
test \
fulltest \
slowtest \
fasttest \
,$(MAKECMDGOALS))
......@@ -159,10 +160,36 @@ endif
endif
# Note [validate and testsuite speed]
#
# There are 3 different validate and testsuite speed settings:
# fast, normal and slow.
#
# how how used
# cd tests config. many many by
# validate && make speed= tests ways whom
# =============================================================================
# --fast fast 2 some 1 Travis (to stay within time limit)
# --normal test 1 all 1 Phabricator (slow takes too long?)
# --slow slow 0 all all Nightly (slow is ok)
#
# accept 1 all 1
#
# `make accept` should run all tests exactly once. There is no point in
# accepting a test for multiple ways, since it should produce the same output
# for all ways.
#
# To make sure all .stderr and .stdout files in the testsuite are never
# out-of-date, it is useful if Phabricator, via a normal `./validate` and `make
# test`, runs each test at least once.
.PHONY: fasttest
fasttest:
$(MAKE) -C testsuite/tests CLEANUP=1 SUMMARY_FILE=../../testsuite_summary.txt fast
.PHONY: fulltest test
fulltest test:
.PHONY: test
test:
$(MAKE) -C testsuite/tests CLEANUP=1 SUMMARY_FILE=../../testsuite_summary.txt
.PHONY: slowtest fulltest
slowtest fulltest:
$(MAKE) -C testsuite/tests CLEANUP=1 SUMMARY_FILE=../../testsuite_summary.txt slow
......@@ -11,7 +11,7 @@ ifneq "$(MAKECMDGOALS)" "maintainer-clean"
include $(TOP)/mk/boilerplate.mk
.PHONY: all boot test verbose accept fast list_broken
.PHONY: all boot test verbose accept fast slow list_broken
boot:
$(MAKE) -C $(TOP)/timeout all
......@@ -31,6 +31,9 @@ accept:
fast:
$(MAKE) -C $(TOP)/tests fast
slow:
$(MAKE) -C $(TOP)/tests slow
list_broken:
$(MAKE) -C $(TOP)/tests list_broken
......
......@@ -54,8 +54,8 @@ class TestConfig:
# Verbosity level
self.verbose = 3
# run the "fast" version of the test suite
self.fast = 0
# See Note [validate and testsuite speed] in toplevel Makefile.
self.speed = 1
self.list_broken = False
......
......@@ -333,7 +333,7 @@ def ghci_dynamic( ):
return config.ghc_dynamic
def fast():
return config.fast
return config.speed == 2
def platform( plat ):
return config.platform == plat
......@@ -674,9 +674,10 @@ def test_common_work (name, opts, func, args):
# Which ways we are asked to skip
do_ways = list(filter (ok_way,all_ways))
# In fast mode, we skip all but one way
if config.fast and len(do_ways) > 0:
do_ways = [do_ways[0]]
# Only run all ways in slow mode.
# See Note [validate and testsuite speed] in toplevel Makefile.
if config.speed > 0:
do_ways = do_ways[:1]
if not config.clean_only:
# Run the required tests...
......
......@@ -247,10 +247,14 @@ else
set_list_broken =
endif
ifeq "$(fast)" "YES"
setfast = -e config.fast=1
# See Note [validate and testsuite speed] in toplevel Makefile.
ifneq "$(SPEED)" ""
setspeed = -e config.speed="$(SPEED)"
else ifeq "$(fast)" "YES"
# Backward compatibility. Maybe some people are running 'make accept fast=YES'?
setspeed = -e config.speed=2
else
setfast =
setspeed =
endif
ifeq "$(accept)" "YES"
......@@ -259,7 +263,7 @@ else
setaccept =
endif
.PHONY: all boot test verbose accept fast list_broken
.PHONY: all boot test verbose accept fast slow list_broken
all: test
......@@ -278,7 +282,7 @@ test: $(TIMEOUT_PROGRAM)
$(patsubst %, --way=%, $(WAY)) \
$(patsubst %, --skipway=%, $(SKIPWAY)) \
$(set_list_broken) \
$(setfast) \
$(setspeed) \
$(setaccept)
verbose: test
......@@ -287,7 +291,11 @@ accept:
$(MAKE) accept=YES
fast:
$(MAKE) fast=YES
# See Note [validate and testsuite speed] in toplevel Makefile.
$(MAKE) SPEED=2
slow:
$(MAKE) SPEED=0
list_broken:
$(MAKE) list_broken=YES
......
......@@ -111,11 +111,7 @@ test('performGC', [ only_ways(['threaded1','threaded2'])
# -----------------------------------------------------------------------------
# These tests we only do for a full run
def f( name, opts ):
if config.fast:
opts.skip = 1
setTestOpts(f)
setTestOpts(when(fast(), skip))
test('conc001', normal, compile_and_run, [''])
test('conc002', normal, compile_and_run, [''])
......
......@@ -52,7 +52,7 @@ test('arrow', normal, compile, [''])
test('tdpe', normal, compile, [''])
test('Nilsson', when(fast(), skip), compile, [''])
if config.fast:
if fast():
test('records', normal, compile, [''])
else:
test('records', normal, compile_and_run, [''])
......
......@@ -18,10 +18,7 @@ test('TcCoercible', when(compiler_lt('ghc', '7.7'), skip), compile_and_run, ['']
# -----------------------------------------------------------------------------
# Skip everything else if fast is on
def f(name, opts):
if config.fast:
opts.skip = 1
setTestOpts(f)
setTestOpts(when(fast(), skip))
test('tcrun006', normal, compile_and_run, [''])
test('tcrun008', normal, compile_and_run, [''])
......
......@@ -18,9 +18,9 @@ Flags:
compiler the test suite covers.
2008-07-01: 63% slower than the default.
HTML generated here: testsuite/hpc_output/hpc_index.html
--fast Omit dyn way, omit binary distribution
--slow Build stage2 with -DDEBUG. Skips tests that call
compiler_stats_num_field.
--fast Omit binary distribution. Omit certain tests.
--slow Build stage2 with -DDEBUG. Run tests for all WAYS,
but skip those that call compiler_stats_num_field.
2008-07-01: 14% slower than the default.
--dph Also build libraries/dph and run associated tests.
--quiet More pretty build log.
......@@ -248,11 +248,11 @@ fi
case "$speed" in
SLOW)
MAKE_TEST_TARGET=fulltest
MAKE_TEST_TARGET=slowtest
BINDIST="BINDIST=YES"
;;
NORMAL)
MAKE_TEST_TARGET=fasttest
MAKE_TEST_TARGET=test
BINDIST="BINDIST=YES"
;;
FAST)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment