test.mk 11.9 KB
Newer Older
1 2 3
# -----------------------------------------------------------------------------
# Examples of use:
#
dterei's avatar
dterei committed
4 5 6
#  make           -- run all the tests in the current directory
#  make verbose   -- as make test, but up the verbosity
#  make accept    -- run the tests, accepting the current output
7 8 9
#
# The following variables may be set on the make command line:
#
dterei's avatar
dterei committed
10 11 12 13 14 15 16
#  TEST      -- specific test to run
#  TESTS     -- specific tests to run (same as $TEST really)
#  EXTRA_HC_OPTS      -- extra flags to send to the Haskell compiler
#  EXTRA_RUNTEST_OPTS -- extra flags to give the test driver
#  CONFIG    -- use a different configuration file
#  COMPILER  -- select a configuration file from config/
#  THREADS   -- run n tests at once
17 18 19
#
# -----------------------------------------------------------------------------

20
# export the value of $MAKE for invocation in tests/driver/
dons's avatar
dons committed
21 22
export MAKE

23
RUNTESTS     = $(TOP)/driver/runtests.py
ross's avatar
ross committed
24
COMPILER     = ghc
25 26
CONFIGDIR    = $(TOP)/config
CONFIG       = $(CONFIGDIR)/$(COMPILER)
27

28 29 30 31 32 33 34
ifeq "$(GhcUnregisterised)" "YES"
    # Otherwise C backend generates many warnings about
    # imcompatible proto casts for GCC's buitins:
    #    memcpy, printf, strlen.
    EXTRA_HC_OPTS += -optc-fno-builtin
endif

35
# TEST_HC_OPTS is passed to every invocation of TEST_HC
36
# in nested Makefiles
37
TEST_HC_OPTS = -dcore-lint -dcmm-lint -no-user-$(GhcPackageDbFlag) -rtsopts $(EXTRA_HC_OPTS)
38

39
ifeq "$(MinGhcVersion711)" "YES"
40 41 42
# Don't warn about missing specialisations. They can only occur with `-O`, but
# we want tests to produce the same output for all test ways.
TEST_HC_OPTS += -fno-warn-missed-specialisations
43
TEST_HC_OPTS += -fshow-warning-groups
44
endif
45

46 47 48 49
ifeq "$(MinGhcVersion801)" "YES"
# Turn off any VT800 codes in the output or they wreak havoc on the
# testsuite output.
TEST_HC_OPTS += -fdiagnostics-color=never
Rufflewind's avatar
Rufflewind committed
50
TEST_HC_OPTS += -fno-diagnostics-show-caret
51 52
endif

53 54 55 56 57 58 59
# Add the no-debug-output last as it is often convenient to copy the test invocation
# removing this line.
TEST_HC_OPTS += -dno-debug-output

TEST_HC_OPTS_INTERACTIVE = $(TEST_HC_OPTS) --interactive -v0 -ignore-dot-ghci -fno-ghci-history


60 61 62 63
RUNTEST_OPTS =

ifeq "$(filter $(TargetOS_CPP), cygwin32 mingw32)" ""
exeext =
64
else
65
exeext = .exe
66
endif
67

68 69 70 71 72 73 74 75
ifneq "$(filter $(TargetOS_CPP),cygwin32 mingw32)" ""
dllext = .dll
else ifeq "$(TargetOS_CPP)" "darwin"
dllext = .dylib
else
dllext = .so
endif

76
RUNTEST_OPTS += -e "ghc_compiler_always_flags='$(TEST_HC_OPTS)'"
77

thomie's avatar
thomie committed
78
RUNTEST_OPTS += -e config.compiler_debugged=$(GhcDebugged)
79

80
ifeq "$(GhcWithNativeCodeGen)" "YES"
81 82 83
RUNTEST_OPTS += -e ghc_with_native_codegen=1
else
RUNTEST_OPTS += -e ghc_with_native_codegen=0
84 85
endif

86
GHC_PRIM_LIBDIR := $(subst library-dirs: ,,$(shell "$(GHC_PKG)" field ghc-prim library-dirs --simple-output))
87 88 89
HAVE_VANILLA := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.hi ]; then echo YES; else echo NO; fi)
HAVE_DYNAMIC := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.dyn_hi ]; then echo YES; else echo NO; fi)
HAVE_PROFILING := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.p_hi ]; then echo YES; else echo NO; fi)
90

91
ifeq "$(HAVE_VANILLA)" "YES"
thomie's avatar
thomie committed
92
RUNTEST_OPTS += -e config.have_vanilla=True
93
else
thomie's avatar
thomie committed
94
RUNTEST_OPTS += -e config.have_vanilla=False
95 96
endif

97
ifeq "$(HAVE_DYNAMIC)" "YES"
thomie's avatar
thomie committed
98
RUNTEST_OPTS += -e config.have_dynamic=True
99
else
thomie's avatar
thomie committed
100
RUNTEST_OPTS += -e config.have_dynamic=False
101 102
endif

103
ifeq "$(HAVE_PROFILING)" "YES"
thomie's avatar
thomie committed
104
RUNTEST_OPTS += -e config.have_profiling=True
105
else
thomie's avatar
thomie committed
106
RUNTEST_OPTS += -e config.have_profiling=False
107
endif
108

109 110 111 112 113 114
ifeq "$(filter thr, $(GhcRTSWays))" "thr"
RUNTEST_OPTS += -e ghc_with_threaded_rts=1
else
RUNTEST_OPTS += -e ghc_with_threaded_rts=0
endif

115 116 117 118 119 120
ifeq "$(filter dyn, $(GhcRTSWays))" "dyn"
RUNTEST_OPTS += -e ghc_with_dynamic_rts=1
else
RUNTEST_OPTS += -e ghc_with_dynamic_rts=0
endif

121
ifeq "$(GhcWithInterpreter)" "NO"
thomie's avatar
thomie committed
122
RUNTEST_OPTS += -e config.have_interp=False
123
else ifeq "$(GhcStage)" "1"
thomie's avatar
thomie committed
124
RUNTEST_OPTS += -e config.have_interp=False
125
else
thomie's avatar
thomie committed
126
RUNTEST_OPTS += -e config.have_interp=True
127 128
endif

129
ifeq "$(GhcUnregisterised)" "YES"
thomie's avatar
thomie committed
130
RUNTEST_OPTS += -e config.unregisterised=True
131
else
thomie's avatar
thomie committed
132
RUNTEST_OPTS += -e config.unregisterised=False
133 134
endif

135
ifeq "$(GhcDynamicByDefault)" "YES"
thomie's avatar
thomie committed
136
RUNTEST_OPTS += -e config.ghc_dynamic_by_default=True
137
CABAL_MINIMAL_BUILD = --enable-shared --disable-library-vanilla
138
else
thomie's avatar
thomie committed
139
RUNTEST_OPTS += -e config.ghc_dynamic_by_default=False
140
CABAL_MINIMAL_BUILD = --enable-library-vanilla --disable-shared
141 142
endif

ian@well-typed.com's avatar
ian@well-typed.com committed
143
ifeq "$(GhcDynamic)" "YES"
thomie's avatar
thomie committed
144
RUNTEST_OPTS += -e config.ghc_dynamic=True
ian@well-typed.com's avatar
ian@well-typed.com committed
145 146
CABAL_PLUGIN_BUILD = --enable-shared --disable-library-vanilla
else
thomie's avatar
thomie committed
147
RUNTEST_OPTS += -e config.ghc_dynamic=False
ian@well-typed.com's avatar
ian@well-typed.com committed
148 149 150
CABAL_PLUGIN_BUILD = --enable-library-vanilla --disable-shared
endif

151 152 153 154 155 156
ifeq "$(GhcWithSMP)" "YES"
RUNTEST_OPTS += -e ghc_with_smp=1
else
RUNTEST_OPTS += -e ghc_with_smp=0
endif

157 158
ifeq "$(LLC)" ""
RUNTEST_OPTS += -e ghc_with_llvm=0
159 160
else ifeq "$(TargetARCH_CPP)" "powerpc"
RUNTEST_OPTS += -e ghc_with_llvm=0
161 162 163
else ifneq "$(LLC)" "llc"
# If we have a real detected value for LLVM, then it really ought to work
RUNTEST_OPTS += -e ghc_with_llvm=1
164 165 166 167
else
RUNTEST_OPTS += -e ghc_with_llvm=0
endif

Ian Lynagh's avatar
Ian Lynagh committed
168 169 170 171 172 173
ifeq "$(WINDOWS)" "YES"
RUNTEST_OPTS += -e windows=True
else
RUNTEST_OPTS += -e windows=False
endif

Ian Lynagh's avatar
Ian Lynagh committed
174 175 176 177 178 179
ifeq "$(DARWIN)" "YES"
RUNTEST_OPTS += -e darwin=True
else
RUNTEST_OPTS += -e darwin=False
endif

180
ifeq "$(IN_TREE_COMPILER)" "YES"
thomie's avatar
thomie committed
181
RUNTEST_OPTS += -e config.in_tree_compiler=True
182
else
thomie's avatar
thomie committed
183
RUNTEST_OPTS += -e config.in_tree_compiler=False
184 185
endif

ei@vuokko.info's avatar
ei@vuokko.info committed
186
ifneq "$(THREADS)" ""
187
RUNTEST_OPTS += --threads=$(THREADS)
ei@vuokko.info's avatar
ei@vuokko.info committed
188 189
endif

190 191 192 193
ifneq "$(VERBOSE)" ""
RUNTEST_OPTS += --verbose=$(VERBOSE)
endif

194 195 196 197
ifeq "$(SKIP_PERF_TESTS)" "YES"
RUNTEST_OPTS += --skip-perf-tests
endif

198
ifeq "$(CLEANUP)" "0"
199
RUNTEST_OPTS += -e config.cleanup=False
200
else ifeq "$(CLEANUP)" "NO"
201
RUNTEST_OPTS += -e config.cleanup=False
202
else
203 204 205 206 207 208 209 210 211 212
RUNTEST_OPTS += -e config.cleanup=True
endif

ifeq "$(LOCAL)" "0"
# See Note [Running tests in /tmp].
RUNTEST_OPTS += -e config.local=False
else ifeq "$(LOCAL)" "NO"
RUNTEST_OPTS += -e config.local=False
else
RUNTEST_OPTS += -e config.local=True
213 214
endif

215
RUNTEST_OPTS +=  \
216
	--rootdir=. \
217
	--config-file=$(CONFIG) \
Ian Lynagh's avatar
Ian Lynagh committed
218 219 220
	-e 'config.confdir="$(CONFIGDIR)"' \
	-e 'config.platform="$(TARGETPLATFORM)"' \
	-e 'config.os="$(TargetOS_CPP)"' \
221
	-e 'config.arch="$(TargetARCH_CPP)"' \
Ian Lynagh's avatar
Ian Lynagh committed
222
	-e 'config.wordsize="$(WORDSIZE)"' \
Ian Lynagh's avatar
Ian Lynagh committed
223
	-e 'config.timeout=int($(TIMEOUT)) or config.timeout' \
Ian Lynagh's avatar
Ian Lynagh committed
224
	-e 'config.exeext="$(exeext)"' \
225 226
	-e 'config.top="$(TOP_ABS)"'

227 228 229 230 231 232 233 234
# Wrap non-empty program paths in quotes, because they may contain spaces. Do
# it here, so we don't have to (and don't forget to do it) in the .T test
# scripts (search for '{compiler}' or '{hpc}'). This may or may not be a good
# idea.
# Use `--config` instead of `-e`, because `-e` (which calls Python's `eval`
# function) would require another pair of (escaped) quotes, which interfers
# with MinGW's magic path handling (see #10449, and
# http://www.mingw.org/wiki/Posix_path_conversion).
235 236
# We use double instead of single quotes, which may or may not be important
# when using msys2 (#9626, #10441).
237
quote_path = $(if $1,"$1")
238
RUNTEST_OPTS +=  \
239 240 241 242 243 244 245
	--config 'compiler=$(call quote_path,$(TEST_HC))' \
	--config 'ghc_pkg=$(call quote_path,$(GHC_PKG))' \
	--config 'haddock=$(call quote_path,$(HADDOCK))' \
	--config 'hp2ps=$(call quote_path,$(HP2PS_ABS))' \
	--config 'hpc=$(call quote_path,$(HPC))' \
	--config 'gs=$(call quote_path,$(GS))' \
	--config 'timeout_prog=$(call quote_path,$(TIMEOUT_PROGRAM))'
246

247 248
RUNTEST_OPTS += -e "config.stage=$(GhcStage)"

thomie's avatar
thomie committed
249
ifneq "$(SUMMARY_FILE)" ""
250
RUNTEST_OPTS +=  \
thomie's avatar
thomie committed
251 252 253 254
	--summary-file "$(SUMMARY_FILE)"
endif
ifeq "$(NO_PRINT_SUMMARY)" "YES"
RUNTEST_OPTS +=  \
255
	--no-print-summary
256 257 258
endif

RUNTEST_OPTS +=  \
259 260
	$(EXTRA_RUNTEST_OPTS)

ian@well-typed.com's avatar
ian@well-typed.com committed
261 262 263
ifeq "$(list_broken)" "YES"
set_list_broken = -e config.list_broken=True
else
264
set_list_broken =
ian@well-typed.com's avatar
ian@well-typed.com committed
265 266
endif

267 268 269 270 271 272
# See Note [validate and testsuite speed] in toplevel Makefile.
ifneq "$(SPEED)" ""
setspeed = -e config.speed="$(SPEED)"
else ifeq "$(fast)" "YES"
# Backward compatibility. Maybe some people are running 'make accept fast=YES'?
setspeed = -e config.speed=2
273
else
274
setspeed =
275 276 277 278 279
endif

ifeq "$(accept)" "YES"
setaccept = -e config.accept=1
else
280
setaccept =
281 282
endif

283
.PHONY: all boot test verbose accept fast slow list_broken
284 285 286

all: test

287
TIMEOUT_PROGRAM = $(TOP)/timeout/install-inplace/bin/timeout$(exeext)
288

289 290
boot: $(TIMEOUT_PROGRAM)

291
$(TIMEOUT_PROGRAM) :
292
	@echo "Looks like you don't have timeout, building it first..."
Ian Lynagh's avatar
Ian Lynagh committed
293
	$(MAKE) -C $(TOP)/timeout all
294

295 296 297
# Use a '+' to make sure that any sub-MAKEs that python spawns can
# communicate with the topmake.
# See Note [Communicating options and variables to a submake]
298
test: $(TIMEOUT_PROGRAM)
299
	+PYTHON="$(PYTHON)" "$(PYTHON)" $(RUNTESTS) $(RUNTEST_OPTS) \
300
		$(patsubst %, --only=%, $(TEST)) \
301
		$(patsubst %, --only=%, $(TESTS)) \
302
		$(patsubst %, --way=%, $(WAY)) \
303
		$(patsubst %, --skipway=%, $(SKIPWAY)) \
ian@well-typed.com's avatar
ian@well-typed.com committed
304
		$(set_list_broken) \
305
		$(setspeed) \
306
		$(setaccept)
307

308
verbose: test
309 310

accept:
311 312
	$(MAKE) accept=YES

313
fast fasttest:
314 315 316
	# See Note [validate and testsuite speed] in toplevel Makefile.
	$(MAKE) SPEED=2

317
slow slowtest:
318
	$(MAKE) SPEED=0
319

ian@well-typed.com's avatar
ian@well-typed.com committed
320 321 322
list_broken:
	$(MAKE) list_broken=YES

323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342
# Note [Communicating options and variables to a submake]
#
# Consider the following scenario:
#   * A test foo is defined as
#     test('foo', [], run_command, ['$MAKE footarget'])
#   * A user calls 'make -j24 TEST=foo'
#
# What happens is something like this:
#   * make (topmake) reads all options and variables given on the commandline
#     and adds them to the variable MAKEFLAGS [1]. This variable is exported by
#     default [1], so submakes can use them.
#   * The 'test' target calls 'python ..'
#   * Python calls 'make footarget' (submake)
#
# **First question**: what happens to the '-j24' option when calling make
# recursively?
#
# From
# https://www.gnu.org/software/make/manual/html_node/Variables_002fRecursion.html:
#
343 344 345 346
#     "The '-j' option is a special case (see Parallel Execution). If you set
#     it to some numeric value 'N' and your operating system supports it (most
#     any UNIX system will; others typically won't), the parent make and all the
#     sub-makes will communicate to ensure that there are only 'N' jobs running
347 348 349 350
#     at the same time between them all."
#
# In our scenario, the user will actually see the following warning [2]:
#
351
#     'warning: jobserver unavailable: using -j1. Add '+' to parent make rule.'
352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384
#
# The problem is that topmake and submake don't know about eachother, since
# python is in between. To let them communicate, we have to use the '+'
# option, by calling '+python' instead of 'python' [2]. This works,
# magically, and fixes #11569.
#
# **Second question**: can't we just unexport MAKEFLAGS, instead of using
# that '+' trick? The testsuite driver (python) mangages parallelism by
# itself already, so '-j24' doesn't do the right thing anyway. You have to
# use 'make test THREADS=24'. Unexporting MAKEFLAGS would mean ignoring
# any '-j' flags passed to make (either from the user calling 'make -j'
# explicitly or from having MAKEFLAGS=-j set in the shell, see #11569).
#
# This almost works, except when calling 'make fast/slow/accept TEST_HC=ghc'
# instead of just 'make test'. These targets call 'make test FAST=YES'
# recursively (and 'make test' calls python, as before).
#
# The problem is that in boilerplate.mk we try to override the variable
# TEST_HC (See Note [The TEST_HC variable]). Somewhere somehow this
# information (of us wanting to update TEST_HC) gets lost in the process,
# resulting in the final TEST_HC always getting set to the inplace compiler.
# It seems possible to remedy this yet again by exporting TEST_HC explicitly,
# but I didn't understand nor test it thoroughly (what about the other
# variables we override, see calls to canonicalise), and the '+' trick seems
# to work at least equally well (just don't run something like
# 'make test fast slow accept').
#
# Tests:
# * `make TEST=T3307 -j2` should not show a warning.
# * `make TEST=tc001 TEST_HC=ghc fast` should not use the inplace compiler.
#
# [1] https://www.gnu.org/software/make/manual/html_node/Variables_002fRecursion.html
# [2] https://www.gnu.org/software/make/manual/html_node/Error-Messages.html