test.mk 11.8 KB
Newer Older
1 2 3
# -----------------------------------------------------------------------------
# Examples of use:
#
dterei's avatar
dterei committed
4 5 6
#  make           -- run all the tests in the current directory
#  make verbose   -- as make test, but up the verbosity
#  make accept    -- run the tests, accepting the current output
7 8 9
#
# The following variables may be set on the make command line:
#
dterei's avatar
dterei committed
10 11 12 13 14 15 16
#  TEST      -- specific test to run
#  TESTS     -- specific tests to run (same as $TEST really)
#  EXTRA_HC_OPTS      -- extra flags to send to the Haskell compiler
#  EXTRA_RUNTEST_OPTS -- extra flags to give the test driver
#  CONFIG    -- use a different configuration file
#  COMPILER  -- select a configuration file from config/
#  THREADS   -- run n tests at once
17 18 19
#
# -----------------------------------------------------------------------------

20
# export the value of $MAKE for invocation in tests/driver/
dons's avatar
dons committed
21 22
export MAKE

23
RUNTESTS     = $(TOP)/driver/runtests.py
ross's avatar
ross committed
24
COMPILER     = ghc
25 26
CONFIGDIR    = $(TOP)/config
CONFIG       = $(CONFIGDIR)/$(COMPILER)
27

28 29 30 31 32 33 34
ifeq "$(GhcUnregisterised)" "YES"
    # Otherwise C backend generates many warnings about
    # imcompatible proto casts for GCC's buitins:
    #    memcpy, printf, strlen.
    EXTRA_HC_OPTS += -optc-fno-builtin
endif

35
# TEST_HC_OPTS is passed to every invocation of TEST_HC
36
# in nested Makefiles
37
TEST_HC_OPTS = -dcore-lint -dcmm-lint -no-user-$(GhcPackageDbFlag) -rtsopts $(EXTRA_HC_OPTS)
38

39
ifeq "$(MinGhcVersion711)" "YES"
40 41 42
# Don't warn about missing specialisations. They can only occur with `-O`, but
# we want tests to produce the same output for all test ways.
TEST_HC_OPTS += -fno-warn-missed-specialisations
43
TEST_HC_OPTS += -fshow-warning-groups
44
endif
45

46 47 48 49 50 51
ifeq "$(MinGhcVersion801)" "YES"
# Turn off any VT800 codes in the output or they wreak havoc on the
# testsuite output.
TEST_HC_OPTS += -fdiagnostics-color=never
endif

52 53 54 55 56 57 58
# Add the no-debug-output last as it is often convenient to copy the test invocation
# removing this line.
TEST_HC_OPTS += -dno-debug-output

TEST_HC_OPTS_INTERACTIVE = $(TEST_HC_OPTS) --interactive -v0 -ignore-dot-ghci -fno-ghci-history


59 60 61 62
RUNTEST_OPTS =

ifeq "$(filter $(TargetOS_CPP), cygwin32 mingw32)" ""
exeext =
63
else
64
exeext = .exe
65
endif
66

67 68 69 70 71 72 73 74
ifneq "$(filter $(TargetOS_CPP),cygwin32 mingw32)" ""
dllext = .dll
else ifeq "$(TargetOS_CPP)" "darwin"
dllext = .dylib
else
dllext = .so
endif

75 76
RUNTEST_OPTS += -e ghc_compiler_always_flags="'$(TEST_HC_OPTS)'"

thomie's avatar
thomie committed
77
RUNTEST_OPTS += -e config.compiler_debugged=$(GhcDebugged)
78

79
ifeq "$(GhcWithNativeCodeGen)" "YES"
80 81 82
RUNTEST_OPTS += -e ghc_with_native_codegen=1
else
RUNTEST_OPTS += -e ghc_with_native_codegen=0
83 84
endif

85
GHC_PRIM_LIBDIR := $(subst library-dirs: ,,$(shell "$(GHC_PKG)" field ghc-prim library-dirs --simple-output))
86 87 88
HAVE_VANILLA := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.hi ]; then echo YES; else echo NO; fi)
HAVE_DYNAMIC := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.dyn_hi ]; then echo YES; else echo NO; fi)
HAVE_PROFILING := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.p_hi ]; then echo YES; else echo NO; fi)
89

90
ifeq "$(HAVE_VANILLA)" "YES"
thomie's avatar
thomie committed
91
RUNTEST_OPTS += -e config.have_vanilla=True
92
else
thomie's avatar
thomie committed
93
RUNTEST_OPTS += -e config.have_vanilla=False
94 95
endif

96
ifeq "$(HAVE_DYNAMIC)" "YES"
thomie's avatar
thomie committed
97
RUNTEST_OPTS += -e config.have_dynamic=True
98
else
thomie's avatar
thomie committed
99
RUNTEST_OPTS += -e config.have_dynamic=False
100 101
endif

102
ifeq "$(HAVE_PROFILING)" "YES"
thomie's avatar
thomie committed
103
RUNTEST_OPTS += -e config.have_profiling=True
104
else
thomie's avatar
thomie committed
105
RUNTEST_OPTS += -e config.have_profiling=False
106
endif
107

108 109 110 111 112 113
ifeq "$(filter thr, $(GhcRTSWays))" "thr"
RUNTEST_OPTS += -e ghc_with_threaded_rts=1
else
RUNTEST_OPTS += -e ghc_with_threaded_rts=0
endif

114 115 116 117 118 119
ifeq "$(filter dyn, $(GhcRTSWays))" "dyn"
RUNTEST_OPTS += -e ghc_with_dynamic_rts=1
else
RUNTEST_OPTS += -e ghc_with_dynamic_rts=0
endif

120
ifeq "$(GhcWithInterpreter)" "NO"
thomie's avatar
thomie committed
121
RUNTEST_OPTS += -e config.have_interp=False
122
else ifeq "$(GhcStage)" "1"
thomie's avatar
thomie committed
123
RUNTEST_OPTS += -e config.have_interp=False
124
else
thomie's avatar
thomie committed
125
RUNTEST_OPTS += -e config.have_interp=True
126 127
endif

128
ifeq "$(GhcUnregisterised)" "YES"
thomie's avatar
thomie committed
129
RUNTEST_OPTS += -e config.unregisterised=True
130
else
thomie's avatar
thomie committed
131
RUNTEST_OPTS += -e config.unregisterised=False
132 133
endif

134
ifeq "$(GhcDynamicByDefault)" "YES"
thomie's avatar
thomie committed
135
RUNTEST_OPTS += -e config.ghc_dynamic_by_default=True
136
CABAL_MINIMAL_BUILD = --enable-shared --disable-library-vanilla
137
else
thomie's avatar
thomie committed
138
RUNTEST_OPTS += -e config.ghc_dynamic_by_default=False
139
CABAL_MINIMAL_BUILD = --enable-library-vanilla --disable-shared
140 141
endif

ian@well-typed.com's avatar
ian@well-typed.com committed
142
ifeq "$(GhcDynamic)" "YES"
thomie's avatar
thomie committed
143
RUNTEST_OPTS += -e config.ghc_dynamic=True
ian@well-typed.com's avatar
ian@well-typed.com committed
144 145
CABAL_PLUGIN_BUILD = --enable-shared --disable-library-vanilla
else
thomie's avatar
thomie committed
146
RUNTEST_OPTS += -e config.ghc_dynamic=False
ian@well-typed.com's avatar
ian@well-typed.com committed
147 148 149
CABAL_PLUGIN_BUILD = --enable-library-vanilla --disable-shared
endif

150 151 152 153 154 155
ifeq "$(GhcWithSMP)" "YES"
RUNTEST_OPTS += -e ghc_with_smp=1
else
RUNTEST_OPTS += -e ghc_with_smp=0
endif

156 157
ifeq "$(LLC)" ""
RUNTEST_OPTS += -e ghc_with_llvm=0
158 159
else ifeq "$(TargetARCH_CPP)" "powerpc"
RUNTEST_OPTS += -e ghc_with_llvm=0
160 161 162
else ifneq "$(LLC)" "llc"
# If we have a real detected value for LLVM, then it really ought to work
RUNTEST_OPTS += -e ghc_with_llvm=1
163 164 165 166
else
RUNTEST_OPTS += -e ghc_with_llvm=0
endif

Ian Lynagh's avatar
Ian Lynagh committed
167 168 169 170 171 172
ifeq "$(WINDOWS)" "YES"
RUNTEST_OPTS += -e windows=True
else
RUNTEST_OPTS += -e windows=False
endif

Ian Lynagh's avatar
Ian Lynagh committed
173 174 175 176 177 178
ifeq "$(DARWIN)" "YES"
RUNTEST_OPTS += -e darwin=True
else
RUNTEST_OPTS += -e darwin=False
endif

179
ifeq "$(IN_TREE_COMPILER)" "YES"
thomie's avatar
thomie committed
180
RUNTEST_OPTS += -e config.in_tree_compiler=True
181
else
thomie's avatar
thomie committed
182
RUNTEST_OPTS += -e config.in_tree_compiler=False
183 184
endif

ei@vuokko.info's avatar
ei@vuokko.info committed
185
ifneq "$(THREADS)" ""
186
RUNTEST_OPTS += --threads=$(THREADS)
ei@vuokko.info's avatar
ei@vuokko.info committed
187 188
endif

189 190 191 192
ifneq "$(VERBOSE)" ""
RUNTEST_OPTS += --verbose=$(VERBOSE)
endif

193 194 195 196
ifeq "$(SKIP_PERF_TESTS)" "YES"
RUNTEST_OPTS += --skip-perf-tests
endif

197
ifeq "$(CLEANUP)" "0"
198
RUNTEST_OPTS += -e config.cleanup=False
199
else ifeq "$(CLEANUP)" "NO"
200
RUNTEST_OPTS += -e config.cleanup=False
201
else
202 203 204 205 206 207 208 209 210 211
RUNTEST_OPTS += -e config.cleanup=True
endif

ifeq "$(LOCAL)" "0"
# See Note [Running tests in /tmp].
RUNTEST_OPTS += -e config.local=False
else ifeq "$(LOCAL)" "NO"
RUNTEST_OPTS += -e config.local=False
else
RUNTEST_OPTS += -e config.local=True
212 213
endif

214
RUNTEST_OPTS +=  \
215
	--rootdir=. \
216
	--configfile=$(CONFIG) \
Ian Lynagh's avatar
Ian Lynagh committed
217 218 219
	-e 'config.confdir="$(CONFIGDIR)"' \
	-e 'config.platform="$(TARGETPLATFORM)"' \
	-e 'config.os="$(TargetOS_CPP)"' \
220
	-e 'config.arch="$(TargetARCH_CPP)"' \
Ian Lynagh's avatar
Ian Lynagh committed
221
	-e 'config.wordsize="$(WORDSIZE)"' \
Ian Lynagh's avatar
Ian Lynagh committed
222
	-e 'config.timeout=int($(TIMEOUT)) or config.timeout' \
Ian Lynagh's avatar
Ian Lynagh committed
223
	-e 'config.exeext="$(exeext)"' \
224 225
	-e 'config.top="$(TOP_ABS)"'

226 227 228 229 230 231 232 233
# Wrap non-empty program paths in quotes, because they may contain spaces. Do
# it here, so we don't have to (and don't forget to do it) in the .T test
# scripts (search for '{compiler}' or '{hpc}'). This may or may not be a good
# idea.
# Use `--config` instead of `-e`, because `-e` (which calls Python's `eval`
# function) would require another pair of (escaped) quotes, which interfers
# with MinGW's magic path handling (see #10449, and
# http://www.mingw.org/wiki/Posix_path_conversion).
234 235
# We use double instead of single quotes, which may or may not be important
# when using msys2 (#9626, #10441).
236
quote_path = $(if $1,"$1")
237
RUNTEST_OPTS +=  \
238 239 240 241 242 243 244
	--config 'compiler=$(call quote_path,$(TEST_HC))' \
	--config 'ghc_pkg=$(call quote_path,$(GHC_PKG))' \
	--config 'haddock=$(call quote_path,$(HADDOCK))' \
	--config 'hp2ps=$(call quote_path,$(HP2PS_ABS))' \
	--config 'hpc=$(call quote_path,$(HPC))' \
	--config 'gs=$(call quote_path,$(GS))' \
	--config 'timeout_prog=$(call quote_path,$(TIMEOUT_PROGRAM))'
245

246 247
RUNTEST_OPTS += -e "config.stage=$(GhcStage)"

thomie's avatar
thomie committed
248
ifneq "$(SUMMARY_FILE)" ""
249
RUNTEST_OPTS +=  \
thomie's avatar
thomie committed
250 251 252 253 254
	--summary-file "$(SUMMARY_FILE)"
endif
ifeq "$(NO_PRINT_SUMMARY)" "YES"
RUNTEST_OPTS +=  \
	--no-print-summary 1
255 256 257
endif

RUNTEST_OPTS +=  \
258 259
	$(EXTRA_RUNTEST_OPTS)

ian@well-typed.com's avatar
ian@well-typed.com committed
260 261 262
ifeq "$(list_broken)" "YES"
set_list_broken = -e config.list_broken=True
else
263
set_list_broken =
ian@well-typed.com's avatar
ian@well-typed.com committed
264 265
endif

266 267 268 269 270 271
# See Note [validate and testsuite speed] in toplevel Makefile.
ifneq "$(SPEED)" ""
setspeed = -e config.speed="$(SPEED)"
else ifeq "$(fast)" "YES"
# Backward compatibility. Maybe some people are running 'make accept fast=YES'?
setspeed = -e config.speed=2
272
else
273
setspeed =
274 275 276 277 278
endif

ifeq "$(accept)" "YES"
setaccept = -e config.accept=1
else
279
setaccept =
280 281
endif

282
.PHONY: all boot test verbose accept fast slow list_broken
283 284 285

all: test

286
TIMEOUT_PROGRAM = $(TOP)/timeout/install-inplace/bin/timeout$(exeext)
287

288 289
boot: $(TIMEOUT_PROGRAM)

290
$(TIMEOUT_PROGRAM) :
291
	@echo "Looks like you don't have timeout, building it first..."
Ian Lynagh's avatar
Ian Lynagh committed
292
	$(MAKE) -C $(TOP)/timeout all
293

294 295 296
# Use a '+' to make sure that any sub-MAKEs that python spawns can
# communicate with the topmake.
# See Note [Communicating options and variables to a submake]
297
test: $(TIMEOUT_PROGRAM)
298
	+PYTHON="$(PYTHON)" "$(PYTHON)" $(RUNTESTS) $(RUNTEST_OPTS) \
299
		$(patsubst %, --only=%, $(TEST)) \
300
		$(patsubst %, --only=%, $(TESTS)) \
301
		$(patsubst %, --way=%, $(WAY)) \
302
		$(patsubst %, --skipway=%, $(SKIPWAY)) \
ian@well-typed.com's avatar
ian@well-typed.com committed
303
		$(set_list_broken) \
304
		$(setspeed) \
305
		$(setaccept)
306

307
verbose: test
308 309

accept:
310 311
	$(MAKE) accept=YES

312
fast fasttest:
313 314 315
	# See Note [validate and testsuite speed] in toplevel Makefile.
	$(MAKE) SPEED=2

316
slow slowtest:
317
	$(MAKE) SPEED=0
318

ian@well-typed.com's avatar
ian@well-typed.com committed
319 320 321
list_broken:
	$(MAKE) list_broken=YES

322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341
# Note [Communicating options and variables to a submake]
#
# Consider the following scenario:
#   * A test foo is defined as
#     test('foo', [], run_command, ['$MAKE footarget'])
#   * A user calls 'make -j24 TEST=foo'
#
# What happens is something like this:
#   * make (topmake) reads all options and variables given on the commandline
#     and adds them to the variable MAKEFLAGS [1]. This variable is exported by
#     default [1], so submakes can use them.
#   * The 'test' target calls 'python ..'
#   * Python calls 'make footarget' (submake)
#
# **First question**: what happens to the '-j24' option when calling make
# recursively?
#
# From
# https://www.gnu.org/software/make/manual/html_node/Variables_002fRecursion.html:
#
342 343 344 345
#     "The '-j' option is a special case (see Parallel Execution). If you set
#     it to some numeric value 'N' and your operating system supports it (most
#     any UNIX system will; others typically won't), the parent make and all the
#     sub-makes will communicate to ensure that there are only 'N' jobs running
346 347 348 349
#     at the same time between them all."
#
# In our scenario, the user will actually see the following warning [2]:
#
350
#     'warning: jobserver unavailable: using -j1. Add '+' to parent make rule.'
351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383
#
# The problem is that topmake and submake don't know about eachother, since
# python is in between. To let them communicate, we have to use the '+'
# option, by calling '+python' instead of 'python' [2]. This works,
# magically, and fixes #11569.
#
# **Second question**: can't we just unexport MAKEFLAGS, instead of using
# that '+' trick? The testsuite driver (python) mangages parallelism by
# itself already, so '-j24' doesn't do the right thing anyway. You have to
# use 'make test THREADS=24'. Unexporting MAKEFLAGS would mean ignoring
# any '-j' flags passed to make (either from the user calling 'make -j'
# explicitly or from having MAKEFLAGS=-j set in the shell, see #11569).
#
# This almost works, except when calling 'make fast/slow/accept TEST_HC=ghc'
# instead of just 'make test'. These targets call 'make test FAST=YES'
# recursively (and 'make test' calls python, as before).
#
# The problem is that in boilerplate.mk we try to override the variable
# TEST_HC (See Note [The TEST_HC variable]). Somewhere somehow this
# information (of us wanting to update TEST_HC) gets lost in the process,
# resulting in the final TEST_HC always getting set to the inplace compiler.
# It seems possible to remedy this yet again by exporting TEST_HC explicitly,
# but I didn't understand nor test it thoroughly (what about the other
# variables we override, see calls to canonicalise), and the '+' trick seems
# to work at least equally well (just don't run something like
# 'make test fast slow accept').
#
# Tests:
# * `make TEST=T3307 -j2` should not show a warning.
# * `make TEST=tc001 TEST_HC=ghc fast` should not use the inplace compiler.
#
# [1] https://www.gnu.org/software/make/manual/html_node/Variables_002fRecursion.html
# [2] https://www.gnu.org/software/make/manual/html_node/Error-Messages.html