test.mk 11.8 KB
Newer Older
1
2
3
# -----------------------------------------------------------------------------
# Examples of use:
#
dterei's avatar
dterei committed
4
5
6
#  make           -- run all the tests in the current directory
#  make verbose   -- as make test, but up the verbosity
#  make accept    -- run the tests, accepting the current output
7
8
9
#
# The following variables may be set on the make command line:
#
dterei's avatar
dterei committed
10
11
12
13
14
15
16
#  TEST      -- specific test to run
#  TESTS     -- specific tests to run (same as $TEST really)
#  EXTRA_HC_OPTS      -- extra flags to send to the Haskell compiler
#  EXTRA_RUNTEST_OPTS -- extra flags to give the test driver
#  CONFIG    -- use a different configuration file
#  COMPILER  -- select a configuration file from config/
#  THREADS   -- run n tests at once
17
18
19
#
# -----------------------------------------------------------------------------

20
# export the value of $MAKE for invocation in tests/driver/
dons's avatar
dons committed
21
22
export MAKE

23
RUNTESTS     = $(TOP)/driver/runtests.py
ross's avatar
ross committed
24
COMPILER     = ghc
25
26
CONFIGDIR    = $(TOP)/config
CONFIG       = $(CONFIGDIR)/$(COMPILER)
27

28
29
30
31
32
33
34
ifeq "$(GhcUnregisterised)" "YES"
    # Otherwise C backend generates many warnings about
    # imcompatible proto casts for GCC's buitins:
    #    memcpy, printf, strlen.
    EXTRA_HC_OPTS += -optc-fno-builtin
endif

35
# TEST_HC_OPTS is passed to every invocation of TEST_HC
36
# in nested Makefiles
37
TEST_HC_OPTS = -dcore-lint -dcmm-lint -no-user-$(GhcPackageDbFlag) -rtsopts $(EXTRA_HC_OPTS)
38

39
ifeq "$(MinGhcVersion711)" "YES"
40
41
42
# Don't warn about missing specialisations. They can only occur with `-O`, but
# we want tests to produce the same output for all test ways.
TEST_HC_OPTS += -fno-warn-missed-specialisations
43
TEST_HC_OPTS += -fshow-warning-groups
44
endif
45

46
47
48
49
50
51
ifeq "$(MinGhcVersion801)" "YES"
# Turn off any VT800 codes in the output or they wreak havoc on the
# testsuite output.
TEST_HC_OPTS += -fdiagnostics-color=never
endif

52
53
54
55
56
57
58
# Add the no-debug-output last as it is often convenient to copy the test invocation
# removing this line.
TEST_HC_OPTS += -dno-debug-output

TEST_HC_OPTS_INTERACTIVE = $(TEST_HC_OPTS) --interactive -v0 -ignore-dot-ghci -fno-ghci-history


59
60
61
62
RUNTEST_OPTS =

ifeq "$(filter $(TargetOS_CPP), cygwin32 mingw32)" ""
exeext =
63
else
64
exeext = .exe
65
endif
66

67
68
69
70
71
72
73
74
ifneq "$(filter $(TargetOS_CPP),cygwin32 mingw32)" ""
dllext = .dll
else ifeq "$(TargetOS_CPP)" "darwin"
dllext = .dylib
else
dllext = .so
endif

75
76
RUNTEST_OPTS += -e ghc_compiler_always_flags="'$(TEST_HC_OPTS)'"

thomie's avatar
thomie committed
77
RUNTEST_OPTS += -e config.compiler_debugged=$(GhcDebugged)
78

79
ifeq "$(GhcWithNativeCodeGen)" "YES"
80
81
82
RUNTEST_OPTS += -e ghc_with_native_codegen=1
else
RUNTEST_OPTS += -e ghc_with_native_codegen=0
83
84
endif

85
GHC_PRIM_LIBDIR := $(subst library-dirs: ,,$(shell "$(GHC_PKG)" field ghc-prim library-dirs --simple-output))
86
87
88
HAVE_VANILLA := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.hi ]; then echo YES; else echo NO; fi)
HAVE_DYNAMIC := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.dyn_hi ]; then echo YES; else echo NO; fi)
HAVE_PROFILING := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.p_hi ]; then echo YES; else echo NO; fi)
89

90
ifeq "$(HAVE_VANILLA)" "YES"
thomie's avatar
thomie committed
91
RUNTEST_OPTS += -e config.have_vanilla=True
92
else
thomie's avatar
thomie committed
93
RUNTEST_OPTS += -e config.have_vanilla=False
94
95
endif

96
ifeq "$(HAVE_DYNAMIC)" "YES"
thomie's avatar
thomie committed
97
RUNTEST_OPTS += -e config.have_dynamic=True
98
else
thomie's avatar
thomie committed
99
RUNTEST_OPTS += -e config.have_dynamic=False
100
101
endif

102
ifeq "$(HAVE_PROFILING)" "YES"
thomie's avatar
thomie committed
103
RUNTEST_OPTS += -e config.have_profiling=True
104
else
thomie's avatar
thomie committed
105
RUNTEST_OPTS += -e config.have_profiling=False
106
endif
107

108
109
110
111
112
113
ifeq "$(filter thr, $(GhcRTSWays))" "thr"
RUNTEST_OPTS += -e ghc_with_threaded_rts=1
else
RUNTEST_OPTS += -e ghc_with_threaded_rts=0
endif

114
115
116
117
118
119
ifeq "$(filter dyn, $(GhcRTSWays))" "dyn"
RUNTEST_OPTS += -e ghc_with_dynamic_rts=1
else
RUNTEST_OPTS += -e ghc_with_dynamic_rts=0
endif

120
ifeq "$(GhcWithInterpreter)" "NO"
thomie's avatar
thomie committed
121
RUNTEST_OPTS += -e config.have_interp=False
122
else ifeq "$(GhcStage)" "1"
thomie's avatar
thomie committed
123
RUNTEST_OPTS += -e config.have_interp=False
124
else
thomie's avatar
thomie committed
125
RUNTEST_OPTS += -e config.have_interp=True
126
127
endif

128
ifeq "$(GhcUnregisterised)" "YES"
thomie's avatar
thomie committed
129
RUNTEST_OPTS += -e config.unregisterised=True
130
else
thomie's avatar
thomie committed
131
RUNTEST_OPTS += -e config.unregisterised=False
132
133
endif

134
ifeq "$(GhcDynamicByDefault)" "YES"
thomie's avatar
thomie committed
135
RUNTEST_OPTS += -e config.ghc_dynamic_by_default=True
136
CABAL_MINIMAL_BUILD = --enable-shared --disable-library-vanilla
137
else
thomie's avatar
thomie committed
138
RUNTEST_OPTS += -e config.ghc_dynamic_by_default=False
139
CABAL_MINIMAL_BUILD = --enable-library-vanilla --disable-shared
140
141
endif

ian@well-typed.com's avatar
ian@well-typed.com committed
142
ifeq "$(GhcDynamic)" "YES"
thomie's avatar
thomie committed
143
RUNTEST_OPTS += -e config.ghc_dynamic=True
ian@well-typed.com's avatar
ian@well-typed.com committed
144
145
CABAL_PLUGIN_BUILD = --enable-shared --disable-library-vanilla
else
thomie's avatar
thomie committed
146
RUNTEST_OPTS += -e config.ghc_dynamic=False
ian@well-typed.com's avatar
ian@well-typed.com committed
147
148
149
CABAL_PLUGIN_BUILD = --enable-library-vanilla --disable-shared
endif

150
151
152
153
154
155
ifeq "$(GhcWithSMP)" "YES"
RUNTEST_OPTS += -e ghc_with_smp=1
else
RUNTEST_OPTS += -e ghc_with_smp=0
endif

156
157
ifeq "$(LLC)" ""
RUNTEST_OPTS += -e ghc_with_llvm=0
158
159
else ifeq "$(TargetARCH_CPP)" "powerpc"
RUNTEST_OPTS += -e ghc_with_llvm=0
160
161
162
else ifneq "$(LLC)" "llc"
# If we have a real detected value for LLVM, then it really ought to work
RUNTEST_OPTS += -e ghc_with_llvm=1
163
164
165
166
else
RUNTEST_OPTS += -e ghc_with_llvm=0
endif

Ian Lynagh's avatar
Ian Lynagh committed
167
168
169
170
171
172
ifeq "$(WINDOWS)" "YES"
RUNTEST_OPTS += -e windows=True
else
RUNTEST_OPTS += -e windows=False
endif

Ian Lynagh's avatar
Ian Lynagh committed
173
174
175
176
177
178
ifeq "$(DARWIN)" "YES"
RUNTEST_OPTS += -e darwin=True
else
RUNTEST_OPTS += -e darwin=False
endif

179
ifeq "$(IN_TREE_COMPILER)" "YES"
thomie's avatar
thomie committed
180
RUNTEST_OPTS += -e config.in_tree_compiler=True
181
else
thomie's avatar
thomie committed
182
RUNTEST_OPTS += -e config.in_tree_compiler=False
183
184
endif

ei@vuokko.info's avatar
ei@vuokko.info committed
185
ifneq "$(THREADS)" ""
186
RUNTEST_OPTS += --threads=$(THREADS)
ei@vuokko.info's avatar
ei@vuokko.info committed
187
188
endif

189
190
191
192
ifneq "$(VERBOSE)" ""
RUNTEST_OPTS += --verbose=$(VERBOSE)
endif

193
194
195
196
ifeq "$(SKIP_PERF_TESTS)" "YES"
RUNTEST_OPTS += --skip-perf-tests
endif

197
ifeq "$(CLEANUP)" "0"
198
RUNTEST_OPTS += -e config.cleanup=False
199
else ifeq "$(CLEANUP)" "NO"
200
RUNTEST_OPTS += -e config.cleanup=False
201
else
202
203
204
205
206
207
208
209
210
211
RUNTEST_OPTS += -e config.cleanup=True
endif

ifeq "$(LOCAL)" "0"
# See Note [Running tests in /tmp].
RUNTEST_OPTS += -e config.local=False
else ifeq "$(LOCAL)" "NO"
RUNTEST_OPTS += -e config.local=False
else
RUNTEST_OPTS += -e config.local=True
212
213
endif

214
RUNTEST_OPTS +=  \
215
	--rootdir=. \
216
	--configfile=$(CONFIG) \
Ian Lynagh's avatar
Ian Lynagh committed
217
218
219
	-e 'config.confdir="$(CONFIGDIR)"' \
	-e 'config.platform="$(TARGETPLATFORM)"' \
	-e 'config.os="$(TargetOS_CPP)"' \
220
	-e 'config.arch="$(TargetARCH_CPP)"' \
Ian Lynagh's avatar
Ian Lynagh committed
221
	-e 'config.wordsize="$(WORDSIZE)"' \
Ian Lynagh's avatar
Ian Lynagh committed
222
	-e 'config.timeout=int($(TIMEOUT)) or config.timeout' \
Ian Lynagh's avatar
Ian Lynagh committed
223
	-e 'config.exeext="$(exeext)"' \
224
225
	-e 'config.top="$(TOP_ABS)"'

226
227
228
229
230
231
232
233
# Wrap non-empty program paths in quotes, because they may contain spaces. Do
# it here, so we don't have to (and don't forget to do it) in the .T test
# scripts (search for '{compiler}' or '{hpc}'). This may or may not be a good
# idea.
# Use `--config` instead of `-e`, because `-e` (which calls Python's `eval`
# function) would require another pair of (escaped) quotes, which interfers
# with MinGW's magic path handling (see #10449, and
# http://www.mingw.org/wiki/Posix_path_conversion).
234
235
# We use double instead of single quotes, which may or may not be important
# when using msys2 (#9626, #10441).
236
quote_path = $(if $1,"$1")
237
RUNTEST_OPTS +=  \
238
239
240
241
242
243
244
	--config 'compiler=$(call quote_path,$(TEST_HC))' \
	--config 'ghc_pkg=$(call quote_path,$(GHC_PKG))' \
	--config 'haddock=$(call quote_path,$(HADDOCK))' \
	--config 'hp2ps=$(call quote_path,$(HP2PS_ABS))' \
	--config 'hpc=$(call quote_path,$(HPC))' \
	--config 'gs=$(call quote_path,$(GS))' \
	--config 'timeout_prog=$(call quote_path,$(TIMEOUT_PROGRAM))'
245

246
247
RUNTEST_OPTS += -e "config.stage=$(GhcStage)"

thomie's avatar
thomie committed
248
ifneq "$(SUMMARY_FILE)" ""
249
RUNTEST_OPTS +=  \
thomie's avatar
thomie committed
250
251
252
253
254
	--summary-file "$(SUMMARY_FILE)"
endif
ifeq "$(NO_PRINT_SUMMARY)" "YES"
RUNTEST_OPTS +=  \
	--no-print-summary 1
255
256
257
endif

RUNTEST_OPTS +=  \
258
259
	$(EXTRA_RUNTEST_OPTS)

ian@well-typed.com's avatar
ian@well-typed.com committed
260
261
262
ifeq "$(list_broken)" "YES"
set_list_broken = -e config.list_broken=True
else
263
set_list_broken =
ian@well-typed.com's avatar
ian@well-typed.com committed
264
265
endif

266
267
268
269
270
271
# See Note [validate and testsuite speed] in toplevel Makefile.
ifneq "$(SPEED)" ""
setspeed = -e config.speed="$(SPEED)"
else ifeq "$(fast)" "YES"
# Backward compatibility. Maybe some people are running 'make accept fast=YES'?
setspeed = -e config.speed=2
272
else
273
setspeed =
274
275
276
277
278
endif

ifeq "$(accept)" "YES"
setaccept = -e config.accept=1
else
279
setaccept =
280
281
endif

282
.PHONY: all boot test verbose accept fast slow list_broken
283
284
285

all: test

286
TIMEOUT_PROGRAM = $(TOP)/timeout/install-inplace/bin/timeout$(exeext)
287

288
289
boot: $(TIMEOUT_PROGRAM)

290
$(TIMEOUT_PROGRAM) :
291
	@echo "Looks like you don't have timeout, building it first..."
Ian Lynagh's avatar
Ian Lynagh committed
292
	$(MAKE) -C $(TOP)/timeout all
293

294
295
296
# Use a '+' to make sure that any sub-MAKEs that python spawns can
# communicate with the topmake.
# See Note [Communicating options and variables to a submake]
297
test: $(TIMEOUT_PROGRAM)
298
	+PYTHON="$(PYTHON)" "$(PYTHON)" $(RUNTESTS) $(RUNTEST_OPTS) \
299
		$(patsubst %, --only=%, $(TEST)) \
300
		$(patsubst %, --only=%, $(TESTS)) \
301
		$(patsubst %, --way=%, $(WAY)) \
302
		$(patsubst %, --skipway=%, $(SKIPWAY)) \
ian@well-typed.com's avatar
ian@well-typed.com committed
303
		$(set_list_broken) \
304
		$(setspeed) \
305
		$(setaccept)
306

307
verbose: test
308
309

accept:
310
311
	$(MAKE) accept=YES

312
fast fasttest:
313
314
315
	# See Note [validate and testsuite speed] in toplevel Makefile.
	$(MAKE) SPEED=2

316
slow slowtest:
317
	$(MAKE) SPEED=0
318

ian@well-typed.com's avatar
ian@well-typed.com committed
319
320
321
list_broken:
	$(MAKE) list_broken=YES

322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
# Note [Communicating options and variables to a submake]
#
# Consider the following scenario:
#   * A test foo is defined as
#     test('foo', [], run_command, ['$MAKE footarget'])
#   * A user calls 'make -j24 TEST=foo'
#
# What happens is something like this:
#   * make (topmake) reads all options and variables given on the commandline
#     and adds them to the variable MAKEFLAGS [1]. This variable is exported by
#     default [1], so submakes can use them.
#   * The 'test' target calls 'python ..'
#   * Python calls 'make footarget' (submake)
#
# **First question**: what happens to the '-j24' option when calling make
# recursively?
#
# From
# https://www.gnu.org/software/make/manual/html_node/Variables_002fRecursion.html:
#
342
343
344
345
#     "The '-j' option is a special case (see Parallel Execution). If you set
#     it to some numeric value 'N' and your operating system supports it (most
#     any UNIX system will; others typically won't), the parent make and all the
#     sub-makes will communicate to ensure that there are only 'N' jobs running
346
347
348
349
#     at the same time between them all."
#
# In our scenario, the user will actually see the following warning [2]:
#
350
#     'warning: jobserver unavailable: using -j1. Add '+' to parent make rule.'
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
#
# The problem is that topmake and submake don't know about eachother, since
# python is in between. To let them communicate, we have to use the '+'
# option, by calling '+python' instead of 'python' [2]. This works,
# magically, and fixes #11569.
#
# **Second question**: can't we just unexport MAKEFLAGS, instead of using
# that '+' trick? The testsuite driver (python) mangages parallelism by
# itself already, so '-j24' doesn't do the right thing anyway. You have to
# use 'make test THREADS=24'. Unexporting MAKEFLAGS would mean ignoring
# any '-j' flags passed to make (either from the user calling 'make -j'
# explicitly or from having MAKEFLAGS=-j set in the shell, see #11569).
#
# This almost works, except when calling 'make fast/slow/accept TEST_HC=ghc'
# instead of just 'make test'. These targets call 'make test FAST=YES'
# recursively (and 'make test' calls python, as before).
#
# The problem is that in boilerplate.mk we try to override the variable
# TEST_HC (See Note [The TEST_HC variable]). Somewhere somehow this
# information (of us wanting to update TEST_HC) gets lost in the process,
# resulting in the final TEST_HC always getting set to the inplace compiler.
# It seems possible to remedy this yet again by exporting TEST_HC explicitly,
# but I didn't understand nor test it thoroughly (what about the other
# variables we override, see calls to canonicalise), and the '+' trick seems
# to work at least equally well (just don't run something like
# 'make test fast slow accept').
#
# Tests:
# * `make TEST=T3307 -j2` should not show a warning.
# * `make TEST=tc001 TEST_HC=ghc fast` should not use the inplace compiler.
#
# [1] https://www.gnu.org/software/make/manual/html_node/Variables_002fRecursion.html
# [2] https://www.gnu.org/software/make/manual/html_node/Error-Messages.html