test.mk 12.3 KB
Newer Older
1 2 3
# -----------------------------------------------------------------------------
# Examples of use:
#
dterei's avatar
dterei committed
4 5 6
#  make           -- run all the tests in the current directory
#  make verbose   -- as make test, but up the verbosity
#  make accept    -- run the tests, accepting the current output
7 8 9
#
# The following variables may be set on the make command line:
#
dterei's avatar
dterei committed
10 11 12 13 14 15 16
#  TEST      -- specific test to run
#  TESTS     -- specific tests to run (same as $TEST really)
#  EXTRA_HC_OPTS      -- extra flags to send to the Haskell compiler
#  EXTRA_RUNTEST_OPTS -- extra flags to give the test driver
#  CONFIG    -- use a different configuration file
#  COMPILER  -- select a configuration file from config/
#  THREADS   -- run n tests at once
17 18 19
#  PLATFORM  -- if accepting a result, accept it for the current platform.
#  OS        -- if accepting a result, accept it for all wordsizes of the
#               current os.
20 21 22
#
# -----------------------------------------------------------------------------

23
# export the value of $MAKE for invocation in tests/driver/
dons's avatar
dons committed
24 25
export MAKE

26
RUNTESTS     = $(TOP)/driver/runtests.py
ross's avatar
ross committed
27
COMPILER     = ghc
28 29
CONFIGDIR    = $(TOP)/config
CONFIG       = $(CONFIGDIR)/$(COMPILER)
30

31 32 33 34 35 36 37
ifeq "$(GhcUnregisterised)" "YES"
    # Otherwise C backend generates many warnings about
    # imcompatible proto casts for GCC's buitins:
    #    memcpy, printf, strlen.
    EXTRA_HC_OPTS += -optc-fno-builtin
endif

38
# TEST_HC_OPTS is passed to every invocation of TEST_HC
39
# in nested Makefiles
40
TEST_HC_OPTS = -dcore-lint -dcmm-lint -no-user-$(GhcPackageDbFlag) -rtsopts $(EXTRA_HC_OPTS)
41

42
ifeq "$(MinGhcVersion711)" "YES"
43 44 45
# Don't warn about missing specialisations. They can only occur with `-O`, but
# we want tests to produce the same output for all test ways.
TEST_HC_OPTS += -fno-warn-missed-specialisations
46
TEST_HC_OPTS += -fshow-warning-groups
47
endif
48

49 50 51 52
ifeq "$(MinGhcVersion801)" "YES"
# Turn off any VT800 codes in the output or they wreak havoc on the
# testsuite output.
TEST_HC_OPTS += -fdiagnostics-color=never
Rufflewind's avatar
Rufflewind committed
53
TEST_HC_OPTS += -fno-diagnostics-show-caret
54 55
endif

56 57 58 59 60 61 62
# Add the no-debug-output last as it is often convenient to copy the test invocation
# removing this line.
TEST_HC_OPTS += -dno-debug-output

TEST_HC_OPTS_INTERACTIVE = $(TEST_HC_OPTS) --interactive -v0 -ignore-dot-ghci -fno-ghci-history


63 64 65 66
RUNTEST_OPTS =

ifeq "$(filter $(TargetOS_CPP), cygwin32 mingw32)" ""
exeext =
67
else
68
exeext = .exe
69
endif
70

71 72 73 74 75 76 77 78
ifneq "$(filter $(TargetOS_CPP),cygwin32 mingw32)" ""
dllext = .dll
else ifeq "$(TargetOS_CPP)" "darwin"
dllext = .dylib
else
dllext = .so
endif

79
RUNTEST_OPTS += -e "ghc_compiler_always_flags='$(TEST_HC_OPTS)'"
80

thomie's avatar
thomie committed
81
RUNTEST_OPTS += -e config.compiler_debugged=$(GhcDebugged)
82

83
ifeq "$(GhcWithNativeCodeGen)" "YES"
84 85 86
RUNTEST_OPTS += -e ghc_with_native_codegen=1
else
RUNTEST_OPTS += -e ghc_with_native_codegen=0
87 88
endif

89
GHC_PRIM_LIBDIR := $(subst library-dirs: ,,$(shell "$(GHC_PKG)" field ghc-prim library-dirs --simple-output))
90 91 92
HAVE_VANILLA := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.hi ]; then echo YES; else echo NO; fi)
HAVE_DYNAMIC := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.dyn_hi ]; then echo YES; else echo NO; fi)
HAVE_PROFILING := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.p_hi ]; then echo YES; else echo NO; fi)
93

94
ifeq "$(HAVE_VANILLA)" "YES"
thomie's avatar
thomie committed
95
RUNTEST_OPTS += -e config.have_vanilla=True
96
else
thomie's avatar
thomie committed
97
RUNTEST_OPTS += -e config.have_vanilla=False
98 99
endif

100
ifeq "$(HAVE_DYNAMIC)" "YES"
thomie's avatar
thomie committed
101
RUNTEST_OPTS += -e config.have_dynamic=True
102
else
thomie's avatar
thomie committed
103
RUNTEST_OPTS += -e config.have_dynamic=False
104 105
endif

106
ifeq "$(HAVE_PROFILING)" "YES"
thomie's avatar
thomie committed
107
RUNTEST_OPTS += -e config.have_profiling=True
108
else
thomie's avatar
thomie committed
109
RUNTEST_OPTS += -e config.have_profiling=False
110
endif
111

112 113 114 115 116 117
ifeq "$(filter thr, $(GhcRTSWays))" "thr"
RUNTEST_OPTS += -e ghc_with_threaded_rts=1
else
RUNTEST_OPTS += -e ghc_with_threaded_rts=0
endif

118 119 120 121 122 123
ifeq "$(filter dyn, $(GhcRTSWays))" "dyn"
RUNTEST_OPTS += -e ghc_with_dynamic_rts=1
else
RUNTEST_OPTS += -e ghc_with_dynamic_rts=0
endif

124
ifeq "$(GhcWithInterpreter)" "NO"
thomie's avatar
thomie committed
125
RUNTEST_OPTS += -e config.have_interp=False
126
else ifeq "$(GhcStage)" "1"
thomie's avatar
thomie committed
127
RUNTEST_OPTS += -e config.have_interp=False
128
else
thomie's avatar
thomie committed
129
RUNTEST_OPTS += -e config.have_interp=True
130 131
endif

132
ifeq "$(GhcUnregisterised)" "YES"
thomie's avatar
thomie committed
133
RUNTEST_OPTS += -e config.unregisterised=True
134
else
thomie's avatar
thomie committed
135
RUNTEST_OPTS += -e config.unregisterised=False
136 137
endif

138
ifeq "$(GhcDynamicByDefault)" "YES"
thomie's avatar
thomie committed
139
RUNTEST_OPTS += -e config.ghc_dynamic_by_default=True
140
CABAL_MINIMAL_BUILD = --enable-shared --disable-library-vanilla
141
else
thomie's avatar
thomie committed
142
RUNTEST_OPTS += -e config.ghc_dynamic_by_default=False
143
CABAL_MINIMAL_BUILD = --enable-library-vanilla --disable-shared
144 145
endif

ian@well-typed.com's avatar
ian@well-typed.com committed
146
ifeq "$(GhcDynamic)" "YES"
thomie's avatar
thomie committed
147
RUNTEST_OPTS += -e config.ghc_dynamic=True
ian@well-typed.com's avatar
ian@well-typed.com committed
148 149
CABAL_PLUGIN_BUILD = --enable-shared --disable-library-vanilla
else
thomie's avatar
thomie committed
150
RUNTEST_OPTS += -e config.ghc_dynamic=False
ian@well-typed.com's avatar
ian@well-typed.com committed
151 152 153
CABAL_PLUGIN_BUILD = --enable-library-vanilla --disable-shared
endif

154 155 156 157 158 159
ifeq "$(GhcWithSMP)" "YES"
RUNTEST_OPTS += -e ghc_with_smp=1
else
RUNTEST_OPTS += -e ghc_with_smp=0
endif

160 161
ifeq "$(LLC)" ""
RUNTEST_OPTS += -e ghc_with_llvm=0
162 163
else ifeq "$(TargetARCH_CPP)" "powerpc"
RUNTEST_OPTS += -e ghc_with_llvm=0
164 165 166
else ifneq "$(LLC)" "llc"
# If we have a real detected value for LLVM, then it really ought to work
RUNTEST_OPTS += -e ghc_with_llvm=1
167 168 169 170
else
RUNTEST_OPTS += -e ghc_with_llvm=0
endif

Ian Lynagh's avatar
Ian Lynagh committed
171 172 173 174 175 176
ifeq "$(WINDOWS)" "YES"
RUNTEST_OPTS += -e windows=True
else
RUNTEST_OPTS += -e windows=False
endif

Ian Lynagh's avatar
Ian Lynagh committed
177 178 179 180 181 182
ifeq "$(DARWIN)" "YES"
RUNTEST_OPTS += -e darwin=True
else
RUNTEST_OPTS += -e darwin=False
endif

183
ifeq "$(IN_TREE_COMPILER)" "YES"
thomie's avatar
thomie committed
184
RUNTEST_OPTS += -e config.in_tree_compiler=True
185
else
thomie's avatar
thomie committed
186
RUNTEST_OPTS += -e config.in_tree_compiler=False
187 188
endif

ei@vuokko.info's avatar
ei@vuokko.info committed
189
ifneq "$(THREADS)" ""
190
RUNTEST_OPTS += --threads=$(THREADS)
ei@vuokko.info's avatar
ei@vuokko.info committed
191 192
endif

193 194 195 196
ifneq "$(VERBOSE)" ""
RUNTEST_OPTS += --verbose=$(VERBOSE)
endif

197 198 199 200
ifeq "$(SKIP_PERF_TESTS)" "YES"
RUNTEST_OPTS += --skip-perf-tests
endif

201
ifeq "$(CLEANUP)" "0"
202
RUNTEST_OPTS += -e config.cleanup=False
203
else ifeq "$(CLEANUP)" "NO"
204
RUNTEST_OPTS += -e config.cleanup=False
205
else
206 207 208 209 210 211 212 213 214 215
RUNTEST_OPTS += -e config.cleanup=True
endif

ifeq "$(LOCAL)" "0"
# See Note [Running tests in /tmp].
RUNTEST_OPTS += -e config.local=False
else ifeq "$(LOCAL)" "NO"
RUNTEST_OPTS += -e config.local=False
else
RUNTEST_OPTS += -e config.local=True
216 217
endif

218
RUNTEST_OPTS +=  \
219
	--rootdir=. \
220
	--config-file=$(CONFIG) \
Ian Lynagh's avatar
Ian Lynagh committed
221 222 223
	-e 'config.confdir="$(CONFIGDIR)"' \
	-e 'config.platform="$(TARGETPLATFORM)"' \
	-e 'config.os="$(TargetOS_CPP)"' \
224
	-e 'config.arch="$(TargetARCH_CPP)"' \
Ian Lynagh's avatar
Ian Lynagh committed
225
	-e 'config.wordsize="$(WORDSIZE)"' \
Ian Lynagh's avatar
Ian Lynagh committed
226
	-e 'config.timeout=int($(TIMEOUT)) or config.timeout' \
Ian Lynagh's avatar
Ian Lynagh committed
227
	-e 'config.exeext="$(exeext)"' \
228 229
	-e 'config.top="$(TOP_ABS)"'

230 231 232 233 234 235 236 237
# Wrap non-empty program paths in quotes, because they may contain spaces. Do
# it here, so we don't have to (and don't forget to do it) in the .T test
# scripts (search for '{compiler}' or '{hpc}'). This may or may not be a good
# idea.
# Use `--config` instead of `-e`, because `-e` (which calls Python's `eval`
# function) would require another pair of (escaped) quotes, which interfers
# with MinGW's magic path handling (see #10449, and
# http://www.mingw.org/wiki/Posix_path_conversion).
238 239
# We use double instead of single quotes, which may or may not be important
# when using msys2 (#9626, #10441).
240
quote_path = $(if $1,"$1")
241
RUNTEST_OPTS +=  \
242 243 244 245 246 247 248
	--config 'compiler=$(call quote_path,$(TEST_HC))' \
	--config 'ghc_pkg=$(call quote_path,$(GHC_PKG))' \
	--config 'haddock=$(call quote_path,$(HADDOCK))' \
	--config 'hp2ps=$(call quote_path,$(HP2PS_ABS))' \
	--config 'hpc=$(call quote_path,$(HPC))' \
	--config 'gs=$(call quote_path,$(GS))' \
	--config 'timeout_prog=$(call quote_path,$(TIMEOUT_PROGRAM))'
249

250 251
RUNTEST_OPTS += -e "config.stage=$(GhcStage)"

Ben Gamari's avatar
Ben Gamari committed
252 253 254 255
ifneq "$(JUNIT_FILE)" ""
RUNTEST_OPTS +=  \
  --junit "$(JUNIT_FILE)"
endif
thomie's avatar
thomie committed
256
ifneq "$(SUMMARY_FILE)" ""
257
RUNTEST_OPTS +=  \
thomie's avatar
thomie committed
258 259 260 261
	--summary-file "$(SUMMARY_FILE)"
endif
ifeq "$(NO_PRINT_SUMMARY)" "YES"
RUNTEST_OPTS +=  \
262
	--no-print-summary
263 264 265
endif

RUNTEST_OPTS +=  \
266 267
	$(EXTRA_RUNTEST_OPTS)

ian@well-typed.com's avatar
ian@well-typed.com committed
268 269 270
ifeq "$(list_broken)" "YES"
set_list_broken = -e config.list_broken=True
else
271
set_list_broken =
ian@well-typed.com's avatar
ian@well-typed.com committed
272 273
endif

274 275 276 277 278 279
# See Note [validate and testsuite speed] in toplevel Makefile.
ifneq "$(SPEED)" ""
setspeed = -e config.speed="$(SPEED)"
else ifeq "$(fast)" "YES"
# Backward compatibility. Maybe some people are running 'make accept fast=YES'?
setspeed = -e config.speed=2
280
else
281
setspeed =
282 283 284 285
endif

ifeq "$(accept)" "YES"
setaccept = -e config.accept=1
286 287 288 289 290 291 292 293 294

ifeq "$(PLATFORM)" "YES"
setaccept += -e config.accept_platform=1
endif

ifeq "$(OS)" "YES"
setaccept += -e config.accept_os=1
endif

295
else
296
setaccept =
297 298
endif

299
.PHONY: all boot test verbose accept fast slow list_broken
300 301 302

all: test

303
TIMEOUT_PROGRAM = $(TOP)/timeout/install-inplace/bin/timeout$(exeext)
304

305 306
boot: $(TIMEOUT_PROGRAM)

307
$(TIMEOUT_PROGRAM) :
308
	@echo "Looks like you don't have timeout, building it first..."
Ian Lynagh's avatar
Ian Lynagh committed
309
	$(MAKE) -C $(TOP)/timeout all
310

311 312 313
# Use a '+' to make sure that any sub-MAKEs that python spawns can
# communicate with the topmake.
# See Note [Communicating options and variables to a submake]
314
test: $(TIMEOUT_PROGRAM)
315
	+PYTHON="$(PYTHON)" "$(PYTHON)" $(RUNTESTS) $(RUNTEST_OPTS) \
316
		$(patsubst %, --only=%, $(TEST)) \
317
		$(patsubst %, --only=%, $(TESTS)) \
318
		$(patsubst %, --way=%, $(WAY)) \
319
		$(patsubst %, --skipway=%, $(SKIPWAY)) \
ian@well-typed.com's avatar
ian@well-typed.com committed
320
		$(set_list_broken) \
321
		$(setspeed) \
322
		$(setaccept)
323

324
verbose: test
325 326

accept:
327 328
	$(MAKE) accept=YES

329
fast fasttest:
330 331 332
	# See Note [validate and testsuite speed] in toplevel Makefile.
	$(MAKE) SPEED=2

333
slow slowtest:
334
	$(MAKE) SPEED=0
335

ian@well-typed.com's avatar
ian@well-typed.com committed
336 337 338
list_broken:
	$(MAKE) list_broken=YES

339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358
# Note [Communicating options and variables to a submake]
#
# Consider the following scenario:
#   * A test foo is defined as
#     test('foo', [], run_command, ['$MAKE footarget'])
#   * A user calls 'make -j24 TEST=foo'
#
# What happens is something like this:
#   * make (topmake) reads all options and variables given on the commandline
#     and adds them to the variable MAKEFLAGS [1]. This variable is exported by
#     default [1], so submakes can use them.
#   * The 'test' target calls 'python ..'
#   * Python calls 'make footarget' (submake)
#
# **First question**: what happens to the '-j24' option when calling make
# recursively?
#
# From
# https://www.gnu.org/software/make/manual/html_node/Variables_002fRecursion.html:
#
359 360 361 362
#     "The '-j' option is a special case (see Parallel Execution). If you set
#     it to some numeric value 'N' and your operating system supports it (most
#     any UNIX system will; others typically won't), the parent make and all the
#     sub-makes will communicate to ensure that there are only 'N' jobs running
363 364 365 366
#     at the same time between them all."
#
# In our scenario, the user will actually see the following warning [2]:
#
367
#     'warning: jobserver unavailable: using -j1. Add '+' to parent make rule.'
368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400
#
# The problem is that topmake and submake don't know about eachother, since
# python is in between. To let them communicate, we have to use the '+'
# option, by calling '+python' instead of 'python' [2]. This works,
# magically, and fixes #11569.
#
# **Second question**: can't we just unexport MAKEFLAGS, instead of using
# that '+' trick? The testsuite driver (python) mangages parallelism by
# itself already, so '-j24' doesn't do the right thing anyway. You have to
# use 'make test THREADS=24'. Unexporting MAKEFLAGS would mean ignoring
# any '-j' flags passed to make (either from the user calling 'make -j'
# explicitly or from having MAKEFLAGS=-j set in the shell, see #11569).
#
# This almost works, except when calling 'make fast/slow/accept TEST_HC=ghc'
# instead of just 'make test'. These targets call 'make test FAST=YES'
# recursively (and 'make test' calls python, as before).
#
# The problem is that in boilerplate.mk we try to override the variable
# TEST_HC (See Note [The TEST_HC variable]). Somewhere somehow this
# information (of us wanting to update TEST_HC) gets lost in the process,
# resulting in the final TEST_HC always getting set to the inplace compiler.
# It seems possible to remedy this yet again by exporting TEST_HC explicitly,
# but I didn't understand nor test it thoroughly (what about the other
# variables we override, see calls to canonicalise), and the '+' trick seems
# to work at least equally well (just don't run something like
# 'make test fast slow accept').
#
# Tests:
# * `make TEST=T3307 -j2` should not show a warning.
# * `make TEST=tc001 TEST_HC=ghc fast` should not use the inplace compiler.
#
# [1] https://www.gnu.org/software/make/manual/html_node/Variables_002fRecursion.html
# [2] https://www.gnu.org/software/make/manual/html_node/Error-Messages.html