testlib.py 71.5 KB
Newer Older
1
# coding=utf8
2
#
3 4 5
# (c) Simon Marlow 2002
#

6
from __future__ import print_function
7

8
import io
9
import shutil
10
import os
11
import errno
12 13 14
import string
import re
import traceback
15 16
import time
import datetime
17
import copy
18
import glob
19
import sys
ian@well-typed.com's avatar
ian@well-typed.com committed
20
from math import ceil, trunc
21
import collections
22
import subprocess
23

24
from testglobals import *
25
from testutil import *
26
extra_src_files = {'T4198': ['exitminus1.c']} # TODO: See #12223
27

28 29
if config.use_threads:
    import threading
30 31 32 33
    try:
        import thread
    except ImportError: # Python 3
        import _thread as thread
34

35 36
global wantToStop
wantToStop = False
37 38 39 40 41

global pool_sema
if config.use_threads:
    pool_sema = threading.BoundedSemaphore(value=config.threads)

42 43 44 45 46 47
def stopNow():
    global wantToStop
    wantToStop = True
def stopping():
    return wantToStop

48 49
# Options valid for the current test only (these get reset to
# testdir_testopts after each test).
50

ei@vuokko.info's avatar
ei@vuokko.info committed
51
global testopts_local
52 53 54 55 56 57
if config.use_threads:
    testopts_local = threading.local()
else:
    class TestOpts_Local:
        pass
    testopts_local = TestOpts_Local()
58 59

def getTestOpts():
ei@vuokko.info's avatar
ei@vuokko.info committed
60
    return testopts_local.x
61

ei@vuokko.info's avatar
ei@vuokko.info committed
62 63 64
def setLocalTestOpts(opts):
    global testopts_local
    testopts_local.x=opts
65

66 67
def isStatsTest():
    opts = getTestOpts()
68
    return bool(opts.compiler_stats_range_fields or opts.stats_range_fields)
69 70


71 72 73
# This can be called at the top of a file of tests, to set default test options
# for the following tests.
def setTestOpts( f ):
74
    global thisdir_settings
75
    thisdir_settings = [thisdir_settings, f]
76 77 78 79 80 81 82 83 84 85 86

# -----------------------------------------------------------------------------
# Canned setup functions for common cases.  eg. for a test you might say
#
#      test('test001', normal, compile, [''])
#
# to run it without any options, but change it to
#
#      test('test001', expect_fail, compile, [''])
#
# to expect failure for this test.
87 88
#
# type TestOpt = (name :: String, opts :: Object) -> IO ()
89

90
def normal( name, opts ):
91 92
    return;

93
def skip( name, opts ):
94 95
    opts.skip = 1

96
def expect_fail( name, opts ):
97 98 99
    # The compiler, testdriver, OS or platform is missing a certain
    # feature, and we don't plan to or can't fix it now or in the
    # future.
100 101
    opts.expect = 'fail';

102
def reqlib( lib ):
103
    return lambda name, opts, l=lib: _reqlib (name, opts, l )
104

105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134
def stage1(name, opts):
    # See Note [Why is there no stage1 setup function?]
    framework_fail(name, 'stage1 setup function does not exist',
                   'add your test to testsuite/tests/stage1 instead')

# Note [Why is there no stage1 setup function?]
#
# Presumably a stage1 setup function would signal that the stage1
# compiler should be used to compile a test.
#
# Trouble is, the path to the compiler + the `ghc --info` settings for
# that compiler are currently passed in from the `make` part of the
# testsuite driver.
#
# Switching compilers in the Python part would be entirely too late, as
# all ghc_with_* settings would be wrong. See config/ghc for possible
# consequences (for example, config.run_ways would still be
# based on the default compiler, quite likely causing ./validate --slow
# to fail).
#
# It would be possible to let the Python part of the testsuite driver
# make the call to `ghc --info`, but doing so would require quite some
# work. Care has to be taken to not affect the run_command tests for
# example, as they also use the `ghc --info` settings:
#     quasiquotation/qq007/Makefile:ifeq "$(GhcDynamic)" "YES"
#
# If you want a test to run using the stage1 compiler, add it to the
# testsuite/tests/stage1 directory. Validate runs the tests in that
# directory with `make stage=1`.

135 136 137 138
# Cache the results of looking to see if we have a library or not.
# This makes quite a difference, especially on Windows.
have_lib = {}

139
def _reqlib( name, opts, lib ):
140
    if lib in have_lib:
141
        got_it = have_lib[lib]
142
    else:
143 144 145
        cmd = strip_quotes(config.ghc_pkg)
        p = subprocess.Popen([cmd, '--no-user-package-db', 'describe', lib],
                             stdout=subprocess.PIPE,
146 147
                             stderr=subprocess.PIPE,
                             env=ghc_env)
148 149 150 151
        # read from stdout and stderr to avoid blocking due to
        # buffers filling
        p.communicate()
        r = p.wait()
152 153 154 155
        got_it = r == 0
        have_lib[lib] = got_it

    if not got_it:
156
        opts.expect = 'missing-lib'
157

158 159 160 161
def req_haddock( name, opts ):
    if not config.haddock:
        opts.expect = 'missing-lib'

162
def req_profiling( name, opts ):
163
    '''Require the profiling libraries (add 'GhcLibWays += p' to mk/build.mk)'''
164 165 166
    if not config.have_profiling:
        opts.expect = 'fail'

167
def req_shared_libs( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
168 169 170
    if not config.have_shared_libs:
        opts.expect = 'fail'

171
def req_interp( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
172 173 174
    if not config.have_interp:
        opts.expect = 'fail'

175
def req_smp( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
176 177 178
    if not config.have_smp:
        opts.expect = 'fail'

179 180 181 182 183
def ignore_stdout(name, opts):
    opts.ignore_stdout = True

def ignore_stderr(name, opts):
    opts.ignore_stderr = True
184

185
def combined_output( name, opts ):
pcapriotti's avatar
pcapriotti committed
186 187
    opts.combined_output = True

188 189 190
# -----

def expect_fail_for( ways ):
191
    return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
192

193
def _expect_fail_for( name, opts, ways ):
194 195
    opts.expect_fail_for = ways

196
def expect_broken( bug ):
197 198
    # This test is a expected not to work due to the indicated trac bug
    # number.
199 200 201
    return lambda name, opts, b=bug: _expect_broken (name, opts, b )

def _expect_broken( name, opts, bug ):
202
    record_broken(name, opts, bug)
203 204
    opts.expect = 'fail';

Ian Lynagh's avatar
Ian Lynagh committed
205
def expect_broken_for( bug, ways ):
206
    return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
Ian Lynagh's avatar
Ian Lynagh committed
207

208
def _expect_broken_for( name, opts, bug, ways ):
209
    record_broken(name, opts, bug)
Ian Lynagh's avatar
Ian Lynagh committed
210
    opts.expect_fail_for = ways
Ian Lynagh's avatar
Ian Lynagh committed
211

212
def record_broken(name, opts, bug):
213
    global brokens
214
    me = (bug, opts.testdir, name)
215 216 217
    if not me in brokens:
        brokens.append(me)

218 219 220 221 222
def _expect_pass(way):
    # Helper function. Not intended for use in .T files.
    opts = getTestOpts()
    return opts.expect == 'pass' and way not in opts.expect_fail_for

223 224 225
# -----

def omit_ways( ways ):
226
    return lambda name, opts, w=ways: _omit_ways( name, opts, w )
227

228
def _omit_ways( name, opts, ways ):
229 230 231 232
    opts.omit_ways = ways

# -----

233
def only_ways( ways ):
234
    return lambda name, opts, w=ways: _only_ways( name, opts, w )
235

236
def _only_ways( name, opts, ways ):
237 238 239 240
    opts.only_ways = ways

# -----

241
def extra_ways( ways ):
242
    return lambda name, opts, w=ways: _extra_ways( name, opts, w )
243

244
def _extra_ways( name, opts, ways ):
245 246 247 248
    opts.extra_ways = ways

# -----

249
def set_stdin( file ):
250
   return lambda name, opts, f=file: _set_stdin(name, opts, f);
251

252
def _set_stdin( name, opts, f ):
253 254 255 256 257
   opts.stdin = f

# -----

def exit_code( val ):
258
    return lambda name, opts, v=val: _exit_code(name, opts, v);
259

260
def _exit_code( name, opts, v ):
261 262
    opts.exit_code = v

263 264 265 266 267 268 269 270 271 272 273
def signal_exit_code( val ):
    if opsys('solaris2'):
        return exit_code( val );
    else:
        # When application running on Linux receives fatal error
        # signal, then its exit code is encoded as 128 + signal
        # value. See http://www.tldp.org/LDP/abs/html/exitcodes.html
        # I assume that Mac OS X behaves in the same way at least Mac
        # OS X builder behavior suggests this.
        return exit_code( val+128 );

274 275
# -----

276 277
def compile_timeout_multiplier( val ):
    return lambda name, opts, v=val: _compile_timeout_multiplier(name, opts, v)
278

279 280 281 282 283 284 285 286
def _compile_timeout_multiplier( name, opts, v ):
    opts.compile_timeout_multiplier = v

def run_timeout_multiplier( val ):
    return lambda name, opts, v=val: _run_timeout_multiplier(name, opts, v)

def _run_timeout_multiplier( name, opts, v ):
    opts.run_timeout_multiplier = v
287 288 289

# -----

290
def extra_run_opts( val ):
291
    return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
292

293
def _extra_run_opts( name, opts, v ):
294 295
    opts.extra_run_opts = v

296 297
# -----

Simon Marlow's avatar
Simon Marlow committed
298
def extra_hc_opts( val ):
299
    return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
Simon Marlow's avatar
Simon Marlow committed
300

301
def _extra_hc_opts( name, opts, v ):
Simon Marlow's avatar
Simon Marlow committed
302 303 304 305
    opts.extra_hc_opts = v

# -----

306
def extra_clean( files ):
307 308
    # TODO. Remove all calls to extra_clean.
    return lambda _name, _opts: None
309

310 311 312 313 314 315
def extra_files(files):
    return lambda name, opts: _extra_files(name, opts, files)

def _extra_files(name, opts, files):
    opts.extra_files.extend(files)

316 317
# -----

318 319 320 321 322 323 324
def stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);

def _stats_num_field( name, opts, field, expecteds ):
    if field in opts.stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

325
    if type(expecteds) is list:
326 327 328 329
        for (b, expected, dev) in expecteds:
            if b:
                opts.stats_range_fields[field] = (expected, dev)
                return
330
        framework_warn(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
331

332 333 334
    else:
        (expected, dev) = expecteds
        opts.stats_range_fields[field] = (expected, dev)
335 336 337

def compiler_stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
338

339 340 341 342
def _compiler_stats_num_field( name, opts, field, expecteds ):
    if field in opts.compiler_stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

343 344
    # Compiler performance numbers change when debugging is on, making the results
    # useless and confusing. Therefore, skip if debugging is on.
345 346 347
    if compiler_debugged():
        skip(name, opts)

348 349 350 351
    for (b, expected, dev) in expecteds:
        if b:
            opts.compiler_stats_range_fields[field] = (expected, dev)
            return
352

353
    framework_warn(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
354

355 356
# -----

357
def when(b, f):
ian@well-typed.com's avatar
ian@well-typed.com committed
358 359 360
    # When list_brokens is on, we want to see all expect_broken calls,
    # so we always do f
    if b or config.list_broken:
361 362 363 364 365 366 367
        return f
    else:
        return normal

def unless(b, f):
    return when(not b, f)

ian@well-typed.com's avatar
ian@well-typed.com committed
368 369 370
def doing_ghci():
    return 'ghci' in config.run_ways

371
def ghc_dynamic():
ian@well-typed.com's avatar
ian@well-typed.com committed
372
    return config.ghc_dynamic
373 374

def fast():
375
    return config.speed == 2
376

377 378
def platform( plat ):
    return config.platform == plat
Ian Lynagh's avatar
Ian Lynagh committed
379

380 381
def opsys( os ):
    return config.os == os
Ian Lynagh's avatar
Ian Lynagh committed
382

383 384
def arch( arch ):
    return config.arch == arch
385

386 387
def wordsize( ws ):
    return config.wordsize == str(ws)
tibbe's avatar
tibbe committed
388

389 390
def msys( ):
    return config.msys
ian@well-typed.com's avatar
ian@well-typed.com committed
391

392 393
def cygwin( ):
    return config.cygwin
Ian Lynagh's avatar
Ian Lynagh committed
394

395 396
def have_vanilla( ):
    return config.have_vanilla
397

398 399
def have_dynamic( ):
    return config.have_dynamic
400

401 402
def have_profiling( ):
    return config.have_profiling
403

404 405
def in_tree_compiler( ):
    return config.in_tree_compiler
406

407 408 409 410 411 412
def unregisterised( ):
    return config.unregisterised

def compiler_profiled( ):
    return config.compiler_profiled

413 414
def compiler_debugged( ):
    return config.compiler_debugged
415

416 417 418 419 420 421
def have_gdb( ):
    return config.have_gdb

def have_readelf( ):
    return config.have_readelf

422 423
# ---

424
def high_memory_usage(name, opts):
425 426
    opts.alone = True

427 428 429 430 431
# If a test is for a multi-CPU race, then running the test alone
# increases the chance that we'll actually see it.
def multi_cpu_race(name, opts):
    opts.alone = True

Ian Lynagh's avatar
Ian Lynagh committed
432
# ---
433
def literate( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
434 435
    opts.literate = 1;

436
def c_src( name, opts ):
437 438
    opts.c_src = 1;

439
def objc_src( name, opts ):
Austin Seipp's avatar
Austin Seipp committed
440 441
    opts.objc_src = 1;

442
def objcpp_src( name, opts ):
443 444
    opts.objcpp_src = 1;

445
def cmm_src( name, opts ):
446 447
    opts.cmm_src = 1;

448
def outputdir( odir ):
449
    return lambda name, opts, d=odir: _outputdir(name, opts, d)
450

451
def _outputdir( name, opts, odir ):
452 453
    opts.outputdir = odir;

454 455
# ----

456
def pre_cmd( cmd ):
457
    return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
458

459
def _pre_cmd( name, opts, cmd ):
460 461 462 463
    opts.pre_cmd = cmd

# ----

464
def clean_cmd( cmd ):
465 466
    # TODO. Remove all calls to clean_cmd.
    return lambda _name, _opts: None
467 468 469

# ----

470
def cmd_prefix( prefix ):
471
    return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
472

473
def _cmd_prefix( name, opts, prefix ):
474 475 476 477 478
    opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;

# ----

def cmd_wrapper( fun ):
479
    return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
480

481
def _cmd_wrapper( name, opts, fun ):
482
    opts.cmd_wrapper = fun
483

484 485
# ----

Ian Lynagh's avatar
Ian Lynagh committed
486
def compile_cmd_prefix( prefix ):
487
    return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
Ian Lynagh's avatar
Ian Lynagh committed
488

489
def _compile_cmd_prefix( name, opts, prefix ):
Ian Lynagh's avatar
Ian Lynagh committed
490 491 492 493
    opts.compile_cmd_prefix = prefix

# ----

494 495 496 497 498 499
def check_stdout( f ):
    return lambda name, opts, f=f: _check_stdout(name, opts, f)

def _check_stdout( name, opts, f ):
    opts.check_stdout = f

500 501 502
def no_check_hp(name, opts):
    opts.check_hp = False

503 504
# ----

505 506 507 508 509 510 511
def filter_stdout_lines( regex ):
    """ Filter lines of stdout with the given regular expression """
    import re
    def f( name, opts ):
        _normalise_fun(name, opts, lambda s: '\n'.join(re.findall(regex, s)))
    return f

512
def normalise_slashes( name, opts ):
513
    _normalise_fun(name, opts, normalise_slashes_)
514

515
def normalise_exe( name, opts ):
516
    _normalise_fun(name, opts, normalise_exe_)
517

518 519
def normalise_fun( *fs ):
    return lambda name, opts: _normalise_fun(name, opts, fs)
520

521
def _normalise_fun( name, opts, *fs ):
522
    opts.extra_normaliser = join_normalisers(opts.extra_normaliser, fs)
523

524 525
def normalise_errmsg_fun( *fs ):
    return lambda name, opts: _normalise_errmsg_fun(name, opts, fs)
526

527
def _normalise_errmsg_fun( name, opts, *fs ):
528 529
    opts.extra_errmsg_normaliser =  join_normalisers(opts.extra_errmsg_normaliser, fs)

Joachim Breitner's avatar
Joachim Breitner committed
530 531 532 533 534 535 536 537 538 539 540 541 542
def check_errmsg(needle):
    def norm(str):
        if needle in str:
            return "%s contained in -ddump-simpl\n" % needle
        else:
            return "%s not contained in -ddump-simpl\n" % needle
    return normalise_errmsg_fun(norm)

def grep_errmsg(needle):
    def norm(str):
        return "".join(filter(lambda l: re.search(needle, l), str.splitlines(True)))
    return normalise_errmsg_fun(norm)

543 544 545 546 547 548
def normalise_whitespace_fun(f):
    return lambda name, opts: _normalise_whitespace_fun(name, opts, f)

def _normalise_whitespace_fun(name, opts, f):
    opts.whitespace_normaliser = f

549 550 551 552 553 554 555 556 557 558 559
def normalise_version_( *pkgs ):
    def normalise_version__( str ):
        return re.sub('(' + '|'.join(map(re.escape,pkgs)) + ')-[0-9.]+',
                      '\\1-<VERSION>', str)
    return normalise_version__

def normalise_version( *pkgs ):
    def normalise_version__( name, opts ):
        _normalise_fun(name, opts, normalise_version_(*pkgs))
        _normalise_errmsg_fun(name, opts, normalise_version_(*pkgs))
    return normalise_version__
560

thomie's avatar
thomie committed
561 562 563 564
def normalise_drive_letter(name, opts):
    # Windows only. Change D:\\ to C:\\.
    _normalise_fun(name, opts, lambda str: re.sub(r'[A-Z]:\\', r'C:\\', str))

565 566 567 568 569 570 571
def keep_prof_callstacks(name, opts):
    """Keep profiling callstacks.

    Use together with `only_ways(prof_ways)`.
    """
    opts.keep_prof_callstacks = True

572 573
def join_normalisers(*a):
    """
574
    Compose functions, flattening sequences.
575

576
       join_normalisers(f1,[f2,f3],f4)
577 578 579

    is the same as

580
       lambda x: f1(f2(f3(f4(x))))
581 582
    """

583 584 585 586 587
    def flatten(l):
        """
        Taken from http://stackoverflow.com/a/2158532/946226
        """
        for el in l:
588 589
            if (isinstance(el, collections.Iterable)
                and not isinstance(el, (bytes, str))):
590 591 592 593 594 595
                for sub in flatten(el):
                    yield sub
            else:
                yield el

    a = flatten(a)
596 597 598

    fn = lambda x:x # identity function
    for f in a:
599
        assert callable(f)
600 601 602
        fn = lambda x,f=f,fn=fn: fn(f(x))
    return fn

603 604 605
# ----
# Function for composing two opt-fns together

606
def executeSetups(fs, name, opts):
607
    if type(fs) is list:
608
        # If we have a list of setups, then execute each one
609 610
        for f in fs:
            executeSetups(f, name, opts)
611 612 613
    else:
        # fs is a single function, so just apply it
        fs(name, opts)
614

615 616 617
# -----------------------------------------------------------------------------
# The current directory of tests

618 619
def newTestDir(tempdir, dir):

620
    global thisdir_settings
621
    # reset the options for this test directory
622 623 624
    def settings(name, opts, tempdir=tempdir, dir=dir):
        return _newTestDir(name, opts, tempdir, dir)
    thisdir_settings = settings
625

626 627
# Should be equal to entry in toplevel .gitignore.
testdir_suffix = '.run'
628 629 630

def _newTestDir(name, opts, tempdir, dir):
    opts.srcdir = os.path.join(os.getcwd(), dir)
631
    opts.testdir = os.path.join(tempdir, dir, name + testdir_suffix)
632
    opts.compiler_always_flags = config.compiler_always_flags
633 634 635 636

# -----------------------------------------------------------------------------
# Actually doing tests

637 638
parallelTests = []
aloneTests = []
639
allTestNames = set([])
640

641
def runTest(watcher, opts, name, func, args):
642
    if config.use_threads:
643 644 645 646 647 648
        pool_sema.acquire()
        t = threading.Thread(target=test_common_thread,
                             name=name,
                             args=(watcher, name, opts, func, args))
        t.daemon = False
        t.start()
649
    else:
650
        test_common_work(watcher, name, opts, func, args)
651

652
# name  :: String
653
# setup :: [TestOpt] -> IO ()
654
def test(name, setup, func, args):
655 656 657 658 659 660 661 662 663
    global aloneTests
    global parallelTests
    global allTestNames
    global thisdir_settings
    if name in allTestNames:
        framework_fail(name, 'duplicate', 'There are multiple tests with this name')
    if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
        framework_fail(name, 'bad_name', 'This test has an invalid name')

664 665 666 667 668
    if config.run_only_some_tests:
        if name not in config.only:
            return
        else:
            # Note [Mutating config.only]
Gabor Greif's avatar
Gabor Greif committed
669
            # config.only is initially the set of tests requested by
670 671 672 673
            # the user (via 'make TEST='). We then remove all tests that
            # we've already seen (in .T files), so that we can later
            # report on any tests we couldn't find and error out.
            config.only.remove(name)
674

675 676 677 678
    # Make a deep copy of the default_testopts, as we need our own copy
    # of any dictionaries etc inside it. Otherwise, if one test modifies
    # them, all tests will see the modified version!
    myTestOpts = copy.deepcopy(default_testopts)
679

680
    executeSetups([thisdir_settings, setup], name, myTestOpts)
681

682
    thisTest = lambda watcher: runTest(watcher, myTestOpts, name, func, args)
683 684 685 686
    if myTestOpts.alone:
        aloneTests.append(thisTest)
    else:
        parallelTests.append(thisTest)
687
    allTestNames.add(name)
688

689
if config.use_threads:
690 691 692 693 694
    def test_common_thread(watcher, name, opts, func, args):
            try:
                test_common_work(watcher, name, opts, func, args)
            finally:
                pool_sema.release()
695

696 697 698 699 700 701 702 703 704
def get_package_cache_timestamp():
    if config.package_conf_cache_file == '':
        return 0.0
    else:
        try:
            return os.stat(config.package_conf_cache_file).st_mtime
        except:
            return 0.0

Edward Z. Yang's avatar
Edward Z. Yang committed
705
do_not_copy = ('.hi', '.o', '.dyn_hi', '.dyn_o', '.out') # 12112
706

707
def test_common_work(watcher, name, opts, func, args):
708
    try:
709
        t.total_tests += 1
710 711 712 713 714 715 716
        setLocalTestOpts(opts)

        package_conf_cache_file_start_timestamp = get_package_cache_timestamp()

        # All the ways we might run this test
        if func == compile or func == multimod_compile:
            all_ways = config.compile_ways
717
        elif func == compile_and_run or func == multimod_compile_and_run:
718 719 720 721 722 723
            all_ways = config.run_ways
        elif func == ghci_script:
            if 'ghci' in config.run_ways:
                all_ways = ['ghci']
            else:
                all_ways = []
724
        else:
725 726 727
            all_ways = ['normal']

        # A test itself can request extra ways by setting opts.extra_ways
728
        all_ways = all_ways + [way for way in opts.extra_ways if way not in all_ways]
729

730
        t.total_test_cases += len(all_ways)
731 732 733

        ok_way = lambda way: \
            not getTestOpts().skip \
734
            and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
735
            and (config.cmdline_ways == [] or way in config.cmdline_ways) \
736
            and (not (config.skip_perf_tests and isStatsTest())) \
737 738 739
            and way not in getTestOpts().omit_ways

        # Which ways we are asked to skip
740
        do_ways = list(filter (ok_way,all_ways))
741

742 743
        # Only run all ways in slow mode.
        # See Note [validate and testsuite speed] in toplevel Makefile.
744 745
        if config.accept:
            # Only ever run one way
746
            do_ways = do_ways[:1]
747 748 749
        elif config.speed > 0:
            # However, if we EXPLICITLY asked for a way (with extra_ways)
            # please test it!
750 751
            explicit_ways = list(filter(lambda way: way in opts.extra_ways, do_ways))
            other_ways = list(filter(lambda way: way not in opts.extra_ways, do_ways))
752
            do_ways = other_ways[:1] + explicit_ways
753

754 755 756 757 758 759 760 761
        # Find all files in the source directory that this test
        # depends on. Do this only once for all ways.
        # Generously add all filenames that start with the name of
        # the test to this set, as a convenience to test authors.
        # They will have to use the `extra_files` setup function to
        # specify all other files that their test depends on (but
        # this seems to be necessary for only about 10% of all
        # tests).
762 763 764 765
        files = set(f for f in os.listdir(opts.srcdir)
                       if f.startswith(name) and not f == name and
                          not f.endswith(testdir_suffix) and
                          not os.path.splitext(f)[1] in do_not_copy)
766
        for filename in (opts.extra_files + extra_src_files.get(name, [])):
767
            if filename.startswith('/'):
768 769 770 771 772 773 774 775 776
                framework_fail(name, 'whole-test',
                    'no absolute paths in extra_files please: ' + filename)

            elif '*' in filename:
                # Don't use wildcards in extra_files too much, as
                # globbing is slow.
                files.update((os.path.relpath(f, opts.srcdir)
                            for f in glob.iglob(in_srcdir(filename))))

thomie's avatar
thomie committed
777
            elif filename:
778 779
                files.add(filename)

thomie's avatar
thomie committed
780 781 782
            else:
                framework_fail(name, 'whole-test', 'extra_file is empty string')

783 784 785 786
        # Run the required tests...
        for way in do_ways:
            if stopping():
                break
787 788 789 790 791 792 793
            try:
                do_test(name, way, func, args, files)
            except KeyboardInterrupt:
                stopNow()
            except Exception as e:
                framework_fail(name, way, str(e))
                traceback.print_exc()
794

795
        t.n_tests_skipped += len(set(all_ways) - set(do_ways))
796

797
        if config.cleanup and do_ways:
798 799 800 801
            try:
                cleanup()
            except Exception as e:
                framework_fail(name, 'runTest', 'Unhandled exception during cleanup: ' + str(e))
802

803
        package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
804

805 806
        if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
            framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
807

808
    except Exception as e:
809
        framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
810 811
    finally:
        watcher.notify()
Ian Lynagh's avatar
Ian Lynagh committed
812

813 814 815
def do_test(name, way, func, args, files):
    opts = getTestOpts()

816 817
    full_name = name + '(' + way + ')'

818
    if_verbose(2, "=====> {0} {1} of {2} {3}".format(
819
        full_name, t.total_tests, len(allTestNames),
820 821 822
        [len(t.unexpected_passes),
         len(t.unexpected_failures),
         len(t.framework_failures)]))
823 824 825 826

    # Clean up prior to the test, so that we can't spuriously conclude
    # that it passed on the basis of old run outputs.
    cleanup()
827
    os.makedirs(opts.testdir)
828 829 830 831 832 833 834 835 836

    # Link all source files for this test into a new directory in
    # /tmp, and run the test in that directory. This makes it
    # possible to run tests in parallel, without modification, that
    # would otherwise (accidentally) write to the same output file.
    # It also makes it easier to keep the testsuite clean.

    for extra_file in files:
        src = in_srcdir(extra_file)
837
        dst = in_testdir(os.path.basename(extra_file.rstrip('/\\')))
838
        if os.path.isfile(src):
839
            link_or_copy_file(src, dst)
840
        elif os.path.isdir(src):
841
            os.mkdir(dst)
842 843
            lndir(src, dst)
        else:
844
            if not config.haddock and os.path.splitext(extra_file)[1] == '.t':
845 846 847 848 849 850 851 852 853 854 855 856 857 858
                # When using a ghc built without haddock support, .t
                # files are rightfully missing. Don't
                # framework_fail. Test will be skipped later.
                pass
            else:
                framework_fail(name, way,
                    'extra_file does not exist: ' + extra_file)

    if func.__name__ == 'run_command' or opts.pre_cmd:
        # When running 'MAKE' make sure 'TOP' still points to the
        # root of the testsuite.
        src_makefile = in_srcdir('Makefile')
        dst_makefile = in_testdir('Makefile')
        if os.path.exists(src_makefile):
859
            with io.open(src_makefile, 'r', encoding='utf8') as src:
860
                makefile = re.sub('TOP=.*', 'TOP=' + config.top, src.read(), 1)
861
                with io.open(dst_makefile, 'w', encoding='utf8') as dst:
862
                    dst.write(makefile)
863

864
    if opts.pre_cmd:
865 866 867 868
        exit_code = runCmd('cd "{0}" && {1}'.format(opts.testdir, override_options(opts.pre_cmd)),
                           stderr = subprocess.STDOUT,