testlib.py 71.4 KB
Newer Older
1
# coding=utf8
2
#
3 4 5
# (c) Simon Marlow 2002
#

6
from __future__ import print_function
7

8
import io
9
import shutil
10
import os
11
import errno
12 13 14
import string
import re
import traceback
15 16
import time
import datetime
17
import copy
18
import glob
19
import sys
ian@well-typed.com's avatar
ian@well-typed.com committed
20
from math import ceil, trunc
21
import collections
22
import subprocess
23

24
from testglobals import *
25
from testutil import *
26
extra_src_files = {'T4198': ['exitminus1.c']} # TODO: See #12223
27

28 29
if config.use_threads:
    import threading
30 31 32 33
    try:
        import thread
    except ImportError: # Python 3
        import _thread as thread
34

35 36
global wantToStop
wantToStop = False
37 38 39 40 41

global pool_sema
if config.use_threads:
    pool_sema = threading.BoundedSemaphore(value=config.threads)

42 43 44 45 46 47
def stopNow():
    global wantToStop
    wantToStop = True
def stopping():
    return wantToStop

48 49
# Options valid for the current test only (these get reset to
# testdir_testopts after each test).
50

ei@vuokko.info's avatar
ei@vuokko.info committed
51
global testopts_local
52 53 54 55 56 57
if config.use_threads:
    testopts_local = threading.local()
else:
    class TestOpts_Local:
        pass
    testopts_local = TestOpts_Local()
58 59

def getTestOpts():
ei@vuokko.info's avatar
ei@vuokko.info committed
60
    return testopts_local.x
61

ei@vuokko.info's avatar
ei@vuokko.info committed
62 63 64
def setLocalTestOpts(opts):
    global testopts_local
    testopts_local.x=opts
65

66 67
def isStatsTest():
    opts = getTestOpts()
68
    return bool(opts.compiler_stats_range_fields or opts.stats_range_fields)
69 70


71 72 73
# This can be called at the top of a file of tests, to set default test options
# for the following tests.
def setTestOpts( f ):
74
    global thisdir_settings
75
    thisdir_settings = [thisdir_settings, f]
76 77 78 79 80 81 82 83 84 85 86

# -----------------------------------------------------------------------------
# Canned setup functions for common cases.  eg. for a test you might say
#
#      test('test001', normal, compile, [''])
#
# to run it without any options, but change it to
#
#      test('test001', expect_fail, compile, [''])
#
# to expect failure for this test.
87 88
#
# type TestOpt = (name :: String, opts :: Object) -> IO ()
89

90
def normal( name, opts ):
91 92
    return;

93
def skip( name, opts ):
94 95
    opts.skip = 1

96
def expect_fail( name, opts ):
97 98 99
    # The compiler, testdriver, OS or platform is missing a certain
    # feature, and we don't plan to or can't fix it now or in the
    # future.
100 101
    opts.expect = 'fail';

102
def reqlib( lib ):
103
    return lambda name, opts, l=lib: _reqlib (name, opts, l )
104

105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134
def stage1(name, opts):
    # See Note [Why is there no stage1 setup function?]
    framework_fail(name, 'stage1 setup function does not exist',
                   'add your test to testsuite/tests/stage1 instead')

# Note [Why is there no stage1 setup function?]
#
# Presumably a stage1 setup function would signal that the stage1
# compiler should be used to compile a test.
#
# Trouble is, the path to the compiler + the `ghc --info` settings for
# that compiler are currently passed in from the `make` part of the
# testsuite driver.
#
# Switching compilers in the Python part would be entirely too late, as
# all ghc_with_* settings would be wrong. See config/ghc for possible
# consequences (for example, config.run_ways would still be
# based on the default compiler, quite likely causing ./validate --slow
# to fail).
#
# It would be possible to let the Python part of the testsuite driver
# make the call to `ghc --info`, but doing so would require quite some
# work. Care has to be taken to not affect the run_command tests for
# example, as they also use the `ghc --info` settings:
#     quasiquotation/qq007/Makefile:ifeq "$(GhcDynamic)" "YES"
#
# If you want a test to run using the stage1 compiler, add it to the
# testsuite/tests/stage1 directory. Validate runs the tests in that
# directory with `make stage=1`.

135 136 137 138
# Cache the results of looking to see if we have a library or not.
# This makes quite a difference, especially on Windows.
have_lib = {}

139
def _reqlib( name, opts, lib ):
140
    if lib in have_lib:
141
        got_it = have_lib[lib]
142
    else:
143 144 145
        cmd = strip_quotes(config.ghc_pkg)
        p = subprocess.Popen([cmd, '--no-user-package-db', 'describe', lib],
                             stdout=subprocess.PIPE,
146 147
                             stderr=subprocess.PIPE,
                             env=ghc_env)
148 149 150 151
        # read from stdout and stderr to avoid blocking due to
        # buffers filling
        p.communicate()
        r = p.wait()
152 153 154 155
        got_it = r == 0
        have_lib[lib] = got_it

    if not got_it:
156
        opts.expect = 'missing-lib'
157

158 159 160 161
def req_haddock( name, opts ):
    if not config.haddock:
        opts.expect = 'missing-lib'

162
def req_profiling( name, opts ):
163
    '''Require the profiling libraries (add 'GhcLibWays += p' to mk/build.mk)'''
164 165 166
    if not config.have_profiling:
        opts.expect = 'fail'

167
def req_shared_libs( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
168 169 170
    if not config.have_shared_libs:
        opts.expect = 'fail'

171
def req_interp( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
172 173 174
    if not config.have_interp:
        opts.expect = 'fail'

175
def req_smp( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
176 177 178
    if not config.have_smp:
        opts.expect = 'fail'

179 180 181 182 183
def ignore_stdout(name, opts):
    opts.ignore_stdout = True

def ignore_stderr(name, opts):
    opts.ignore_stderr = True
184

185
def combined_output( name, opts ):
pcapriotti's avatar
pcapriotti committed
186 187
    opts.combined_output = True

188 189 190
# -----

def expect_fail_for( ways ):
191
    return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
192

193
def _expect_fail_for( name, opts, ways ):
194 195
    opts.expect_fail_for = ways

196
def expect_broken( bug ):
197 198
    # This test is a expected not to work due to the indicated trac bug
    # number.
199 200 201
    return lambda name, opts, b=bug: _expect_broken (name, opts, b )

def _expect_broken( name, opts, bug ):
202
    record_broken(name, opts, bug)
203 204
    opts.expect = 'fail';

Ian Lynagh's avatar
Ian Lynagh committed
205
def expect_broken_for( bug, ways ):
206
    return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
Ian Lynagh's avatar
Ian Lynagh committed
207

208
def _expect_broken_for( name, opts, bug, ways ):
209
    record_broken(name, opts, bug)
Ian Lynagh's avatar
Ian Lynagh committed
210
    opts.expect_fail_for = ways
Ian Lynagh's avatar
Ian Lynagh committed
211

212
def record_broken(name, opts, bug):
213
    global brokens
214
    me = (bug, opts.testdir, name)
215 216 217
    if not me in brokens:
        brokens.append(me)

218 219 220 221 222
def _expect_pass(way):
    # Helper function. Not intended for use in .T files.
    opts = getTestOpts()
    return opts.expect == 'pass' and way not in opts.expect_fail_for

223 224 225
# -----

def omit_ways( ways ):
226
    return lambda name, opts, w=ways: _omit_ways( name, opts, w )
227

228
def _omit_ways( name, opts, ways ):
229 230 231 232
    opts.omit_ways = ways

# -----

233
def only_ways( ways ):
234
    return lambda name, opts, w=ways: _only_ways( name, opts, w )
235

236
def _only_ways( name, opts, ways ):
237 238 239 240
    opts.only_ways = ways

# -----

241
def extra_ways( ways ):
242
    return lambda name, opts, w=ways: _extra_ways( name, opts, w )
243

244
def _extra_ways( name, opts, ways ):
245 246 247 248
    opts.extra_ways = ways

# -----

249
def set_stdin( file ):
250
   return lambda name, opts, f=file: _set_stdin(name, opts, f);
251

252
def _set_stdin( name, opts, f ):
253 254 255 256 257
   opts.stdin = f

# -----

def exit_code( val ):
258
    return lambda name, opts, v=val: _exit_code(name, opts, v);
259

260
def _exit_code( name, opts, v ):
261 262
    opts.exit_code = v

263 264 265 266 267 268 269 270 271 272 273
def signal_exit_code( val ):
    if opsys('solaris2'):
        return exit_code( val );
    else:
        # When application running on Linux receives fatal error
        # signal, then its exit code is encoded as 128 + signal
        # value. See http://www.tldp.org/LDP/abs/html/exitcodes.html
        # I assume that Mac OS X behaves in the same way at least Mac
        # OS X builder behavior suggests this.
        return exit_code( val+128 );

274 275
# -----

276 277
def compile_timeout_multiplier( val ):
    return lambda name, opts, v=val: _compile_timeout_multiplier(name, opts, v)
278

279 280 281 282 283 284 285 286
def _compile_timeout_multiplier( name, opts, v ):
    opts.compile_timeout_multiplier = v

def run_timeout_multiplier( val ):
    return lambda name, opts, v=val: _run_timeout_multiplier(name, opts, v)

def _run_timeout_multiplier( name, opts, v ):
    opts.run_timeout_multiplier = v
287 288 289

# -----

290
def extra_run_opts( val ):
291
    return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
292

293
def _extra_run_opts( name, opts, v ):
294 295
    opts.extra_run_opts = v

296 297
# -----

Simon Marlow's avatar
Simon Marlow committed
298
def extra_hc_opts( val ):
299
    return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
Simon Marlow's avatar
Simon Marlow committed
300

301
def _extra_hc_opts( name, opts, v ):
Simon Marlow's avatar
Simon Marlow committed
302 303 304 305
    opts.extra_hc_opts = v

# -----

306
def extra_clean( files ):
307 308
    # TODO. Remove all calls to extra_clean.
    return lambda _name, _opts: None
309

310 311 312 313 314 315
def extra_files(files):
    return lambda name, opts: _extra_files(name, opts, files)

def _extra_files(name, opts, files):
    opts.extra_files.extend(files)

316 317
# -----

318 319 320 321 322 323 324
def stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);

def _stats_num_field( name, opts, field, expecteds ):
    if field in opts.stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

325
    if type(expecteds) is list:
326 327 328 329
        for (b, expected, dev) in expecteds:
            if b:
                opts.stats_range_fields[field] = (expected, dev)
                return
330
        framework_warn(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
331

332 333 334
    else:
        (expected, dev) = expecteds
        opts.stats_range_fields[field] = (expected, dev)
335 336 337

def compiler_stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
338

339 340 341 342
def _compiler_stats_num_field( name, opts, field, expecteds ):
    if field in opts.compiler_stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

343 344
    # Compiler performance numbers change when debugging is on, making the results
    # useless and confusing. Therefore, skip if debugging is on.
345 346 347
    if compiler_debugged():
        skip(name, opts)

348 349 350 351
    for (b, expected, dev) in expecteds:
        if b:
            opts.compiler_stats_range_fields[field] = (expected, dev)
            return
352

353
    framework_warn(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
354

355 356
# -----

357
def when(b, f):
ian@well-typed.com's avatar
ian@well-typed.com committed
358 359 360
    # When list_brokens is on, we want to see all expect_broken calls,
    # so we always do f
    if b or config.list_broken:
361 362 363 364 365 366 367
        return f
    else:
        return normal

def unless(b, f):
    return when(not b, f)

ian@well-typed.com's avatar
ian@well-typed.com committed
368 369 370
def doing_ghci():
    return 'ghci' in config.run_ways

371
def ghc_dynamic():
ian@well-typed.com's avatar
ian@well-typed.com committed
372
    return config.ghc_dynamic
373 374

def fast():
375
    return config.speed == 2
376

377 378
def platform( plat ):
    return config.platform == plat
Ian Lynagh's avatar
Ian Lynagh committed
379

380 381
def opsys( os ):
    return config.os == os
Ian Lynagh's avatar
Ian Lynagh committed
382

383 384
def arch( arch ):
    return config.arch == arch
385

386 387
def wordsize( ws ):
    return config.wordsize == str(ws)
tibbe's avatar
tibbe committed
388

389 390
def msys( ):
    return config.msys
ian@well-typed.com's avatar
ian@well-typed.com committed
391

392 393
def cygwin( ):
    return config.cygwin
Ian Lynagh's avatar
Ian Lynagh committed
394

395 396
def have_vanilla( ):
    return config.have_vanilla
397

398 399
def have_dynamic( ):
    return config.have_dynamic
400

401 402
def have_profiling( ):
    return config.have_profiling
403

404 405
def in_tree_compiler( ):
    return config.in_tree_compiler
406

407 408 409 410 411 412
def unregisterised( ):
    return config.unregisterised

def compiler_profiled( ):
    return config.compiler_profiled

413 414
def compiler_debugged( ):
    return config.compiler_debugged
415

416 417
# ---

418
def high_memory_usage(name, opts):
419 420
    opts.alone = True

421 422 423 424 425
# If a test is for a multi-CPU race, then running the test alone
# increases the chance that we'll actually see it.
def multi_cpu_race(name, opts):
    opts.alone = True

Ian Lynagh's avatar
Ian Lynagh committed
426
# ---
427
def literate( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
428 429
    opts.literate = 1;

430
def c_src( name, opts ):
431 432
    opts.c_src = 1;

433
def objc_src( name, opts ):
Austin Seipp's avatar
Austin Seipp committed
434 435
    opts.objc_src = 1;

436
def objcpp_src( name, opts ):
437 438
    opts.objcpp_src = 1;

439
def cmm_src( name, opts ):
440 441
    opts.cmm_src = 1;

442
def outputdir( odir ):
443
    return lambda name, opts, d=odir: _outputdir(name, opts, d)
444

445
def _outputdir( name, opts, odir ):
446 447
    opts.outputdir = odir;

448 449
# ----

450
def pre_cmd( cmd ):
451
    return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
452

453
def _pre_cmd( name, opts, cmd ):
454 455 456 457
    opts.pre_cmd = cmd

# ----

458
def clean_cmd( cmd ):
459 460
    # TODO. Remove all calls to clean_cmd.
    return lambda _name, _opts: None
461 462 463

# ----

464
def cmd_prefix( prefix ):
465
    return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
466

467
def _cmd_prefix( name, opts, prefix ):
468 469 470 471 472
    opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;

# ----

def cmd_wrapper( fun ):
473
    return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
474

475
def _cmd_wrapper( name, opts, fun ):
476
    opts.cmd_wrapper = fun
477

478 479
# ----

Ian Lynagh's avatar
Ian Lynagh committed
480
def compile_cmd_prefix( prefix ):
481
    return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
Ian Lynagh's avatar
Ian Lynagh committed
482

483
def _compile_cmd_prefix( name, opts, prefix ):
Ian Lynagh's avatar
Ian Lynagh committed
484 485 486 487
    opts.compile_cmd_prefix = prefix

# ----

488 489 490 491 492 493
def check_stdout( f ):
    return lambda name, opts, f=f: _check_stdout(name, opts, f)

def _check_stdout( name, opts, f ):
    opts.check_stdout = f

494 495 496
def no_check_hp(name, opts):
    opts.check_hp = False

497 498
# ----

499 500 501 502 503 504 505
def filter_stdout_lines( regex ):
    """ Filter lines of stdout with the given regular expression """
    import re
    def f( name, opts ):
        _normalise_fun(name, opts, lambda s: '\n'.join(re.findall(regex, s)))
    return f

506
def normalise_slashes( name, opts ):
507
    _normalise_fun(name, opts, normalise_slashes_)
508

509
def normalise_exe( name, opts ):
510
    _normalise_fun(name, opts, normalise_exe_)
511

512 513
def normalise_fun( *fs ):
    return lambda name, opts: _normalise_fun(name, opts, fs)
514

515
def _normalise_fun( name, opts, *fs ):
516
    opts.extra_normaliser = join_normalisers(opts.extra_normaliser, fs)
517

518 519
def normalise_errmsg_fun( *fs ):
    return lambda name, opts: _normalise_errmsg_fun(name, opts, fs)
520

521
def _normalise_errmsg_fun( name, opts, *fs ):
522 523
    opts.extra_errmsg_normaliser =  join_normalisers(opts.extra_errmsg_normaliser, fs)

Joachim Breitner's avatar
Joachim Breitner committed
524 525 526 527 528 529 530 531 532 533 534 535 536
def check_errmsg(needle):
    def norm(str):
        if needle in str:
            return "%s contained in -ddump-simpl\n" % needle
        else:
            return "%s not contained in -ddump-simpl\n" % needle
    return normalise_errmsg_fun(norm)

def grep_errmsg(needle):
    def norm(str):
        return "".join(filter(lambda l: re.search(needle, l), str.splitlines(True)))
    return normalise_errmsg_fun(norm)

537 538 539 540 541 542
def normalise_whitespace_fun(f):
    return lambda name, opts: _normalise_whitespace_fun(name, opts, f)

def _normalise_whitespace_fun(name, opts, f):
    opts.whitespace_normaliser = f

543 544 545 546 547 548 549 550 551 552 553
def normalise_version_( *pkgs ):
    def normalise_version__( str ):
        return re.sub('(' + '|'.join(map(re.escape,pkgs)) + ')-[0-9.]+',
                      '\\1-<VERSION>', str)
    return normalise_version__

def normalise_version( *pkgs ):
    def normalise_version__( name, opts ):
        _normalise_fun(name, opts, normalise_version_(*pkgs))
        _normalise_errmsg_fun(name, opts, normalise_version_(*pkgs))
    return normalise_version__
554

thomie's avatar
thomie committed
555 556 557 558
def normalise_drive_letter(name, opts):
    # Windows only. Change D:\\ to C:\\.
    _normalise_fun(name, opts, lambda str: re.sub(r'[A-Z]:\\', r'C:\\', str))

559 560 561 562 563 564 565
def keep_prof_callstacks(name, opts):
    """Keep profiling callstacks.

    Use together with `only_ways(prof_ways)`.
    """
    opts.keep_prof_callstacks = True

566 567
def join_normalisers(*a):
    """
568
    Compose functions, flattening sequences.
569

570
       join_normalisers(f1,[f2,f3],f4)
571 572 573

    is the same as

574
       lambda x: f1(f2(f3(f4(x))))
575 576
    """

577 578 579 580 581
    def flatten(l):
        """
        Taken from http://stackoverflow.com/a/2158532/946226
        """
        for el in l:
582 583
            if (isinstance(el, collections.Iterable)
                and not isinstance(el, (bytes, str))):
584 585 586 587 588 589
                for sub in flatten(el):
                    yield sub
            else:
                yield el

    a = flatten(a)
590 591 592

    fn = lambda x:x # identity function
    for f in a:
593
        assert callable(f)
594 595 596
        fn = lambda x,f=f,fn=fn: fn(f(x))
    return fn

597 598 599
# ----
# Function for composing two opt-fns together

600
def executeSetups(fs, name, opts):
601
    if type(fs) is list:
602
        # If we have a list of setups, then execute each one
603 604
        for f in fs:
            executeSetups(f, name, opts)
605 606 607
    else:
        # fs is a single function, so just apply it
        fs(name, opts)
608

609 610 611
# -----------------------------------------------------------------------------
# The current directory of tests

612 613
def newTestDir(tempdir, dir):

614
    global thisdir_settings
615
    # reset the options for this test directory
616 617 618
    def settings(name, opts, tempdir=tempdir, dir=dir):
        return _newTestDir(name, opts, tempdir, dir)
    thisdir_settings = settings
619

620 621
# Should be equal to entry in toplevel .gitignore.
testdir_suffix = '.run'
622 623 624

def _newTestDir(name, opts, tempdir, dir):
    opts.srcdir = os.path.join(os.getcwd(), dir)
625
    opts.testdir = os.path.join(tempdir, dir, name + testdir_suffix)
626
    opts.compiler_always_flags = config.compiler_always_flags
627 628 629 630

# -----------------------------------------------------------------------------
# Actually doing tests

631 632
parallelTests = []
aloneTests = []
633
allTestNames = set([])
634

635
def runTest(watcher, opts, name, func, args):
636
    if config.use_threads:
637 638 639 640 641 642
        pool_sema.acquire()
        t = threading.Thread(target=test_common_thread,
                             name=name,
                             args=(watcher, name, opts, func, args))
        t.daemon = False
        t.start()
643
    else:
644
        test_common_work(watcher, name, opts, func, args)
645

646
# name  :: String
647
# setup :: [TestOpt] -> IO ()
648
def test(name, setup, func, args):
649 650 651 652 653 654 655 656 657
    global aloneTests
    global parallelTests
    global allTestNames
    global thisdir_settings
    if name in allTestNames:
        framework_fail(name, 'duplicate', 'There are multiple tests with this name')
    if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
        framework_fail(name, 'bad_name', 'This test has an invalid name')

658 659 660 661 662
    if config.run_only_some_tests:
        if name not in config.only:
            return
        else:
            # Note [Mutating config.only]
Gabor Greif's avatar
Gabor Greif committed
663
            # config.only is initially the set of tests requested by
664 665 666 667
            # the user (via 'make TEST='). We then remove all tests that
            # we've already seen (in .T files), so that we can later
            # report on any tests we couldn't find and error out.
            config.only.remove(name)
668

669 670 671 672
    # Make a deep copy of the default_testopts, as we need our own copy
    # of any dictionaries etc inside it. Otherwise, if one test modifies
    # them, all tests will see the modified version!
    myTestOpts = copy.deepcopy(default_testopts)
673

674
    executeSetups([thisdir_settings, setup], name, myTestOpts)
675

676
    thisTest = lambda watcher: runTest(watcher, myTestOpts, name, func, args)
677 678 679 680
    if myTestOpts.alone:
        aloneTests.append(thisTest)
    else:
        parallelTests.append(thisTest)
681
    allTestNames.add(name)
682

683
if config.use_threads:
684 685 686 687 688
    def test_common_thread(watcher, name, opts, func, args):
            try:
                test_common_work(watcher, name, opts, func, args)
            finally:
                pool_sema.release()
689

690 691 692 693 694 695 696 697 698
def get_package_cache_timestamp():
    if config.package_conf_cache_file == '':
        return 0.0
    else:
        try:
            return os.stat(config.package_conf_cache_file).st_mtime
        except:
            return 0.0

Edward Z. Yang's avatar
Edward Z. Yang committed
699
do_not_copy = ('.hi', '.o', '.dyn_hi', '.dyn_o', '.out') # 12112
700

701
def test_common_work(watcher, name, opts, func, args):
702
    try:
703
        t.total_tests += 1
704 705 706 707 708 709 710
        setLocalTestOpts(opts)

        package_conf_cache_file_start_timestamp = get_package_cache_timestamp()

        # All the ways we might run this test
        if func == compile or func == multimod_compile:
            all_ways = config.compile_ways
711
        elif func == compile_and_run or func == multimod_compile_and_run:
712 713 714 715 716 717
            all_ways = config.run_ways
        elif func == ghci_script:
            if 'ghci' in config.run_ways:
                all_ways = ['ghci']
            else:
                all_ways = []
718
        else:
719 720 721
            all_ways = ['normal']

        # A test itself can request extra ways by setting opts.extra_ways
722
        all_ways = all_ways + [way for way in opts.extra_ways if way not in all_ways]
723

724
        t.total_test_cases += len(all_ways)
725 726 727

        ok_way = lambda way: \
            not getTestOpts().skip \
728
            and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
729
            and (config.cmdline_ways == [] or way in config.cmdline_ways) \
730
            and (not (config.skip_perf_tests and isStatsTest())) \
731 732 733
            and way not in getTestOpts().omit_ways

        # Which ways we are asked to skip
734
        do_ways = list(filter (ok_way,all_ways))
735

736 737
        # Only run all ways in slow mode.
        # See Note [validate and testsuite speed] in toplevel Makefile.
738 739
        if config.accept:
            # Only ever run one way
740
            do_ways = do_ways[:1]
741 742 743
        elif config.speed > 0:
            # However, if we EXPLICITLY asked for a way (with extra_ways)
            # please test it!
744 745
            explicit_ways = list(filter(lambda way: way in opts.extra_ways, do_ways))
            other_ways = list(filter(lambda way: way not in opts.extra_ways, do_ways))
746
            do_ways = other_ways[:1] + explicit_ways
747

748 749 750 751 752 753 754 755
        # Find all files in the source directory that this test
        # depends on. Do this only once for all ways.
        # Generously add all filenames that start with the name of
        # the test to this set, as a convenience to test authors.
        # They will have to use the `extra_files` setup function to
        # specify all other files that their test depends on (but
        # this seems to be necessary for only about 10% of all
        # tests).
756 757 758 759
        files = set(f for f in os.listdir(opts.srcdir)
                       if f.startswith(name) and not f == name and
                          not f.endswith(testdir_suffix) and
                          not os.path.splitext(f)[1] in do_not_copy)
760
        for filename in (opts.extra_files + extra_src_files.get(name, [])):
761
            if filename.startswith('/'):
762 763 764 765 766 767 768 769 770
                framework_fail(name, 'whole-test',
                    'no absolute paths in extra_files please: ' + filename)

            elif '*' in filename:
                # Don't use wildcards in extra_files too much, as
                # globbing is slow.
                files.update((os.path.relpath(f, opts.srcdir)
                            for f in glob.iglob(in_srcdir(filename))))

thomie's avatar
thomie committed
771
            elif filename:
772 773
                files.add(filename)

thomie's avatar
thomie committed
774 775 776
            else:
                framework_fail(name, 'whole-test', 'extra_file is empty string')

777 778 779 780
        # Run the required tests...
        for way in do_ways:
            if stopping():
                break
781 782 783 784 785 786 787
            try:
                do_test(name, way, func, args, files)
            except KeyboardInterrupt:
                stopNow()
            except Exception as e:
                framework_fail(name, way, str(e))
                traceback.print_exc()
788

789
        t.n_tests_skipped += len(set(all_ways) - set(do_ways))
790

791
        if config.cleanup and do_ways:
792 793 794 795
            try:
                cleanup()
            except Exception as e:
                framework_fail(name, 'runTest', 'Unhandled exception during cleanup: ' + str(e))
796

797
        package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
798

799 800
        if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
            framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
801

802
    except Exception as e:
803
        framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
804 805
    finally:
        watcher.notify()
Ian Lynagh's avatar
Ian Lynagh committed
806

807 808 809
def do_test(name, way, func, args, files):
    opts = getTestOpts()

810 811
    full_name = name + '(' + way + ')'

812
    if_verbose(2, "=====> {0} {1} of {2} {3}".format(
813
        full_name, t.total_tests, len(allTestNames),
814 815 816
        [len(t.unexpected_passes),
         len(t.unexpected_failures),
         len(t.framework_failures)]))
817 818 819 820

    # Clean up prior to the test, so that we can't spuriously conclude
    # that it passed on the basis of old run outputs.
    cleanup()
821
    os.makedirs(opts.testdir)
822 823 824 825 826 827 828 829 830

    # Link all source files for this test into a new directory in
    # /tmp, and run the test in that directory. This makes it
    # possible to run tests in parallel, without modification, that
    # would otherwise (accidentally) write to the same output file.
    # It also makes it easier to keep the testsuite clean.

    for extra_file in files:
        src = in_srcdir(extra_file)
831
        dst = in_testdir(os.path.basename(extra_file.rstrip('/\\')))
832
        if os.path.isfile(src):
833
            link_or_copy_file(src, dst)
834
        elif os.path.isdir(src):
835
            os.mkdir(dst)
836 837
            lndir(src, dst)
        else:
838
            if not config.haddock and os.path.splitext(extra_file)[1] == '.t':
839 840 841 842 843 844 845 846 847 848 849 850 851 852
                # When using a ghc built without haddock support, .t
                # files are rightfully missing. Don't
                # framework_fail. Test will be skipped later.
                pass
            else:
                framework_fail(name, way,
                    'extra_file does not exist: ' + extra_file)

    if func.__name__ == 'run_command' or opts.pre_cmd:
        # When running 'MAKE' make sure 'TOP' still points to the
        # root of the testsuite.
        src_makefile = in_srcdir('Makefile')
        dst_makefile = in_testdir('Makefile')
        if os.path.exists(src_makefile):
853
            with io.open(src_makefile, 'r', encoding='utf8') as src:
854
                makefile = re.sub('TOP=.*', 'TOP=' + config.top, src.read(), 1)
855
                with io.open(dst_makefile, 'w', encoding='utf8') as dst:
856
                    dst.write(makefile)
857

858
    if opts.pre_cmd:
859 860 861 862
        exit_code = runCmd('cd "{0}" && {1}'.format(opts.testdir, override_options(opts.pre_cmd)),
                           stderr = subprocess.STDOUT,
                           print_output = config.verbose >= 3)

863 864
        if exit_code != 0:
            framework_fail(name, way, 'pre_cmd failed: {0}'.format(exit_code))