testlib.py 63.9 KB
Newer Older
1
# coding=utf8
2
#
3 4 5
# (c) Simon Marlow 2002
#

6
from __future__ import print_function
7

8
import io
9
import shutil
10
import os
11
import errno
12 13 14
import string
import re
import traceback
15 16
import time
import datetime
17
import copy
18
import glob
ian@well-typed.com's avatar
ian@well-typed.com committed
19
from math import ceil, trunc
20
import collections
21
import subprocess
22

23
from testglobals import *
24
from testutil import *
25
from extra_files import extra_src_files
26

27 28 29 30 31
try:
    basestring
except: # Python 3
    basestring = (str,bytes)

32 33
if config.use_threads:
    import threading
34 35 36 37
    try:
        import thread
    except ImportError: # Python 3
        import _thread as thread
38

39 40 41 42 43 44 45 46
global wantToStop
wantToStop = False
def stopNow():
    global wantToStop
    wantToStop = True
def stopping():
    return wantToStop

47 48
# Options valid for the current test only (these get reset to
# testdir_testopts after each test).
49

ei@vuokko.info's avatar
ei@vuokko.info committed
50
global testopts_local
51 52 53 54 55 56
if config.use_threads:
    testopts_local = threading.local()
else:
    class TestOpts_Local:
        pass
    testopts_local = TestOpts_Local()
57 58

def getTestOpts():
ei@vuokko.info's avatar
ei@vuokko.info committed
59
    return testopts_local.x
60

ei@vuokko.info's avatar
ei@vuokko.info committed
61 62 63
def setLocalTestOpts(opts):
    global testopts_local
    testopts_local.x=opts
64

65 66
def isStatsTest():
    opts = getTestOpts()
67
    return bool(opts.compiler_stats_range_fields or opts.stats_range_fields)
68 69


70 71 72
# This can be called at the top of a file of tests, to set default test options
# for the following tests.
def setTestOpts( f ):
73
    global thisdir_settings
74
    thisdir_settings = [thisdir_settings, f]
75 76 77 78 79 80 81 82 83 84 85 86

# -----------------------------------------------------------------------------
# Canned setup functions for common cases.  eg. for a test you might say
#
#      test('test001', normal, compile, [''])
#
# to run it without any options, but change it to
#
#      test('test001', expect_fail, compile, [''])
#
# to expect failure for this test.

87
def normal( name, opts ):
88 89
    return;

90
def skip( name, opts ):
91 92
    opts.skip = 1

93
def expect_fail( name, opts ):
94 95 96
    # The compiler, testdriver, OS or platform is missing a certain
    # feature, and we don't plan to or can't fix it now or in the
    # future.
97 98
    opts.expect = 'fail';

99
def reqlib( lib ):
100
    return lambda name, opts, l=lib: _reqlib (name, opts, l )
101

102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131
def stage1(name, opts):
    # See Note [Why is there no stage1 setup function?]
    framework_fail(name, 'stage1 setup function does not exist',
                   'add your test to testsuite/tests/stage1 instead')

# Note [Why is there no stage1 setup function?]
#
# Presumably a stage1 setup function would signal that the stage1
# compiler should be used to compile a test.
#
# Trouble is, the path to the compiler + the `ghc --info` settings for
# that compiler are currently passed in from the `make` part of the
# testsuite driver.
#
# Switching compilers in the Python part would be entirely too late, as
# all ghc_with_* settings would be wrong. See config/ghc for possible
# consequences (for example, config.run_ways would still be
# based on the default compiler, quite likely causing ./validate --slow
# to fail).
#
# It would be possible to let the Python part of the testsuite driver
# make the call to `ghc --info`, but doing so would require quite some
# work. Care has to be taken to not affect the run_command tests for
# example, as they also use the `ghc --info` settings:
#     quasiquotation/qq007/Makefile:ifeq "$(GhcDynamic)" "YES"
#
# If you want a test to run using the stage1 compiler, add it to the
# testsuite/tests/stage1 directory. Validate runs the tests in that
# directory with `make stage=1`.

132 133 134 135
# Cache the results of looking to see if we have a library or not.
# This makes quite a difference, especially on Windows.
have_lib = {}

136
def _reqlib( name, opts, lib ):
137
    if lib in have_lib:
138
        got_it = have_lib[lib]
139
    else:
140 141 142 143 144 145 146 147
        cmd = strip_quotes(config.ghc_pkg)
        p = subprocess.Popen([cmd, '--no-user-package-db', 'describe', lib],
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
        # read from stdout and stderr to avoid blocking due to
        # buffers filling
        p.communicate()
        r = p.wait()
148 149 150 151
        got_it = r == 0
        have_lib[lib] = got_it

    if not got_it:
152
        opts.expect = 'missing-lib'
153

154 155 156 157
def req_haddock( name, opts ):
    if not config.haddock:
        opts.expect = 'missing-lib'

158
def req_profiling( name, opts ):
159
    '''Require the profiling libraries (add 'GhcLibWays += p' to mk/build.mk)'''
160 161 162
    if not config.have_profiling:
        opts.expect = 'fail'

163
def req_shared_libs( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
164 165 166
    if not config.have_shared_libs:
        opts.expect = 'fail'

167
def req_interp( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
168 169 170
    if not config.have_interp:
        opts.expect = 'fail'

171
def req_smp( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
172 173 174
    if not config.have_smp:
        opts.expect = 'fail'

175
def ignore_output( name, opts ):
176 177
    opts.ignore_output = 1

178
def combined_output( name, opts ):
pcapriotti's avatar
pcapriotti committed
179 180
    opts.combined_output = True

181 182 183
# -----

def expect_fail_for( ways ):
184
    return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
185

186
def _expect_fail_for( name, opts, ways ):
187 188
    opts.expect_fail_for = ways

189
def expect_broken( bug ):
190 191
    # This test is a expected not to work due to the indicated trac bug
    # number.
192 193 194
    return lambda name, opts, b=bug: _expect_broken (name, opts, b )

def _expect_broken( name, opts, bug ):
195
    record_broken(name, opts, bug)
196 197
    opts.expect = 'fail';

Ian Lynagh's avatar
Ian Lynagh committed
198
def expect_broken_for( bug, ways ):
199
    return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
Ian Lynagh's avatar
Ian Lynagh committed
200

201
def _expect_broken_for( name, opts, bug, ways ):
202
    record_broken(name, opts, bug)
Ian Lynagh's avatar
Ian Lynagh committed
203
    opts.expect_fail_for = ways
Ian Lynagh's avatar
Ian Lynagh committed
204

205
def record_broken(name, opts, bug):
206
    global brokens
207
    me = (bug, opts.testdir, name)
208 209 210
    if not me in brokens:
        brokens.append(me)

211 212 213 214 215
def _expect_pass(way):
    # Helper function. Not intended for use in .T files.
    opts = getTestOpts()
    return opts.expect == 'pass' and way not in opts.expect_fail_for

216 217 218
# -----

def omit_ways( ways ):
219
    return lambda name, opts, w=ways: _omit_ways( name, opts, w )
220

221
def _omit_ways( name, opts, ways ):
222 223 224 225
    opts.omit_ways = ways

# -----

226
def only_ways( ways ):
227
    return lambda name, opts, w=ways: _only_ways( name, opts, w )
228

229
def _only_ways( name, opts, ways ):
230 231 232 233
    opts.only_ways = ways

# -----

234
def extra_ways( ways ):
235
    return lambda name, opts, w=ways: _extra_ways( name, opts, w )
236

237
def _extra_ways( name, opts, ways ):
238 239 240 241
    opts.extra_ways = ways

# -----

242
def set_stdin( file ):
243
   return lambda name, opts, f=file: _set_stdin(name, opts, f);
244

245
def _set_stdin( name, opts, f ):
246 247 248 249 250
   opts.stdin = f

# -----

def exit_code( val ):
251
    return lambda name, opts, v=val: _exit_code(name, opts, v);
252

253
def _exit_code( name, opts, v ):
254 255
    opts.exit_code = v

256 257 258 259 260 261 262 263 264 265 266
def signal_exit_code( val ):
    if opsys('solaris2'):
        return exit_code( val );
    else:
        # When application running on Linux receives fatal error
        # signal, then its exit code is encoded as 128 + signal
        # value. See http://www.tldp.org/LDP/abs/html/exitcodes.html
        # I assume that Mac OS X behaves in the same way at least Mac
        # OS X builder behavior suggests this.
        return exit_code( val+128 );

267 268
# -----

269 270
def compile_timeout_multiplier( val ):
    return lambda name, opts, v=val: _compile_timeout_multiplier(name, opts, v)
271

272 273 274 275 276 277 278 279
def _compile_timeout_multiplier( name, opts, v ):
    opts.compile_timeout_multiplier = v

def run_timeout_multiplier( val ):
    return lambda name, opts, v=val: _run_timeout_multiplier(name, opts, v)

def _run_timeout_multiplier( name, opts, v ):
    opts.run_timeout_multiplier = v
280 281 282

# -----

283
def extra_run_opts( val ):
284
    return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
285

286
def _extra_run_opts( name, opts, v ):
287 288
    opts.extra_run_opts = v

289 290
# -----

Simon Marlow's avatar
Simon Marlow committed
291
def extra_hc_opts( val ):
292
    return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
Simon Marlow's avatar
Simon Marlow committed
293

294
def _extra_hc_opts( name, opts, v ):
Simon Marlow's avatar
Simon Marlow committed
295 296 297 298
    opts.extra_hc_opts = v

# -----

299
def extra_clean( files ):
300 301
    # TODO. Remove all calls to extra_clean.
    return lambda _name, _opts: None
302

303 304 305 306 307 308
def extra_files(files):
    return lambda name, opts: _extra_files(name, opts, files)

def _extra_files(name, opts, files):
    opts.extra_files.extend(files)

309 310
# -----

311 312 313 314 315 316 317
def stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);

def _stats_num_field( name, opts, field, expecteds ):
    if field in opts.stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

318
    if type(expecteds) is list:
319 320 321 322 323
        for (b, expected, dev) in expecteds:
            if b:
                opts.stats_range_fields[field] = (expected, dev)
                return
        framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
324

325 326 327
    else:
        (expected, dev) = expecteds
        opts.stats_range_fields[field] = (expected, dev)
328 329 330

def compiler_stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
331

332 333 334 335
def _compiler_stats_num_field( name, opts, field, expecteds ):
    if field in opts.compiler_stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

336 337
    # Compiler performance numbers change when debugging is on, making the results
    # useless and confusing. Therefore, skip if debugging is on.
338 339 340
    if compiler_debugged():
        skip(name, opts)

341 342 343 344
    for (b, expected, dev) in expecteds:
        if b:
            opts.compiler_stats_range_fields[field] = (expected, dev)
            return
345

346 347
    framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')

348 349
# -----

350
def when(b, f):
ian@well-typed.com's avatar
ian@well-typed.com committed
351 352 353
    # When list_brokens is on, we want to see all expect_broken calls,
    # so we always do f
    if b or config.list_broken:
354 355 356 357 358 359 360
        return f
    else:
        return normal

def unless(b, f):
    return when(not b, f)

ian@well-typed.com's avatar
ian@well-typed.com committed
361 362 363
def doing_ghci():
    return 'ghci' in config.run_ways

364
def ghc_dynamic():
ian@well-typed.com's avatar
ian@well-typed.com committed
365
    return config.ghc_dynamic
366 367

def fast():
368
    return config.speed == 2
369

370 371
def platform( plat ):
    return config.platform == plat
Ian Lynagh's avatar
Ian Lynagh committed
372

373 374
def opsys( os ):
    return config.os == os
Ian Lynagh's avatar
Ian Lynagh committed
375

376 377
def arch( arch ):
    return config.arch == arch
378

379 380
def wordsize( ws ):
    return config.wordsize == str(ws)
tibbe's avatar
tibbe committed
381

382 383
def msys( ):
    return config.msys
ian@well-typed.com's avatar
ian@well-typed.com committed
384

385 386
def cygwin( ):
    return config.cygwin
Ian Lynagh's avatar
Ian Lynagh committed
387

388 389
def have_vanilla( ):
    return config.have_vanilla
390

391 392
def have_dynamic( ):
    return config.have_dynamic
393

394 395
def have_profiling( ):
    return config.have_profiling
396

397 398
def in_tree_compiler( ):
    return config.in_tree_compiler
399

400 401 402 403 404 405
def unregisterised( ):
    return config.unregisterised

def compiler_profiled( ):
    return config.compiler_profiled

406 407
def compiler_debugged( ):
    return config.compiler_debugged
408

409 410
# ---

411
def high_memory_usage(name, opts):
412 413
    opts.alone = True

414 415 416 417 418
# If a test is for a multi-CPU race, then running the test alone
# increases the chance that we'll actually see it.
def multi_cpu_race(name, opts):
    opts.alone = True

Ian Lynagh's avatar
Ian Lynagh committed
419
# ---
420
def literate( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
421 422
    opts.literate = 1;

423
def c_src( name, opts ):
424 425
    opts.c_src = 1;

426
def objc_src( name, opts ):
Austin Seipp's avatar
Austin Seipp committed
427 428
    opts.objc_src = 1;

429
def objcpp_src( name, opts ):
430 431
    opts.objcpp_src = 1;

432
def cmm_src( name, opts ):
433 434
    opts.cmm_src = 1;

435
def outputdir( odir ):
436
    return lambda name, opts, d=odir: _outputdir(name, opts, d)
437

438
def _outputdir( name, opts, odir ):
439 440
    opts.outputdir = odir;

441 442
# ----

443
def pre_cmd( cmd ):
444
    return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
445

446
def _pre_cmd( name, opts, cmd ):
447 448 449 450
    opts.pre_cmd = cmd

# ----

451
def clean_cmd( cmd ):
452 453
    # TODO. Remove all calls to clean_cmd.
    return lambda _name, _opts: None
454 455 456

# ----

457
def cmd_prefix( prefix ):
458
    return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
459

460
def _cmd_prefix( name, opts, prefix ):
461 462 463 464 465
    opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;

# ----

def cmd_wrapper( fun ):
466
    return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
467

468
def _cmd_wrapper( name, opts, fun ):
469
    opts.cmd_wrapper = fun
470

471 472
# ----

Ian Lynagh's avatar
Ian Lynagh committed
473
def compile_cmd_prefix( prefix ):
474
    return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
Ian Lynagh's avatar
Ian Lynagh committed
475

476
def _compile_cmd_prefix( name, opts, prefix ):
Ian Lynagh's avatar
Ian Lynagh committed
477 478 479 480
    opts.compile_cmd_prefix = prefix

# ----

481 482 483 484 485 486 487 488
def check_stdout( f ):
    return lambda name, opts, f=f: _check_stdout(name, opts, f)

def _check_stdout( name, opts, f ):
    opts.check_stdout = f

# ----

489
def normalise_slashes( name, opts ):
490
    _normalise_fun(name, opts, normalise_slashes_)
491

492
def normalise_exe( name, opts ):
493
    _normalise_fun(name, opts, normalise_exe_)
494

495 496
def normalise_fun( *fs ):
    return lambda name, opts: _normalise_fun(name, opts, fs)
497

498
def _normalise_fun( name, opts, *fs ):
499
    opts.extra_normaliser = join_normalisers(opts.extra_normaliser, fs)
500

501 502
def normalise_errmsg_fun( *fs ):
    return lambda name, opts: _normalise_errmsg_fun(name, opts, fs)
503

504
def _normalise_errmsg_fun( name, opts, *fs ):
505 506 507 508 509 510 511 512 513 514 515 516 517
    opts.extra_errmsg_normaliser =  join_normalisers(opts.extra_errmsg_normaliser, fs)

def normalise_version_( *pkgs ):
    def normalise_version__( str ):
        return re.sub('(' + '|'.join(map(re.escape,pkgs)) + ')-[0-9.]+',
                      '\\1-<VERSION>', str)
    return normalise_version__

def normalise_version( *pkgs ):
    def normalise_version__( name, opts ):
        _normalise_fun(name, opts, normalise_version_(*pkgs))
        _normalise_errmsg_fun(name, opts, normalise_version_(*pkgs))
    return normalise_version__
518

thomie's avatar
thomie committed
519 520 521 522
def normalise_drive_letter(name, opts):
    # Windows only. Change D:\\ to C:\\.
    _normalise_fun(name, opts, lambda str: re.sub(r'[A-Z]:\\', r'C:\\', str))

523 524 525 526 527 528 529
def keep_prof_callstacks(name, opts):
    """Keep profiling callstacks.

    Use together with `only_ways(prof_ways)`.
    """
    opts.keep_prof_callstacks = True

530 531
def join_normalisers(*a):
    """
532
    Compose functions, flattening sequences.
533

534
       join_normalisers(f1,[f2,f3],f4)
535 536 537

    is the same as

538
       lambda x: f1(f2(f3(f4(x))))
539 540
    """

541 542 543 544 545 546 547 548 549 550 551 552
    def flatten(l):
        """
        Taken from http://stackoverflow.com/a/2158532/946226
        """
        for el in l:
            if isinstance(el, collections.Iterable) and not isinstance(el, basestring):
                for sub in flatten(el):
                    yield sub
            else:
                yield el

    a = flatten(a)
553 554 555

    fn = lambda x:x # identity function
    for f in a:
556
        assert callable(f)
557 558 559
        fn = lambda x,f=f,fn=fn: fn(f(x))
    return fn

560 561 562
# ----
# Function for composing two opt-fns together

563
def executeSetups(fs, name, opts):
564
    if type(fs) is list:
565
        # If we have a list of setups, then execute each one
566 567
        for f in fs:
            executeSetups(f, name, opts)
568 569 570
    else:
        # fs is a single function, so just apply it
        fs(name, opts)
571

572 573 574
# -----------------------------------------------------------------------------
# The current directory of tests

575 576
def newTestDir(tempdir, dir):

577
    global thisdir_settings
578
    # reset the options for this test directory
579 580 581
    def settings(name, opts, tempdir=tempdir, dir=dir):
        return _newTestDir(name, opts, tempdir, dir)
    thisdir_settings = settings
582

583 584
# Should be equal to entry in toplevel .gitignore.
testdir_suffix = '.run'
585 586 587

def _newTestDir(name, opts, tempdir, dir):
    opts.srcdir = os.path.join(os.getcwd(), dir)
588
    opts.testdir = os.path.join(tempdir, dir, name + testdir_suffix)
589
    opts.compiler_always_flags = config.compiler_always_flags
590 591 592 593

# -----------------------------------------------------------------------------
# Actually doing tests

594 595
parallelTests = []
aloneTests = []
596
allTestNames = set([])
597

598
def runTest (opts, name, func, args):
ei@vuokko.info's avatar
ei@vuokko.info committed
599 600
    ok = 0

601
    if config.use_threads:
ei@vuokko.info's avatar
ei@vuokko.info committed
602
        t.thread_pool.acquire()
603
        try:
604
            while config.threads<(t.running_threads+1):
605
                t.thread_pool.wait()
606
            t.running_threads = t.running_threads+1
607 608
            ok=1
            t.thread_pool.release()
609
            thread.start_new_thread(test_common_thread, (name, opts, func, args))
610 611 612 613 614
        except:
            if not ok:
                t.thread_pool.release()
    else:
        test_common_work (name, opts, func, args)
615

616
# name  :: String
617
# setup :: TestOpts -> IO ()
618
def test (name, setup, func, args):
619 620 621 622 623 624 625 626 627
    global aloneTests
    global parallelTests
    global allTestNames
    global thisdir_settings
    if name in allTestNames:
        framework_fail(name, 'duplicate', 'There are multiple tests with this name')
    if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
        framework_fail(name, 'bad_name', 'This test has an invalid name')

628 629 630 631 632 633 634 635 636 637
    if config.run_only_some_tests:
        if name not in config.only:
            return
        else:
            # Note [Mutating config.only]
            # config.only is initiallly the set of tests requested by
            # the user (via 'make TEST='). We then remove all tests that
            # we've already seen (in .T files), so that we can later
            # report on any tests we couldn't find and error out.
            config.only.remove(name)
638

639 640 641 642
    # Make a deep copy of the default_testopts, as we need our own copy
    # of any dictionaries etc inside it. Otherwise, if one test modifies
    # them, all tests will see the modified version!
    myTestOpts = copy.deepcopy(default_testopts)
643

644
    executeSetups([thisdir_settings, setup], name, myTestOpts)
645 646 647 648 649 650

    thisTest = lambda : runTest(myTestOpts, name, func, args)
    if myTestOpts.alone:
        aloneTests.append(thisTest)
    else:
        parallelTests.append(thisTest)
651
    allTestNames.add(name)
652

653
if config.use_threads:
654
    def test_common_thread(name, opts, func, args):
655 656 657 658 659 660
        t.lock.acquire()
        try:
            test_common_work(name,opts,func,args)
        finally:
            t.lock.release()
            t.thread_pool.acquire()
661
            t.running_threads = t.running_threads - 1
662 663
            t.thread_pool.notify()
            t.thread_pool.release()
664

665 666 667 668 669 670 671 672 673
def get_package_cache_timestamp():
    if config.package_conf_cache_file == '':
        return 0.0
    else:
        try:
            return os.stat(config.package_conf_cache_file).st_mtime
        except:
            return 0.0

674 675
do_not_copy = ('.hi', '.o', '.dyn_hi', '.dyn_o') # 12112

ei@vuokko.info's avatar
ei@vuokko.info committed
676
def test_common_work (name, opts, func, args):
677
    try:
678
        t.total_tests += 1
679 680 681 682 683 684 685
        setLocalTestOpts(opts)

        package_conf_cache_file_start_timestamp = get_package_cache_timestamp()

        # All the ways we might run this test
        if func == compile or func == multimod_compile:
            all_ways = config.compile_ways
686
        elif func == compile_and_run or func == multimod_compile_and_run:
687 688 689 690 691 692
            all_ways = config.run_ways
        elif func == ghci_script:
            if 'ghci' in config.run_ways:
                all_ways = ['ghci']
            else:
                all_ways = []
693
        else:
694 695 696
            all_ways = ['normal']

        # A test itself can request extra ways by setting opts.extra_ways
697
        all_ways = all_ways + [way for way in opts.extra_ways if way not in all_ways]
698

699
        t.total_test_cases += len(all_ways)
700 701 702

        ok_way = lambda way: \
            not getTestOpts().skip \
703
            and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
704
            and (config.cmdline_ways == [] or way in config.cmdline_ways) \
705
            and (not (config.skip_perf_tests and isStatsTest())) \
706 707 708
            and way not in getTestOpts().omit_ways

        # Which ways we are asked to skip
709
        do_ways = list(filter (ok_way,all_ways))
710

711 712
        # Only run all ways in slow mode.
        # See Note [validate and testsuite speed] in toplevel Makefile.
713 714
        if config.accept:
            # Only ever run one way
715
            do_ways = do_ways[:1]
716 717 718
        elif config.speed > 0:
            # However, if we EXPLICITLY asked for a way (with extra_ways)
            # please test it!
719 720
            explicit_ways = list(filter(lambda way: way in opts.extra_ways, do_ways))
            other_ways = list(filter(lambda way: way not in opts.extra_ways, do_ways))
721
            do_ways = other_ways[:1] + explicit_ways
722

723 724 725 726 727 728 729 730
        # Find all files in the source directory that this test
        # depends on. Do this only once for all ways.
        # Generously add all filenames that start with the name of
        # the test to this set, as a convenience to test authors.
        # They will have to use the `extra_files` setup function to
        # specify all other files that their test depends on (but
        # this seems to be necessary for only about 10% of all
        # tests).
731 732 733 734
        files = set(f for f in os.listdir(opts.srcdir)
                       if f.startswith(name) and not f == name and
                          not f.endswith(testdir_suffix) and
                          not os.path.splitext(f)[1] in do_not_copy)
735
        for filename in (opts.extra_files + extra_src_files.get(name, [])):
736
            if filename.startswith('/'):
737 738 739 740 741 742 743 744 745
                framework_fail(name, 'whole-test',
                    'no absolute paths in extra_files please: ' + filename)

            elif '*' in filename:
                # Don't use wildcards in extra_files too much, as
                # globbing is slow.
                files.update((os.path.relpath(f, opts.srcdir)
                            for f in glob.iglob(in_srcdir(filename))))

thomie's avatar
thomie committed
746
            elif filename:
747 748
                files.add(filename)

thomie's avatar
thomie committed
749 750 751
            else:
                framework_fail(name, 'whole-test', 'extra_file is empty string')

752 753 754 755
        # Run the required tests...
        for way in do_ways:
            if stopping():
                break
756 757 758 759 760 761 762
            try:
                do_test(name, way, func, args, files)
            except KeyboardInterrupt:
                stopNow()
            except Exception as e:
                framework_fail(name, way, str(e))
                traceback.print_exc()
763

764
        t.n_tests_skipped += len(set(all_ways) - set(do_ways))
765

766
        if config.cleanup and do_ways:
767
            cleanup()
768

769
        package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
770

771 772
        if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
            framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
773

774
    except Exception as e:
775
        framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
Ian Lynagh's avatar
Ian Lynagh committed
776

777 778 779
def do_test(name, way, func, args, files):
    opts = getTestOpts()

780 781
    full_name = name + '(' + way + ')'

782
    if_verbose(2, "=====> {0} {1} of {2} {3}".format(
783 784 785 786
        full_name, t.total_tests, len(allTestNames), 
        [len(t.unexpected_passes),
         len(t.unexpected_failures),
         len(t.framework_failures)]))
787 788 789 790

    # Clean up prior to the test, so that we can't spuriously conclude
    # that it passed on the basis of old run outputs.
    cleanup()
791
    os.makedirs(opts.testdir)
792 793 794 795 796 797 798 799 800

    # Link all source files for this test into a new directory in
    # /tmp, and run the test in that directory. This makes it
    # possible to run tests in parallel, without modification, that
    # would otherwise (accidentally) write to the same output file.
    # It also makes it easier to keep the testsuite clean.

    for extra_file in files:
        src = in_srcdir(extra_file)
801
        dst = in_testdir(os.path.basename(extra_file.rstrip('/\\')))
802
        if os.path.isfile(src):
803
            link_or_copy_file(src, dst)
804
        elif os.path.isdir(src):
805
            os.mkdir(dst)
806 807
            lndir(src, dst)
        else:
808
            if not config.haddock and os.path.splitext(extra_file)[1] == '.t':
809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826
                # When using a ghc built without haddock support, .t
                # files are rightfully missing. Don't
                # framework_fail. Test will be skipped later.
                pass
            else:
                framework_fail(name, way,
                    'extra_file does not exist: ' + extra_file)

    if func.__name__ == 'run_command' or opts.pre_cmd:
        # When running 'MAKE' make sure 'TOP' still points to the
        # root of the testsuite.
        src_makefile = in_srcdir('Makefile')
        dst_makefile = in_testdir('Makefile')
        if os.path.exists(src_makefile):
            with open(src_makefile, 'r') as src:
                makefile = re.sub('TOP=.*', 'TOP=' + config.top, src.read(), 1)
                with open(dst_makefile, 'w') as dst:
                    dst.write(makefile)
827

828 829
    if config.use_threads:
        t.lock.release()
830

831 832 833 834
    if opts.pre_cmd:
        exit_code = runCmd('cd "{0}" && {1}'.format(opts.testdir, opts.pre_cmd))
        if exit_code != 0:
            framework_fail(name, way, 'pre_cmd failed: {0}'.format(exit_code))
835

836 837 838 839 840
    try:
        result = func(*[name,way] + args)
    finally:
        if config.use_threads:
            t.lock.acquire()