testlib.py 66.5 KB
Newer Older
1
# coding=utf8
2
#
3 4 5
# (c) Simon Marlow 2002
#

6
from __future__ import print_function
7

8
import io
9
import shutil
10
import os
11
import errno
12 13 14
import string
import re
import traceback
15 16
import time
import datetime
17
import copy
18
import glob
ian@well-typed.com's avatar
ian@well-typed.com committed
19
from math import ceil, trunc
20
import collections
21
import subprocess
22

23
from testglobals import *
24
from testutil import *
25
from extra_files import extra_src_files
26

27 28 29 30 31
try:
    basestring
except: # Python 3
    basestring = (str,bytes)

32 33
if config.use_threads:
    import threading
34 35 36 37
    try:
        import thread
    except ImportError: # Python 3
        import _thread as thread
38

39 40
global wantToStop
wantToStop = False
41 42 43 44 45

global pool_sema
if config.use_threads:
    pool_sema = threading.BoundedSemaphore(value=config.threads)

46 47 48 49 50 51
def stopNow():
    global wantToStop
    wantToStop = True
def stopping():
    return wantToStop

52 53
# Options valid for the current test only (these get reset to
# testdir_testopts after each test).
54

ei@vuokko.info's avatar
ei@vuokko.info committed
55
global testopts_local
56 57 58 59 60 61
if config.use_threads:
    testopts_local = threading.local()
else:
    class TestOpts_Local:
        pass
    testopts_local = TestOpts_Local()
62 63

def getTestOpts():
ei@vuokko.info's avatar
ei@vuokko.info committed
64
    return testopts_local.x
65

ei@vuokko.info's avatar
ei@vuokko.info committed
66 67 68
def setLocalTestOpts(opts):
    global testopts_local
    testopts_local.x=opts
69

70 71
def isStatsTest():
    opts = getTestOpts()
72
    return bool(opts.compiler_stats_range_fields or opts.stats_range_fields)
73 74


75 76 77
# This can be called at the top of a file of tests, to set default test options
# for the following tests.
def setTestOpts( f ):
78
    global thisdir_settings
79
    thisdir_settings = [thisdir_settings, f]
80 81 82 83 84 85 86 87 88 89 90 91

# -----------------------------------------------------------------------------
# Canned setup functions for common cases.  eg. for a test you might say
#
#      test('test001', normal, compile, [''])
#
# to run it without any options, but change it to
#
#      test('test001', expect_fail, compile, [''])
#
# to expect failure for this test.

92
def normal( name, opts ):
93 94
    return;

95
def skip( name, opts ):
96 97
    opts.skip = 1

98
def expect_fail( name, opts ):
99 100 101
    # The compiler, testdriver, OS or platform is missing a certain
    # feature, and we don't plan to or can't fix it now or in the
    # future.
102 103
    opts.expect = 'fail';

104
def reqlib( lib ):
105
    return lambda name, opts, l=lib: _reqlib (name, opts, l )
106

107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136
def stage1(name, opts):
    # See Note [Why is there no stage1 setup function?]
    framework_fail(name, 'stage1 setup function does not exist',
                   'add your test to testsuite/tests/stage1 instead')

# Note [Why is there no stage1 setup function?]
#
# Presumably a stage1 setup function would signal that the stage1
# compiler should be used to compile a test.
#
# Trouble is, the path to the compiler + the `ghc --info` settings for
# that compiler are currently passed in from the `make` part of the
# testsuite driver.
#
# Switching compilers in the Python part would be entirely too late, as
# all ghc_with_* settings would be wrong. See config/ghc for possible
# consequences (for example, config.run_ways would still be
# based on the default compiler, quite likely causing ./validate --slow
# to fail).
#
# It would be possible to let the Python part of the testsuite driver
# make the call to `ghc --info`, but doing so would require quite some
# work. Care has to be taken to not affect the run_command tests for
# example, as they also use the `ghc --info` settings:
#     quasiquotation/qq007/Makefile:ifeq "$(GhcDynamic)" "YES"
#
# If you want a test to run using the stage1 compiler, add it to the
# testsuite/tests/stage1 directory. Validate runs the tests in that
# directory with `make stage=1`.

137 138 139 140
# Cache the results of looking to see if we have a library or not.
# This makes quite a difference, especially on Windows.
have_lib = {}

141
def _reqlib( name, opts, lib ):
142
    if lib in have_lib:
143
        got_it = have_lib[lib]
144
    else:
145 146 147 148 149 150 151 152
        cmd = strip_quotes(config.ghc_pkg)
        p = subprocess.Popen([cmd, '--no-user-package-db', 'describe', lib],
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
        # read from stdout and stderr to avoid blocking due to
        # buffers filling
        p.communicate()
        r = p.wait()
153 154 155 156
        got_it = r == 0
        have_lib[lib] = got_it

    if not got_it:
157
        opts.expect = 'missing-lib'
158

159 160 161 162
def req_haddock( name, opts ):
    if not config.haddock:
        opts.expect = 'missing-lib'

163
def req_profiling( name, opts ):
164
    '''Require the profiling libraries (add 'GhcLibWays += p' to mk/build.mk)'''
165 166 167
    if not config.have_profiling:
        opts.expect = 'fail'

168
def req_shared_libs( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
169 170 171
    if not config.have_shared_libs:
        opts.expect = 'fail'

172
def req_interp( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
173 174 175
    if not config.have_interp:
        opts.expect = 'fail'

176
def req_smp( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
177 178 179
    if not config.have_smp:
        opts.expect = 'fail'

180 181 182 183 184
def ignore_stdout(name, opts):
    opts.ignore_stdout = True

def ignore_stderr(name, opts):
    opts.ignore_stderr = True
185

186
def combined_output( name, opts ):
pcapriotti's avatar
pcapriotti committed
187 188
    opts.combined_output = True

189 190 191
# -----

def expect_fail_for( ways ):
192
    return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
193

194
def _expect_fail_for( name, opts, ways ):
195 196
    opts.expect_fail_for = ways

197
def expect_broken( bug ):
198 199
    # This test is a expected not to work due to the indicated trac bug
    # number.
200 201 202
    return lambda name, opts, b=bug: _expect_broken (name, opts, b )

def _expect_broken( name, opts, bug ):
203
    record_broken(name, opts, bug)
204 205
    opts.expect = 'fail';

Ian Lynagh's avatar
Ian Lynagh committed
206
def expect_broken_for( bug, ways ):
207
    return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
Ian Lynagh's avatar
Ian Lynagh committed
208

209
def _expect_broken_for( name, opts, bug, ways ):
210
    record_broken(name, opts, bug)
Ian Lynagh's avatar
Ian Lynagh committed
211
    opts.expect_fail_for = ways
Ian Lynagh's avatar
Ian Lynagh committed
212

213
def record_broken(name, opts, bug):
214
    global brokens
215
    me = (bug, opts.testdir, name)
216 217 218
    if not me in brokens:
        brokens.append(me)

219 220 221 222 223
def _expect_pass(way):
    # Helper function. Not intended for use in .T files.
    opts = getTestOpts()
    return opts.expect == 'pass' and way not in opts.expect_fail_for

224 225 226
# -----

def omit_ways( ways ):
227
    return lambda name, opts, w=ways: _omit_ways( name, opts, w )
228

229
def _omit_ways( name, opts, ways ):
230 231 232 233
    opts.omit_ways = ways

# -----

234
def only_ways( ways ):
235
    return lambda name, opts, w=ways: _only_ways( name, opts, w )
236

237
def _only_ways( name, opts, ways ):
238 239 240 241
    opts.only_ways = ways

# -----

242
def extra_ways( ways ):
243
    return lambda name, opts, w=ways: _extra_ways( name, opts, w )
244

245
def _extra_ways( name, opts, ways ):
246 247 248 249
    opts.extra_ways = ways

# -----

250
def set_stdin( file ):
251
   return lambda name, opts, f=file: _set_stdin(name, opts, f);
252

253
def _set_stdin( name, opts, f ):
254 255 256 257 258
   opts.stdin = f

# -----

def exit_code( val ):
259
    return lambda name, opts, v=val: _exit_code(name, opts, v);
260

261
def _exit_code( name, opts, v ):
262 263
    opts.exit_code = v

264 265 266 267 268 269 270 271 272 273 274
def signal_exit_code( val ):
    if opsys('solaris2'):
        return exit_code( val );
    else:
        # When application running on Linux receives fatal error
        # signal, then its exit code is encoded as 128 + signal
        # value. See http://www.tldp.org/LDP/abs/html/exitcodes.html
        # I assume that Mac OS X behaves in the same way at least Mac
        # OS X builder behavior suggests this.
        return exit_code( val+128 );

275 276
# -----

277 278
def compile_timeout_multiplier( val ):
    return lambda name, opts, v=val: _compile_timeout_multiplier(name, opts, v)
279

280 281 282 283 284 285 286 287
def _compile_timeout_multiplier( name, opts, v ):
    opts.compile_timeout_multiplier = v

def run_timeout_multiplier( val ):
    return lambda name, opts, v=val: _run_timeout_multiplier(name, opts, v)

def _run_timeout_multiplier( name, opts, v ):
    opts.run_timeout_multiplier = v
288 289 290

# -----

291
def extra_run_opts( val ):
292
    return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
293

294
def _extra_run_opts( name, opts, v ):
295 296
    opts.extra_run_opts = v

297 298
# -----

Simon Marlow's avatar
Simon Marlow committed
299
def extra_hc_opts( val ):
300
    return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
Simon Marlow's avatar
Simon Marlow committed
301

302
def _extra_hc_opts( name, opts, v ):
Simon Marlow's avatar
Simon Marlow committed
303 304 305 306
    opts.extra_hc_opts = v

# -----

307
def extra_clean( files ):
308 309
    # TODO. Remove all calls to extra_clean.
    return lambda _name, _opts: None
310

311 312 313 314 315 316
def extra_files(files):
    return lambda name, opts: _extra_files(name, opts, files)

def _extra_files(name, opts, files):
    opts.extra_files.extend(files)

317 318
# -----

319 320 321 322 323 324 325
def stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);

def _stats_num_field( name, opts, field, expecteds ):
    if field in opts.stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

326
    if type(expecteds) is list:
327 328 329 330 331
        for (b, expected, dev) in expecteds:
            if b:
                opts.stats_range_fields[field] = (expected, dev)
                return
        framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
332

333 334 335
    else:
        (expected, dev) = expecteds
        opts.stats_range_fields[field] = (expected, dev)
336 337 338

def compiler_stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
339

340 341 342 343
def _compiler_stats_num_field( name, opts, field, expecteds ):
    if field in opts.compiler_stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

344 345
    # Compiler performance numbers change when debugging is on, making the results
    # useless and confusing. Therefore, skip if debugging is on.
346 347 348
    if compiler_debugged():
        skip(name, opts)

349 350 351 352
    for (b, expected, dev) in expecteds:
        if b:
            opts.compiler_stats_range_fields[field] = (expected, dev)
            return
353

354 355
    framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')

356 357
# -----

358
def when(b, f):
ian@well-typed.com's avatar
ian@well-typed.com committed
359 360 361
    # When list_brokens is on, we want to see all expect_broken calls,
    # so we always do f
    if b or config.list_broken:
362 363 364 365 366 367 368
        return f
    else:
        return normal

def unless(b, f):
    return when(not b, f)

ian@well-typed.com's avatar
ian@well-typed.com committed
369 370 371
def doing_ghci():
    return 'ghci' in config.run_ways

372
def ghc_dynamic():
ian@well-typed.com's avatar
ian@well-typed.com committed
373
    return config.ghc_dynamic
374 375

def fast():
376
    return config.speed == 2
377

378 379
def platform( plat ):
    return config.platform == plat
Ian Lynagh's avatar
Ian Lynagh committed
380

381 382
def opsys( os ):
    return config.os == os
Ian Lynagh's avatar
Ian Lynagh committed
383

384 385
def arch( arch ):
    return config.arch == arch
386

387 388
def wordsize( ws ):
    return config.wordsize == str(ws)
tibbe's avatar
tibbe committed
389

390 391
def msys( ):
    return config.msys
ian@well-typed.com's avatar
ian@well-typed.com committed
392

393 394
def cygwin( ):
    return config.cygwin
Ian Lynagh's avatar
Ian Lynagh committed
395

396 397
def have_vanilla( ):
    return config.have_vanilla
398

399 400
def have_dynamic( ):
    return config.have_dynamic
401

402 403
def have_profiling( ):
    return config.have_profiling
404

405 406
def in_tree_compiler( ):
    return config.in_tree_compiler
407

408 409 410 411 412 413
def unregisterised( ):
    return config.unregisterised

def compiler_profiled( ):
    return config.compiler_profiled

414 415
def compiler_debugged( ):
    return config.compiler_debugged
416

417 418
# ---

419
def high_memory_usage(name, opts):
420 421
    opts.alone = True

422 423 424 425 426
# If a test is for a multi-CPU race, then running the test alone
# increases the chance that we'll actually see it.
def multi_cpu_race(name, opts):
    opts.alone = True

Ian Lynagh's avatar
Ian Lynagh committed
427
# ---
428
def literate( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
429 430
    opts.literate = 1;

431
def c_src( name, opts ):
432 433
    opts.c_src = 1;

434
def objc_src( name, opts ):
Austin Seipp's avatar
Austin Seipp committed
435 436
    opts.objc_src = 1;

437
def objcpp_src( name, opts ):
438 439
    opts.objcpp_src = 1;

440
def cmm_src( name, opts ):
441 442
    opts.cmm_src = 1;

443
def outputdir( odir ):
444
    return lambda name, opts, d=odir: _outputdir(name, opts, d)
445

446
def _outputdir( name, opts, odir ):
447 448
    opts.outputdir = odir;

449 450
# ----

451
def pre_cmd( cmd ):
452
    return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
453

454
def _pre_cmd( name, opts, cmd ):
455 456 457 458
    opts.pre_cmd = cmd

# ----

459
def clean_cmd( cmd ):
460 461
    # TODO. Remove all calls to clean_cmd.
    return lambda _name, _opts: None
462 463 464

# ----

465
def cmd_prefix( prefix ):
466
    return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
467

468
def _cmd_prefix( name, opts, prefix ):
469 470 471 472 473
    opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;

# ----

def cmd_wrapper( fun ):
474
    return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
475

476
def _cmd_wrapper( name, opts, fun ):
477
    opts.cmd_wrapper = fun
478

479 480
# ----

Ian Lynagh's avatar
Ian Lynagh committed
481
def compile_cmd_prefix( prefix ):
482
    return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
Ian Lynagh's avatar
Ian Lynagh committed
483

484
def _compile_cmd_prefix( name, opts, prefix ):
Ian Lynagh's avatar
Ian Lynagh committed
485 486 487 488
    opts.compile_cmd_prefix = prefix

# ----

489 490 491 492 493 494
def check_stdout( f ):
    return lambda name, opts, f=f: _check_stdout(name, opts, f)

def _check_stdout( name, opts, f ):
    opts.check_stdout = f

495 496 497
def no_check_hp(name, opts):
    opts.check_hp = False

498 499
# ----

500
def normalise_slashes( name, opts ):
501
    _normalise_fun(name, opts, normalise_slashes_)
502

503
def normalise_exe( name, opts ):
504
    _normalise_fun(name, opts, normalise_exe_)
505

506 507
def normalise_fun( *fs ):
    return lambda name, opts: _normalise_fun(name, opts, fs)
508

509
def _normalise_fun( name, opts, *fs ):
510
    opts.extra_normaliser = join_normalisers(opts.extra_normaliser, fs)
511

512 513
def normalise_errmsg_fun( *fs ):
    return lambda name, opts: _normalise_errmsg_fun(name, opts, fs)
514

515
def _normalise_errmsg_fun( name, opts, *fs ):
516 517 518 519 520 521 522 523 524 525 526 527 528
    opts.extra_errmsg_normaliser =  join_normalisers(opts.extra_errmsg_normaliser, fs)

def normalise_version_( *pkgs ):
    def normalise_version__( str ):
        return re.sub('(' + '|'.join(map(re.escape,pkgs)) + ')-[0-9.]+',
                      '\\1-<VERSION>', str)
    return normalise_version__

def normalise_version( *pkgs ):
    def normalise_version__( name, opts ):
        _normalise_fun(name, opts, normalise_version_(*pkgs))
        _normalise_errmsg_fun(name, opts, normalise_version_(*pkgs))
    return normalise_version__
529

thomie's avatar
thomie committed
530 531 532 533
def normalise_drive_letter(name, opts):
    # Windows only. Change D:\\ to C:\\.
    _normalise_fun(name, opts, lambda str: re.sub(r'[A-Z]:\\', r'C:\\', str))

534 535 536 537 538 539 540
def keep_prof_callstacks(name, opts):
    """Keep profiling callstacks.

    Use together with `only_ways(prof_ways)`.
    """
    opts.keep_prof_callstacks = True

541 542
def join_normalisers(*a):
    """
543
    Compose functions, flattening sequences.
544

545
       join_normalisers(f1,[f2,f3],f4)
546 547 548

    is the same as

549
       lambda x: f1(f2(f3(f4(x))))
550 551
    """

552 553 554 555 556 557 558 559 560 561 562 563
    def flatten(l):
        """
        Taken from http://stackoverflow.com/a/2158532/946226
        """
        for el in l:
            if isinstance(el, collections.Iterable) and not isinstance(el, basestring):
                for sub in flatten(el):
                    yield sub
            else:
                yield el

    a = flatten(a)
564 565 566

    fn = lambda x:x # identity function
    for f in a:
567
        assert callable(f)
568 569 570
        fn = lambda x,f=f,fn=fn: fn(f(x))
    return fn

571 572 573
# ----
# Function for composing two opt-fns together

574
def executeSetups(fs, name, opts):
575
    if type(fs) is list:
576
        # If we have a list of setups, then execute each one
577 578
        for f in fs:
            executeSetups(f, name, opts)
579 580 581
    else:
        # fs is a single function, so just apply it
        fs(name, opts)
582

583 584 585
# -----------------------------------------------------------------------------
# The current directory of tests

586 587
def newTestDir(tempdir, dir):

588
    global thisdir_settings
589
    # reset the options for this test directory
590 591 592
    def settings(name, opts, tempdir=tempdir, dir=dir):
        return _newTestDir(name, opts, tempdir, dir)
    thisdir_settings = settings
593

594 595
# Should be equal to entry in toplevel .gitignore.
testdir_suffix = '.run'
596 597 598

def _newTestDir(name, opts, tempdir, dir):
    opts.srcdir = os.path.join(os.getcwd(), dir)
599
    opts.testdir = os.path.join(tempdir, dir, name + testdir_suffix)
600
    opts.compiler_always_flags = config.compiler_always_flags
601 602 603 604

# -----------------------------------------------------------------------------
# Actually doing tests

605 606
parallelTests = []
aloneTests = []
607
allTestNames = set([])
608

609
def runTest(watcher, opts, name, func, args):
610
    if config.use_threads:
611 612 613 614 615 616
        pool_sema.acquire()
        t = threading.Thread(target=test_common_thread,
                             name=name,
                             args=(watcher, name, opts, func, args))
        t.daemon = False
        t.start()
617
    else:
618
        test_common_work(watcher, name, opts, func, args)
619

620
# name  :: String
621
# setup :: TestOpts -> IO ()
622
def test(name, setup, func, args):
623 624 625 626 627 628 629 630 631
    global aloneTests
    global parallelTests
    global allTestNames
    global thisdir_settings
    if name in allTestNames:
        framework_fail(name, 'duplicate', 'There are multiple tests with this name')
    if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
        framework_fail(name, 'bad_name', 'This test has an invalid name')

632 633 634 635 636 637 638 639 640 641
    if config.run_only_some_tests:
        if name not in config.only:
            return
        else:
            # Note [Mutating config.only]
            # config.only is initiallly the set of tests requested by
            # the user (via 'make TEST='). We then remove all tests that
            # we've already seen (in .T files), so that we can later
            # report on any tests we couldn't find and error out.
            config.only.remove(name)
642

643 644 645 646
    # Make a deep copy of the default_testopts, as we need our own copy
    # of any dictionaries etc inside it. Otherwise, if one test modifies
    # them, all tests will see the modified version!
    myTestOpts = copy.deepcopy(default_testopts)
647

648
    executeSetups([thisdir_settings, setup], name, myTestOpts)
649

650
    thisTest = lambda watcher: runTest(watcher, myTestOpts, name, func, args)
651 652 653 654
    if myTestOpts.alone:
        aloneTests.append(thisTest)
    else:
        parallelTests.append(thisTest)
655
    allTestNames.add(name)
656

657
if config.use_threads:
658 659 660 661 662
    def test_common_thread(watcher, name, opts, func, args):
            try:
                test_common_work(watcher, name, opts, func, args)
            finally:
                pool_sema.release()
663

664 665 666 667 668 669 670 671 672
def get_package_cache_timestamp():
    if config.package_conf_cache_file == '':
        return 0.0
    else:
        try:
            return os.stat(config.package_conf_cache_file).st_mtime
        except:
            return 0.0

Edward Z. Yang's avatar
Edward Z. Yang committed
673
do_not_copy = ('.hi', '.o', '.dyn_hi', '.dyn_o', '.out') # 12112
674

675
def test_common_work(watcher, name, opts, func, args):
676
    try:
677
        t.total_tests += 1
678 679 680 681 682 683 684
        setLocalTestOpts(opts)

        package_conf_cache_file_start_timestamp = get_package_cache_timestamp()

        # All the ways we might run this test
        if func == compile or func == multimod_compile:
            all_ways = config.compile_ways
685
        elif func == compile_and_run or func == multimod_compile_and_run:
686 687 688 689 690 691
            all_ways = config.run_ways
        elif func == ghci_script:
            if 'ghci' in config.run_ways:
                all_ways = ['ghci']
            else:
                all_ways = []
692
        else:
693 694 695
            all_ways = ['normal']

        # A test itself can request extra ways by setting opts.extra_ways
696
        all_ways = all_ways + [way for way in opts.extra_ways if way not in all_ways]
697

698
        t.total_test_cases += len(all_ways)
699 700 701

        ok_way = lambda way: \
            not getTestOpts().skip \
702
            and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
703
            and (config.cmdline_ways == [] or way in config.cmdline_ways) \
704
            and (not (config.skip_perf_tests and isStatsTest())) \
705 706 707
            and way not in getTestOpts().omit_ways

        # Which ways we are asked to skip
708
        do_ways = list(filter (ok_way,all_ways))
709

710 711
        # Only run all ways in slow mode.
        # See Note [validate and testsuite speed] in toplevel Makefile.
712 713
        if config.accept:
            # Only ever run one way
714
            do_ways = do_ways[:1]
715 716 717
        elif config.speed > 0:
            # However, if we EXPLICITLY asked for a way (with extra_ways)
            # please test it!
718 719
            explicit_ways = list(filter(lambda way: way in opts.extra_ways, do_ways))
            other_ways = list(filter(lambda way: way not in opts.extra_ways, do_ways))
720
            do_ways = other_ways[:1] + explicit_ways
721

722 723 724 725 726 727 728 729
        # Find all files in the source directory that this test
        # depends on. Do this only once for all ways.
        # Generously add all filenames that start with the name of
        # the test to this set, as a convenience to test authors.
        # They will have to use the `extra_files` setup function to
        # specify all other files that their test depends on (but
        # this seems to be necessary for only about 10% of all
        # tests).
730 731 732 733
        files = set(f for f in os.listdir(opts.srcdir)
                       if f.startswith(name) and not f == name and
                          not f.endswith(testdir_suffix) and
                          not os.path.splitext(f)[1] in do_not_copy)
734
        for filename in (opts.extra_files + extra_src_files.get(name, [])):
735
            if filename.startswith('/'):
736 737 738 739 740 741 742 743 744
                framework_fail(name, 'whole-test',
                    'no absolute paths in extra_files please: ' + filename)

            elif '*' in filename:
                # Don't use wildcards in extra_files too much, as
                # globbing is slow.
                files.update((os.path.relpath(f, opts.srcdir)
                            for f in glob.iglob(in_srcdir(filename))))

thomie's avatar
thomie committed
745
            elif filename:
746 747
                files.add(filename)

thomie's avatar
thomie committed
748 749 750
            else:
                framework_fail(name, 'whole-test', 'extra_file is empty string')

751 752 753 754
        # Run the required tests...
        for way in do_ways:
            if stopping():
                break
755 756 757 758 759 760 761
            try:
                do_test(name, way, func, args, files)
            except KeyboardInterrupt:
                stopNow()
            except Exception as e:
                framework_fail(name, way, str(e))
                traceback.print_exc()
762

763
        t.n_tests_skipped += len(set(all_ways) - set(do_ways))
764

765
        if config.cleanup and do_ways:
766
            cleanup()
767

768
        package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
769

770 771
        if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
            framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
772

773
    except Exception as e:
774
        framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
775 776
    finally:
        watcher.notify()
Ian Lynagh's avatar
Ian Lynagh committed
777

778 779 780
def do_test(name, way, func, args, files):
    opts = getTestOpts()

781 782
    full_name = name + '(' + way + ')'

783
    if_verbose(2, "=====> {0} {1} of {2} {3}".format(
784 785 786 787
        full_name, t.total_tests, len(allTestNames), 
        [len(t.unexpected_passes),
         len(t.unexpected_failures),
         len(t.framework_failures)]))
788 789 790 791

    # Clean up prior to the test, so that we can't spuriously conclude
    # that it passed on the basis of old run outputs.
    cleanup()
792
    os.makedirs(opts.testdir)
793 794 795 796 797 798 799 800 801

    # Link all source files for this test into a new directory in
    # /tmp, and run the test in that directory. This makes it
    # possible to run tests in parallel, without modification, that
    # would otherwise (accidentally) write to the same output file.
    # It also makes it easier to keep the testsuite clean.

    for extra_file in files:
        src = in_srcdir(extra_file)
802
        dst = in_testdir(os.path.basename(extra_file.rstrip('/\\')))
803
        if os.path.isfile(src):
804
            link_or_copy_file(src, dst)
805
        elif os.path.isdir(src):
806
            os.mkdir(dst)
807 808
            lndir(src, dst)
        else:
809
            if not config.haddock and os.path.splitext(extra_file)[1] == '.t':
810 811 812 813 814 815 816 817 818 819 820 821 822 823
                # When using a ghc built without haddock support, .t
                # files are rightfully missing. Don't
                # framework_fail. Test will be skipped later.
                pass
            else:
                framework_fail(name, way,
                    'extra_file does not exist: ' + extra_file)

    if func.__name__ == 'run_command' or opts.pre_cmd:
        # When running 'MAKE' make sure 'TOP' still points to the
        # root of the testsuite.
        src_makefile = in_srcdir('Makefile')
        dst_makefile = in_testdir('Makefile')
        if os.path.exists(src_makefile):
824
            with io.open(src_makefile, 'r', encoding='utf8') as src:
825
                makefile = re.sub('TOP=.*', 'TOP=' + config.top, src.read(), 1)
826
                with io.open(dst_makefile, 'w', encoding='utf8') as dst:
827
                    dst.write(makefile)
828

829 830 831 832
    if opts.pre_cmd:
        exit_code = runCmd('cd "{0}" && {1}'.format(opts.testdir, opts.pre_cmd))
        if exit_code != 0:
            framework_fail(name, way, 'pre_cmd failed: {0}'.format(exit_code))
833

834 835
    try:
        result = func(*[name,way] + args)
Tamar Christina's avatar