testlib.py 69.4 KB
Newer Older
1
# coding=utf8
2
#
3 4 5
# (c) Simon Marlow 2002
#

6
from __future__ import print_function
7

8
import io
9
import shutil
10
import os
11
import errno
12 13 14
import string
import re
import traceback
15 16
import time
import datetime
17
import copy
18
import glob
ian@well-typed.com's avatar
ian@well-typed.com committed
19
from math import ceil, trunc
20
import collections
21
import subprocess
22

23
from testglobals import *
24
from testutil import *
25
from extra_files import extra_src_files
26

27 28 29 30 31
try:
    basestring
except: # Python 3
    basestring = (str,bytes)

32 33
if config.use_threads:
    import threading
34 35 36 37
    try:
        import thread
    except ImportError: # Python 3
        import _thread as thread
38

39 40 41 42 43 44 45 46
global wantToStop
wantToStop = False
def stopNow():
    global wantToStop
    wantToStop = True
def stopping():
    return wantToStop

47 48
# Options valid for the current test only (these get reset to
# testdir_testopts after each test).
49

ei@vuokko.info's avatar
ei@vuokko.info committed
50
global testopts_local
51 52 53 54 55 56
if config.use_threads:
    testopts_local = threading.local()
else:
    class TestOpts_Local:
        pass
    testopts_local = TestOpts_Local()
57 58

def getTestOpts():
ei@vuokko.info's avatar
ei@vuokko.info committed
59
    return testopts_local.x
60

ei@vuokko.info's avatar
ei@vuokko.info committed
61 62 63
def setLocalTestOpts(opts):
    global testopts_local
    testopts_local.x=opts
64

65 66 67 68 69
def isStatsTest():
    opts = getTestOpts()
    return len(opts.compiler_stats_range_fields) > 0 or len(opts.stats_range_fields) > 0


70 71 72
# This can be called at the top of a file of tests, to set default test options
# for the following tests.
def setTestOpts( f ):
73
    global thisdir_settings
74
    thisdir_settings = [thisdir_settings, f]
75 76 77 78 79 80 81 82 83 84 85 86

# -----------------------------------------------------------------------------
# Canned setup functions for common cases.  eg. for a test you might say
#
#      test('test001', normal, compile, [''])
#
# to run it without any options, but change it to
#
#      test('test001', expect_fail, compile, [''])
#
# to expect failure for this test.

87
def normal( name, opts ):
88 89
    return;

90
def skip( name, opts ):
91 92
    opts.skip = 1

93
def expect_fail( name, opts ):
94 95 96
    # The compiler, testdriver, OS or platform is missing a certain
    # feature, and we don't plan to or can't fix it now or in the
    # future.
97 98
    opts.expect = 'fail';

99
def reqlib( lib ):
100
    return lambda name, opts, l=lib: _reqlib (name, opts, l )
101

102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131
def stage1(name, opts):
    # See Note [Why is there no stage1 setup function?]
    framework_fail(name, 'stage1 setup function does not exist',
                   'add your test to testsuite/tests/stage1 instead')

# Note [Why is there no stage1 setup function?]
#
# Presumably a stage1 setup function would signal that the stage1
# compiler should be used to compile a test.
#
# Trouble is, the path to the compiler + the `ghc --info` settings for
# that compiler are currently passed in from the `make` part of the
# testsuite driver.
#
# Switching compilers in the Python part would be entirely too late, as
# all ghc_with_* settings would be wrong. See config/ghc for possible
# consequences (for example, config.run_ways would still be
# based on the default compiler, quite likely causing ./validate --slow
# to fail).
#
# It would be possible to let the Python part of the testsuite driver
# make the call to `ghc --info`, but doing so would require quite some
# work. Care has to be taken to not affect the run_command tests for
# example, as they also use the `ghc --info` settings:
#     quasiquotation/qq007/Makefile:ifeq "$(GhcDynamic)" "YES"
#
# If you want a test to run using the stage1 compiler, add it to the
# testsuite/tests/stage1 directory. Validate runs the tests in that
# directory with `make stage=1`.

132 133 134 135
# Cache the results of looking to see if we have a library or not.
# This makes quite a difference, especially on Windows.
have_lib = {}

136
def _reqlib( name, opts, lib ):
137
    if lib in have_lib:
138
        got_it = have_lib[lib]
139
    else:
140 141 142 143 144 145 146 147
        cmd = strip_quotes(config.ghc_pkg)
        p = subprocess.Popen([cmd, '--no-user-package-db', 'describe', lib],
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
        # read from stdout and stderr to avoid blocking due to
        # buffers filling
        p.communicate()
        r = p.wait()
148 149 150 151
        got_it = r == 0
        have_lib[lib] = got_it

    if not got_it:
152
        opts.expect = 'missing-lib'
153

154 155 156 157
def req_haddock( name, opts ):
    if not config.haddock:
        opts.expect = 'missing-lib'

158
def req_profiling( name, opts ):
159
    '''Require the profiling libraries (add 'GhcLibWays += p' to mk/build.mk)'''
160 161 162
    if not config.have_profiling:
        opts.expect = 'fail'

163
def req_shared_libs( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
164 165 166
    if not config.have_shared_libs:
        opts.expect = 'fail'

167
def req_interp( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
168 169 170
    if not config.have_interp:
        opts.expect = 'fail'

171
def req_smp( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
172 173 174
    if not config.have_smp:
        opts.expect = 'fail'

175
def ignore_output( name, opts ):
176 177
    opts.ignore_output = 1

178
def no_stdin( name, opts ):
179 180
    opts.no_stdin = 1

181
def combined_output( name, opts ):
pcapriotti's avatar
pcapriotti committed
182 183
    opts.combined_output = True

184 185 186
# -----

def expect_fail_for( ways ):
187
    return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
188

189
def _expect_fail_for( name, opts, ways ):
190 191
    opts.expect_fail_for = ways

192
def expect_broken( bug ):
193 194
    # This test is a expected not to work due to the indicated trac bug
    # number.
195 196 197
    return lambda name, opts, b=bug: _expect_broken (name, opts, b )

def _expect_broken( name, opts, bug ):
198
    record_broken(name, opts, bug)
199 200
    opts.expect = 'fail';

Ian Lynagh's avatar
Ian Lynagh committed
201
def expect_broken_for( bug, ways ):
202
    return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
Ian Lynagh's avatar
Ian Lynagh committed
203

204
def _expect_broken_for( name, opts, bug, ways ):
205
    record_broken(name, opts, bug)
Ian Lynagh's avatar
Ian Lynagh committed
206
    opts.expect_fail_for = ways
Ian Lynagh's avatar
Ian Lynagh committed
207

208
def record_broken(name, opts, bug):
209
    global brokens
210
    me = (bug, opts.testdir, name)
211 212 213
    if not me in brokens:
        brokens.append(me)

214 215 216 217 218
def _expect_pass(way):
    # Helper function. Not intended for use in .T files.
    opts = getTestOpts()
    return opts.expect == 'pass' and way not in opts.expect_fail_for

219 220 221
# -----

def omit_ways( ways ):
222
    return lambda name, opts, w=ways: _omit_ways( name, opts, w )
223

224
def _omit_ways( name, opts, ways ):
225 226 227 228
    opts.omit_ways = ways

# -----

229
def only_ways( ways ):
230
    return lambda name, opts, w=ways: _only_ways( name, opts, w )
231

232
def _only_ways( name, opts, ways ):
233 234 235 236
    opts.only_ways = ways

# -----

237
def extra_ways( ways ):
238
    return lambda name, opts, w=ways: _extra_ways( name, opts, w )
239

240
def _extra_ways( name, opts, ways ):
241 242 243 244
    opts.extra_ways = ways

# -----

245
def set_stdin( file ):
246
   return lambda name, opts, f=file: _set_stdin(name, opts, f);
247

248
def _set_stdin( name, opts, f ):
249 250 251 252 253
   opts.stdin = f

# -----

def exit_code( val ):
254
    return lambda name, opts, v=val: _exit_code(name, opts, v);
255

256
def _exit_code( name, opts, v ):
257 258
    opts.exit_code = v

259 260 261 262 263 264 265 266 267 268 269
def signal_exit_code( val ):
    if opsys('solaris2'):
        return exit_code( val );
    else:
        # When application running on Linux receives fatal error
        # signal, then its exit code is encoded as 128 + signal
        # value. See http://www.tldp.org/LDP/abs/html/exitcodes.html
        # I assume that Mac OS X behaves in the same way at least Mac
        # OS X builder behavior suggests this.
        return exit_code( val+128 );

270 271
# -----

272 273
def compile_timeout_multiplier( val ):
    return lambda name, opts, v=val: _compile_timeout_multiplier(name, opts, v)
274

275 276 277 278 279 280 281 282
def _compile_timeout_multiplier( name, opts, v ):
    opts.compile_timeout_multiplier = v

def run_timeout_multiplier( val ):
    return lambda name, opts, v=val: _run_timeout_multiplier(name, opts, v)

def _run_timeout_multiplier( name, opts, v ):
    opts.run_timeout_multiplier = v
283 284 285

# -----

286
def extra_run_opts( val ):
287
    return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
288

289
def _extra_run_opts( name, opts, v ):
290 291
    opts.extra_run_opts = v

292 293
# -----

Simon Marlow's avatar
Simon Marlow committed
294
def extra_hc_opts( val ):
295
    return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
Simon Marlow's avatar
Simon Marlow committed
296

297
def _extra_hc_opts( name, opts, v ):
Simon Marlow's avatar
Simon Marlow committed
298 299 300 301
    opts.extra_hc_opts = v

# -----

302
def extra_clean( files ):
303 304
    # TODO. Remove all calls to extra_clean.
    return lambda _name, _opts: None
305

306 307 308 309 310 311
def extra_files(files):
    return lambda name, opts: _extra_files(name, opts, files)

def _extra_files(name, opts, files):
    opts.extra_files.extend(files)

312 313
# -----

314 315 316 317 318 319 320
def stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);

def _stats_num_field( name, opts, field, expecteds ):
    if field in opts.stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

321
    if type(expecteds) is list:
322 323 324 325 326
        for (b, expected, dev) in expecteds:
            if b:
                opts.stats_range_fields[field] = (expected, dev)
                return
        framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
327

328 329 330
    else:
        (expected, dev) = expecteds
        opts.stats_range_fields[field] = (expected, dev)
331 332 333

def compiler_stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
334

335 336 337 338
def _compiler_stats_num_field( name, opts, field, expecteds ):
    if field in opts.compiler_stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

339 340
    # Compiler performance numbers change when debugging is on, making the results
    # useless and confusing. Therefore, skip if debugging is on.
341 342 343
    if compiler_debugged():
        skip(name, opts)

344 345 346 347
    for (b, expected, dev) in expecteds:
        if b:
            opts.compiler_stats_range_fields[field] = (expected, dev)
            return
348

349 350
    framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')

351 352
# -----

353
def when(b, f):
ian@well-typed.com's avatar
ian@well-typed.com committed
354 355 356
    # When list_brokens is on, we want to see all expect_broken calls,
    # so we always do f
    if b or config.list_broken:
357 358 359 360 361 362 363
        return f
    else:
        return normal

def unless(b, f):
    return when(not b, f)

ian@well-typed.com's avatar
ian@well-typed.com committed
364 365 366
def doing_ghci():
    return 'ghci' in config.run_ways

367
def ghc_dynamic():
ian@well-typed.com's avatar
ian@well-typed.com committed
368
    return config.ghc_dynamic
369 370

def fast():
371
    return config.speed == 2
372

373 374
def platform( plat ):
    return config.platform == plat
Ian Lynagh's avatar
Ian Lynagh committed
375

376 377
def opsys( os ):
    return config.os == os
Ian Lynagh's avatar
Ian Lynagh committed
378

379 380
def arch( arch ):
    return config.arch == arch
381

382 383
def wordsize( ws ):
    return config.wordsize == str(ws)
tibbe's avatar
tibbe committed
384

385 386
def msys( ):
    return config.msys
ian@well-typed.com's avatar
ian@well-typed.com committed
387

388 389
def cygwin( ):
    return config.cygwin
Ian Lynagh's avatar
Ian Lynagh committed
390

391 392
def have_vanilla( ):
    return config.have_vanilla
393

394 395
def have_dynamic( ):
    return config.have_dynamic
396

397 398
def have_profiling( ):
    return config.have_profiling
399

400 401
def in_tree_compiler( ):
    return config.in_tree_compiler
402

403 404 405 406 407 408
def unregisterised( ):
    return config.unregisterised

def compiler_profiled( ):
    return config.compiler_profiled

409 410
def compiler_debugged( ):
    return config.compiler_debugged
411

412 413
# ---

414
def high_memory_usage(name, opts):
415 416
    opts.alone = True

417 418 419 420 421
# If a test is for a multi-CPU race, then running the test alone
# increases the chance that we'll actually see it.
def multi_cpu_race(name, opts):
    opts.alone = True

Ian Lynagh's avatar
Ian Lynagh committed
422
# ---
423
def literate( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
424 425
    opts.literate = 1;

426
def c_src( name, opts ):
427 428
    opts.c_src = 1;

429
def objc_src( name, opts ):
Austin Seipp's avatar
Austin Seipp committed
430 431
    opts.objc_src = 1;

432
def objcpp_src( name, opts ):
433 434
    opts.objcpp_src = 1;

435
def cmm_src( name, opts ):
436 437
    opts.cmm_src = 1;

438
def outputdir( odir ):
439
    return lambda name, opts, d=odir: _outputdir(name, opts, d)
440

441
def _outputdir( name, opts, odir ):
442 443
    opts.outputdir = odir;

444 445
# ----

446
def pre_cmd( cmd ):
447
    return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
448

449
def _pre_cmd( name, opts, cmd ):
450 451 452 453
    opts.pre_cmd = cmd

# ----

454
def clean_cmd( cmd ):
455 456
    # TODO. Remove all calls to clean_cmd.
    return lambda _name, _opts: None
457 458 459

# ----

460
def cmd_prefix( prefix ):
461
    return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
462

463
def _cmd_prefix( name, opts, prefix ):
464 465 466 467 468
    opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;

# ----

def cmd_wrapper( fun ):
469
    return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
470

471
def _cmd_wrapper( name, opts, fun ):
472
    opts.cmd_wrapper = fun
473

474 475
# ----

Ian Lynagh's avatar
Ian Lynagh committed
476
def compile_cmd_prefix( prefix ):
477
    return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
Ian Lynagh's avatar
Ian Lynagh committed
478

479
def _compile_cmd_prefix( name, opts, prefix ):
Ian Lynagh's avatar
Ian Lynagh committed
480 481 482 483
    opts.compile_cmd_prefix = prefix

# ----

484 485 486 487 488 489 490 491
def check_stdout( f ):
    return lambda name, opts, f=f: _check_stdout(name, opts, f)

def _check_stdout( name, opts, f ):
    opts.check_stdout = f

# ----

492
def normalise_slashes( name, opts ):
493
    _normalise_fun(name, opts, normalise_slashes_)
494

495
def normalise_exe( name, opts ):
496
    _normalise_fun(name, opts, normalise_exe_)
497

498 499
def normalise_fun( *fs ):
    return lambda name, opts: _normalise_fun(name, opts, fs)
500

501
def _normalise_fun( name, opts, *fs ):
502
    opts.extra_normaliser = join_normalisers(opts.extra_normaliser, fs)
503

504 505
def normalise_errmsg_fun( *fs ):
    return lambda name, opts: _normalise_errmsg_fun(name, opts, fs)
506

507
def _normalise_errmsg_fun( name, opts, *fs ):
508 509 510 511 512 513 514 515 516 517 518 519 520
    opts.extra_errmsg_normaliser =  join_normalisers(opts.extra_errmsg_normaliser, fs)

def normalise_version_( *pkgs ):
    def normalise_version__( str ):
        return re.sub('(' + '|'.join(map(re.escape,pkgs)) + ')-[0-9.]+',
                      '\\1-<VERSION>', str)
    return normalise_version__

def normalise_version( *pkgs ):
    def normalise_version__( name, opts ):
        _normalise_fun(name, opts, normalise_version_(*pkgs))
        _normalise_errmsg_fun(name, opts, normalise_version_(*pkgs))
    return normalise_version__
521

thomie's avatar
thomie committed
522 523 524 525
def normalise_drive_letter(name, opts):
    # Windows only. Change D:\\ to C:\\.
    _normalise_fun(name, opts, lambda str: re.sub(r'[A-Z]:\\', r'C:\\', str))

526 527 528 529 530 531 532
def keep_prof_callstacks(name, opts):
    """Keep profiling callstacks.

    Use together with `only_ways(prof_ways)`.
    """
    opts.keep_prof_callstacks = True

533 534
def join_normalisers(*a):
    """
535
    Compose functions, flattening sequences.
536

537
       join_normalisers(f1,[f2,f3],f4)
538 539 540

    is the same as

541
       lambda x: f1(f2(f3(f4(x))))
542 543
    """

544 545 546 547 548 549 550 551 552 553 554 555
    def flatten(l):
        """
        Taken from http://stackoverflow.com/a/2158532/946226
        """
        for el in l:
            if isinstance(el, collections.Iterable) and not isinstance(el, basestring):
                for sub in flatten(el):
                    yield sub
            else:
                yield el

    a = flatten(a)
556 557 558

    fn = lambda x:x # identity function
    for f in a:
559
        assert callable(f)
560 561 562
        fn = lambda x,f=f,fn=fn: fn(f(x))
    return fn

563 564 565
# ----
# Function for composing two opt-fns together

566
def executeSetups(fs, name, opts):
567
    if type(fs) is list:
568
        # If we have a list of setups, then execute each one
569 570
        for f in fs:
            executeSetups(f, name, opts)
571 572 573
    else:
        # fs is a single function, so just apply it
        fs(name, opts)
574

575 576 577
# -----------------------------------------------------------------------------
# The current directory of tests

578 579
def newTestDir(tempdir, dir):

580
    global thisdir_settings
581
    # reset the options for this test directory
582 583 584
    def settings(name, opts, tempdir=tempdir, dir=dir):
        return _newTestDir(name, opts, tempdir, dir)
    thisdir_settings = settings
585

586 587
# Should be equal to entry in toplevel .gitignore.
testdir_suffix = '.run'
588 589 590

def _newTestDir(name, opts, tempdir, dir):
    opts.srcdir = os.path.join(os.getcwd(), dir)
591
    opts.testdir = os.path.join(tempdir, dir, name + testdir_suffix)
592
    opts.compiler_always_flags = config.compiler_always_flags
593 594 595 596

# -----------------------------------------------------------------------------
# Actually doing tests

597 598
parallelTests = []
aloneTests = []
599
allTestNames = set([])
600

601
def runTest (opts, name, func, args):
ei@vuokko.info's avatar
ei@vuokko.info committed
602 603
    ok = 0

604
    if config.use_threads:
ei@vuokko.info's avatar
ei@vuokko.info committed
605
        t.thread_pool.acquire()
606
        try:
607
            while config.threads<(t.running_threads+1):
608
                t.thread_pool.wait()
609
            t.running_threads = t.running_threads+1
610 611
            ok=1
            t.thread_pool.release()
612
            thread.start_new_thread(test_common_thread, (name, opts, func, args))
613 614 615 616 617
        except:
            if not ok:
                t.thread_pool.release()
    else:
        test_common_work (name, opts, func, args)
618

619
# name  :: String
620
# setup :: TestOpts -> IO ()
621
def test (name, setup, func, args):
622 623 624 625 626 627 628 629 630 631
    if config.run_only_some_tests:
        if name not in config.only:
            return
        else:
            # Note [Mutating config.only]
            # config.only is initiallly the set of tests requested by
            # the user (via 'make TEST='). We then remove all tests that
            # we've already seen (in .T files), so that we can later
            # report on any tests we couldn't find and error out.
            config.only.remove(name)
632

633 634
    global aloneTests
    global parallelTests
635
    global allTestNames
636
    global thisdir_settings
637 638
    if name in allTestNames:
        framework_fail(name, 'duplicate', 'There are multiple tests with this name')
639
    if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
640
        framework_fail(name, 'bad_name', 'This test has an invalid name')
641 642 643 644 645

    # Make a deep copy of the default_testopts, as we need our own copy
    # of any dictionaries etc inside it. Otherwise, if one test modifies
    # them, all tests will see the modified version!
    myTestOpts = copy.deepcopy(default_testopts)
646

647
    executeSetups([thisdir_settings, setup], name, myTestOpts)
648 649 650 651 652 653

    thisTest = lambda : runTest(myTestOpts, name, func, args)
    if myTestOpts.alone:
        aloneTests.append(thisTest)
    else:
        parallelTests.append(thisTest)
654
    allTestNames.add(name)
655

656
if config.use_threads:
657
    def test_common_thread(name, opts, func, args):
658 659 660 661 662 663
        t.lock.acquire()
        try:
            test_common_work(name,opts,func,args)
        finally:
            t.lock.release()
            t.thread_pool.acquire()
664
            t.running_threads = t.running_threads - 1
665 666
            t.thread_pool.notify()
            t.thread_pool.release()
667

668 669 670 671 672 673 674 675 676
def get_package_cache_timestamp():
    if config.package_conf_cache_file == '':
        return 0.0
    else:
        try:
            return os.stat(config.package_conf_cache_file).st_mtime
        except:
            return 0.0

677 678
do_not_copy = ('.hi', '.o', '.dyn_hi', '.dyn_o') # 12112

ei@vuokko.info's avatar
ei@vuokko.info committed
679
def test_common_work (name, opts, func, args):
680 681 682 683 684 685 686 687 688
    try:
        t.total_tests = t.total_tests+1
        setLocalTestOpts(opts)

        package_conf_cache_file_start_timestamp = get_package_cache_timestamp()

        # All the ways we might run this test
        if func == compile or func == multimod_compile:
            all_ways = config.compile_ways
689
        elif func == compile_and_run or func == multimod_compile_and_run:
690 691 692 693 694 695
            all_ways = config.run_ways
        elif func == ghci_script:
            if 'ghci' in config.run_ways:
                all_ways = ['ghci']
            else:
                all_ways = []
696
        else:
697 698 699
            all_ways = ['normal']

        # A test itself can request extra ways by setting opts.extra_ways
700
        all_ways = all_ways + [way for way in opts.extra_ways if way not in all_ways]
701 702 703 704 705

        t.total_test_cases = t.total_test_cases + len(all_ways)

        ok_way = lambda way: \
            not getTestOpts().skip \
706
            and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
707
            and (config.cmdline_ways == [] or way in config.cmdline_ways) \
708
            and (not (config.skip_perf_tests and isStatsTest())) \
709 710 711
            and way not in getTestOpts().omit_ways

        # Which ways we are asked to skip
712
        do_ways = list(filter (ok_way,all_ways))
713

714 715
        # Only run all ways in slow mode.
        # See Note [validate and testsuite speed] in toplevel Makefile.
716 717
        if config.accept:
            # Only ever run one way
718
            do_ways = do_ways[:1]
719 720 721
        elif config.speed > 0:
            # However, if we EXPLICITLY asked for a way (with extra_ways)
            # please test it!
722 723
            explicit_ways = list(filter(lambda way: way in opts.extra_ways, do_ways))
            other_ways = list(filter(lambda way: way not in opts.extra_ways, do_ways))
724
            do_ways = other_ways[:1] + explicit_ways
725

726 727 728 729 730 731 732 733
        # Find all files in the source directory that this test
        # depends on. Do this only once for all ways.
        # Generously add all filenames that start with the name of
        # the test to this set, as a convenience to test authors.
        # They will have to use the `extra_files` setup function to
        # specify all other files that their test depends on (but
        # this seems to be necessary for only about 10% of all
        # tests).
734 735 736 737
        files = set(f for f in os.listdir(opts.srcdir)
                       if f.startswith(name) and not f == name and
                          not f.endswith(testdir_suffix) and
                          not os.path.splitext(f)[1] in do_not_copy)
738
        for filename in (opts.extra_files + extra_src_files.get(name, [])):
739
            if filename.startswith('/'):
740 741 742 743 744 745 746 747 748
                framework_fail(name, 'whole-test',
                    'no absolute paths in extra_files please: ' + filename)

            elif '*' in filename:
                # Don't use wildcards in extra_files too much, as
                # globbing is slow.
                files.update((os.path.relpath(f, opts.srcdir)
                            for f in glob.iglob(in_srcdir(filename))))

thomie's avatar
thomie committed
749
            elif filename:
750 751
                files.add(filename)

thomie's avatar
thomie committed
752 753 754
            else:
                framework_fail(name, 'whole-test', 'extra_file is empty string')

755 756 757 758 759
        # Run the required tests...
        for way in do_ways:
            if stopping():
                break
            do_test(name, way, func, args, files)
760

761 762 763
        for way in all_ways:
            if way not in do_ways:
                skiptest (name,way)
764

765
        if config.cleanup and do_ways:
766
            cleanup()
767

768
        package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
769

770 771
        if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
            framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
772

773
    except Exception as e:
774
        framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
Ian Lynagh's avatar
Ian Lynagh committed
775

776 777 778
def do_test(name, way, func, args, files):
    opts = getTestOpts()

779 780 781
    full_name = name + '(' + way + ')'

    try:
782 783 784 785 786
        if_verbose(2, "=====> %s %d of %d %s " % \
                    (full_name, t.total_tests, len(allTestNames), \
                    [t.n_unexpected_passes, \
                     t.n_unexpected_failures, \
                     t.n_framework_failures]))
787

788 789 790 791 792 793 794 795 796 797
        # Clean up prior to the test, so that we can't spuriously conclude
        # that it passed on the basis of old run outputs.
        cleanup()

        # Link all source files for this test into a new directory in
        # /tmp, and run the test in that directory. This makes it
        # possible to run tests in parallel, without modification, that
        # would otherwise (accidentally) write to the same output file.
        # It also makes it easier to keep the testsuite clean.

798 799 800 801 802 803 804 805 806 807 808 809
        for extra_file in files:
            src = in_srcdir(extra_file)
            if extra_file.startswith('..'):
                # In case the extra_file is a file in an ancestor
                # directory (e.g. extra_files(['../shell.hs'])), make
                # sure it is copied to the test directory
                # (testdir/shell.hs), instead of ending up somewhere
                # else in the tree (testdir/../shell.hs)
                filename = os.path.basename(extra_file)
            else:
                filename = extra_file
            assert not '..' in filename # no funny stuff (foo/../../bar)
810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838
            dst = in_testdir(filename)

            if os.path.isfile(src):
                dirname = os.path.dirname(dst)
                if dirname:
                    mkdirp(dirname)
                try:
                    link_or_copy_file(src, dst)
                except OSError as e:
                    if e.errno == errno.EEXIST and os.path.isfile(dst):
                        # Some tests depend on files from ancestor
                        # directories (e.g. '../shell.hs'). It is
                        # possible such a file was already copied over
                        # for another test, since cleanup() doesn't
                        # delete them.
                        pass
                    else:
                        raise
            elif os.path.isdir(src):
                os.makedirs(dst)
                lndir(src, dst)
            else:
                if not config.haddock and os.path.splitext(filename)[1] == '.t':
                    # When using a ghc built without haddock support, .t
                    # files are rightfully missing. Don't
                    # framework_fail. Test will be skipped later.
                    pass
                else:
                    framework_fail(name, way,
839
                        'extra_file does not exist: ' + extra_file)
840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857

        if not files:
            # Always create the testdir, even when no files were copied
            # (because user forgot to specify extra_files setup function), to
            # prevent the confusing error: can't cd to <testdir>.
            os.makedirs(opts.testdir)

        if func.__name__ == 'run_command' or opts.pre_cmd:
            # When running 'MAKE' make sure 'TOP' still points to the
            # root of the testsuite.
            src_makefile = in_srcdir('Makefile')
            dst_makefile = in_testdir('Makefile')
            if os.path.exists(src_makefile):
                with open(src_makefile, 'r') as src:
                    makefile = re.sub('TOP=.*', 'TOP=' + config.top, src.read(), 1)
                    with open(dst_makefile, 'w') as dst:
                        dst.write(makefile)

858 859
        if config.use_threads:
            t.lock.release()
860

861 862 863 864
        if opts.pre_cmd:
            exit_code = runCmd('cd "{0}" && {1}'.format(opts.testdir, opts.pre_cmd))
            if exit_code != 0:
                framework_fail(name, way, 'pre_cmd failed: {0}'.format(exit_code))
865

ei@vuokko.info's avatar
ei@vuokko.info committed
866
        try:
867
            result = func(*[name,way] + args)
ei@vuokko.info's avatar
ei@vuokko.info committed
868
        finally:
869 870
            if config.use_threads:
                t.lock.acquire()
871

872 873 874
        if getTestOpts().expect != 'pass' and \
                getTestOpts().expect != 'fail' and \
                getTestOpts().expect != 'missing-lib':
875
            framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
876

877 878 879 880 881 882
        try:
            passFail = result['passFail']
        except:
            passFail = 'No passFail found'

        if passFail == 'pass':
883
            if _expect_pass(way):
884
                t.n_expected_passes = t.n_expected_passes + 1
885 886 887 888
                if name in t.expected_passes:
                    t.expected_passes[name].append(way)
                else:
                    t.expected_passes[name] = [way]
889
            else:
890
                if_verbose(1, '*** unexpected pass for %s' % full_name)
891
                t.n_unexpected_passes = t.n_unexpected_passes + 1
892
                addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
893