testlib.py 71.1 KB
Newer Older
1
# coding=utf8
2
#
3 4 5
# (c) Simon Marlow 2002
#

6
from __future__ import print_function
7

8
import shutil
9 10
import sys
import os
11
import errno
12 13 14
import string
import re
import traceback
15 16
import time
import datetime
17
import copy
18
import glob
ian@well-typed.com's avatar
ian@well-typed.com committed
19
from math import ceil, trunc
20
import collections
21
import subprocess
22

23
from testglobals import *
24
from testutil import *
25
from extra_files import extra_src_files
26

27 28 29 30 31
try:
    basestring
except: # Python 3
    basestring = (str,bytes)

32 33
if config.use_threads:
    import threading
34 35 36 37
    try:
        import thread
    except ImportError: # Python 3
        import _thread as thread
38

39 40 41 42 43 44 45 46
global wantToStop
wantToStop = False
def stopNow():
    global wantToStop
    wantToStop = True
def stopping():
    return wantToStop

47 48
# Options valid for the current test only (these get reset to
# testdir_testopts after each test).
49

ei@vuokko.info's avatar
ei@vuokko.info committed
50
global testopts_local
51 52 53 54 55 56
if config.use_threads:
    testopts_local = threading.local()
else:
    class TestOpts_Local:
        pass
    testopts_local = TestOpts_Local()
57 58

def getTestOpts():
ei@vuokko.info's avatar
ei@vuokko.info committed
59
    return testopts_local.x
60

ei@vuokko.info's avatar
ei@vuokko.info committed
61 62 63
def setLocalTestOpts(opts):
    global testopts_local
    testopts_local.x=opts
64

65 66 67 68 69
def isStatsTest():
    opts = getTestOpts()
    return len(opts.compiler_stats_range_fields) > 0 or len(opts.stats_range_fields) > 0


70 71 72
# This can be called at the top of a file of tests, to set default test options
# for the following tests.
def setTestOpts( f ):
73
    global thisdir_settings
74
    thisdir_settings = [thisdir_settings, f]
75 76 77 78 79 80 81 82 83 84 85 86

# -----------------------------------------------------------------------------
# Canned setup functions for common cases.  eg. for a test you might say
#
#      test('test001', normal, compile, [''])
#
# to run it without any options, but change it to
#
#      test('test001', expect_fail, compile, [''])
#
# to expect failure for this test.

87
def normal( name, opts ):
88 89
    return;

90
def skip( name, opts ):
91 92
    opts.skip = 1

93
def expect_fail( name, opts ):
94 95 96
    # The compiler, testdriver, OS or platform is missing a certain
    # feature, and we don't plan to or can't fix it now or in the
    # future.
97 98
    opts.expect = 'fail';

99
def reqlib( lib ):
100
    return lambda name, opts, l=lib: _reqlib (name, opts, l )
101

102 103 104 105
# Cache the results of looking to see if we have a library or not.
# This makes quite a difference, especially on Windows.
have_lib = {}

106
def _reqlib( name, opts, lib ):
107
    if lib in have_lib:
108
        got_it = have_lib[lib]
109
    else:
110 111 112 113 114 115 116 117
        cmd = strip_quotes(config.ghc_pkg)
        p = subprocess.Popen([cmd, '--no-user-package-db', 'describe', lib],
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
        # read from stdout and stderr to avoid blocking due to
        # buffers filling
        p.communicate()
        r = p.wait()
118 119 120 121
        got_it = r == 0
        have_lib[lib] = got_it

    if not got_it:
122
        opts.expect = 'missing-lib'
123

124 125 126 127
def req_haddock( name, opts ):
    if not config.haddock:
        opts.expect = 'missing-lib'

128
def req_profiling( name, opts ):
129
    '''Require the profiling libraries (add 'GhcLibWays += p' to mk/build.mk)'''
130 131 132
    if not config.have_profiling:
        opts.expect = 'fail'

133
def req_shared_libs( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
134 135 136
    if not config.have_shared_libs:
        opts.expect = 'fail'

137
def req_interp( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
138 139 140
    if not config.have_interp:
        opts.expect = 'fail'

141
def req_smp( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
142 143 144
    if not config.have_smp:
        opts.expect = 'fail'

145
def ignore_output( name, opts ):
146 147
    opts.ignore_output = 1

148
def no_stdin( name, opts ):
149 150
    opts.no_stdin = 1

151
def combined_output( name, opts ):
pcapriotti's avatar
pcapriotti committed
152 153
    opts.combined_output = True

154 155 156
# -----

def expect_fail_for( ways ):
157
    return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
158

159
def _expect_fail_for( name, opts, ways ):
160 161
    opts.expect_fail_for = ways

162
def expect_broken( bug ):
163 164
    # This test is a expected not to work due to the indicated trac bug
    # number.
165 166 167
    return lambda name, opts, b=bug: _expect_broken (name, opts, b )

def _expect_broken( name, opts, bug ):
168
    record_broken(name, opts, bug)
169 170
    opts.expect = 'fail';

Ian Lynagh's avatar
Ian Lynagh committed
171
def expect_broken_for( bug, ways ):
172
    return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
Ian Lynagh's avatar
Ian Lynagh committed
173

174
def _expect_broken_for( name, opts, bug, ways ):
175
    record_broken(name, opts, bug)
Ian Lynagh's avatar
Ian Lynagh committed
176
    opts.expect_fail_for = ways
Ian Lynagh's avatar
Ian Lynagh committed
177

178
def record_broken(name, opts, bug):
179
    global brokens
180
    me = (bug, opts.testdir, name)
181 182 183
    if not me in brokens:
        brokens.append(me)

184 185 186 187 188
def _expect_pass(way):
    # Helper function. Not intended for use in .T files.
    opts = getTestOpts()
    return opts.expect == 'pass' and way not in opts.expect_fail_for

189 190 191
# -----

def omit_ways( ways ):
192
    return lambda name, opts, w=ways: _omit_ways( name, opts, w )
193

194
def _omit_ways( name, opts, ways ):
195 196 197 198
    opts.omit_ways = ways

# -----

199
def only_ways( ways ):
200
    return lambda name, opts, w=ways: _only_ways( name, opts, w )
201

202
def _only_ways( name, opts, ways ):
203 204 205 206
    opts.only_ways = ways

# -----

207
def extra_ways( ways ):
208
    return lambda name, opts, w=ways: _extra_ways( name, opts, w )
209

210
def _extra_ways( name, opts, ways ):
211 212 213 214
    opts.extra_ways = ways

# -----

215
def set_stdin( file ):
216
   return lambda name, opts, f=file: _set_stdin(name, opts, f);
217

218
def _set_stdin( name, opts, f ):
219 220 221 222 223
   opts.stdin = f

# -----

def exit_code( val ):
224
    return lambda name, opts, v=val: _exit_code(name, opts, v);
225

226
def _exit_code( name, opts, v ):
227 228
    opts.exit_code = v

229 230 231 232 233 234 235 236 237 238 239
def signal_exit_code( val ):
    if opsys('solaris2'):
        return exit_code( val );
    else:
        # When application running on Linux receives fatal error
        # signal, then its exit code is encoded as 128 + signal
        # value. See http://www.tldp.org/LDP/abs/html/exitcodes.html
        # I assume that Mac OS X behaves in the same way at least Mac
        # OS X builder behavior suggests this.
        return exit_code( val+128 );

240 241
# -----

242 243
def compile_timeout_multiplier( val ):
    return lambda name, opts, v=val: _compile_timeout_multiplier(name, opts, v)
244

245 246 247 248 249 250 251 252
def _compile_timeout_multiplier( name, opts, v ):
    opts.compile_timeout_multiplier = v

def run_timeout_multiplier( val ):
    return lambda name, opts, v=val: _run_timeout_multiplier(name, opts, v)

def _run_timeout_multiplier( name, opts, v ):
    opts.run_timeout_multiplier = v
253 254 255

# -----

256
def extra_run_opts( val ):
257
    return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
258

259
def _extra_run_opts( name, opts, v ):
260 261
    opts.extra_run_opts = v

262 263
# -----

Simon Marlow's avatar
Simon Marlow committed
264
def extra_hc_opts( val ):
265
    return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
Simon Marlow's avatar
Simon Marlow committed
266

267
def _extra_hc_opts( name, opts, v ):
Simon Marlow's avatar
Simon Marlow committed
268 269 270 271
    opts.extra_hc_opts = v

# -----

272
def extra_clean( files ):
273 274
    # TODO. Remove all calls to extra_clean.
    return lambda _name, _opts: None
275

276 277 278 279 280 281
def extra_files(files):
    return lambda name, opts: _extra_files(name, opts, files)

def _extra_files(name, opts, files):
    opts.extra_files.extend(files)

282 283
# -----

284 285 286 287 288 289 290
def stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);

def _stats_num_field( name, opts, field, expecteds ):
    if field in opts.stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

291
    if type(expecteds) is list:
292 293 294 295 296
        for (b, expected, dev) in expecteds:
            if b:
                opts.stats_range_fields[field] = (expected, dev)
                return
        framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
297

298 299 300
    else:
        (expected, dev) = expecteds
        opts.stats_range_fields[field] = (expected, dev)
301 302 303

def compiler_stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
304

305 306 307 308
def _compiler_stats_num_field( name, opts, field, expecteds ):
    if field in opts.compiler_stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

309 310
    # Compiler performance numbers change when debugging is on, making the results
    # useless and confusing. Therefore, skip if debugging is on.
311 312 313
    if compiler_debugged():
        skip(name, opts)

314 315 316 317
    for (b, expected, dev) in expecteds:
        if b:
            opts.compiler_stats_range_fields[field] = (expected, dev)
            return
318

319 320
    framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')

321 322
# -----

323
def when(b, f):
ian@well-typed.com's avatar
ian@well-typed.com committed
324 325 326
    # When list_brokens is on, we want to see all expect_broken calls,
    # so we always do f
    if b or config.list_broken:
327 328 329 330 331 332 333
        return f
    else:
        return normal

def unless(b, f):
    return when(not b, f)

ian@well-typed.com's avatar
ian@well-typed.com committed
334 335 336
def doing_ghci():
    return 'ghci' in config.run_ways

337
def ghc_dynamic():
ian@well-typed.com's avatar
ian@well-typed.com committed
338
    return config.ghc_dynamic
339 340

def fast():
341
    return config.speed == 2
342

343 344
def platform( plat ):
    return config.platform == plat
Ian Lynagh's avatar
Ian Lynagh committed
345

346 347
def opsys( os ):
    return config.os == os
Ian Lynagh's avatar
Ian Lynagh committed
348

349 350
def arch( arch ):
    return config.arch == arch
351

352 353
def wordsize( ws ):
    return config.wordsize == str(ws)
tibbe's avatar
tibbe committed
354

355 356
def msys( ):
    return config.msys
ian@well-typed.com's avatar
ian@well-typed.com committed
357

358 359
def cygwin( ):
    return config.cygwin
Ian Lynagh's avatar
Ian Lynagh committed
360

361 362
def have_vanilla( ):
    return config.have_vanilla
363

364 365
def have_dynamic( ):
    return config.have_dynamic
366

367 368
def have_profiling( ):
    return config.have_profiling
369

370 371
def in_tree_compiler( ):
    return config.in_tree_compiler
372

373 374 375 376 377 378
def unregisterised( ):
    return config.unregisterised

def compiler_profiled( ):
    return config.compiler_profiled

379 380
def compiler_debugged( ):
    return config.compiler_debugged
381

382 383
# ---

384
def high_memory_usage(name, opts):
385 386
    opts.alone = True

387 388 389 390 391
# If a test is for a multi-CPU race, then running the test alone
# increases the chance that we'll actually see it.
def multi_cpu_race(name, opts):
    opts.alone = True

Ian Lynagh's avatar
Ian Lynagh committed
392
# ---
393
def literate( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
394 395
    opts.literate = 1;

396
def c_src( name, opts ):
397 398
    opts.c_src = 1;

399
def objc_src( name, opts ):
Austin Seipp's avatar
Austin Seipp committed
400 401
    opts.objc_src = 1;

402
def objcpp_src( name, opts ):
403 404
    opts.objcpp_src = 1;

405
def cmm_src( name, opts ):
406 407
    opts.cmm_src = 1;

408
def outputdir( odir ):
409
    return lambda name, opts, d=odir: _outputdir(name, opts, d)
410

411
def _outputdir( name, opts, odir ):
412 413
    opts.outputdir = odir;

414 415
# ----

416
def pre_cmd( cmd ):
417
    return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
418

419
def _pre_cmd( name, opts, cmd ):
420 421 422 423
    opts.pre_cmd = cmd

# ----

424
def clean_cmd( cmd ):
425 426
    # TODO. Remove all calls to clean_cmd.
    return lambda _name, _opts: None
427 428 429

# ----

430
def cmd_prefix( prefix ):
431
    return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
432

433
def _cmd_prefix( name, opts, prefix ):
434 435 436 437 438
    opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;

# ----

def cmd_wrapper( fun ):
439
    return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
440

441
def _cmd_wrapper( name, opts, fun ):
442
    opts.cmd_wrapper = fun
443

444 445
# ----

Ian Lynagh's avatar
Ian Lynagh committed
446
def compile_cmd_prefix( prefix ):
447
    return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
Ian Lynagh's avatar
Ian Lynagh committed
448

449
def _compile_cmd_prefix( name, opts, prefix ):
Ian Lynagh's avatar
Ian Lynagh committed
450 451 452 453
    opts.compile_cmd_prefix = prefix

# ----

454 455 456 457 458 459 460 461
def check_stdout( f ):
    return lambda name, opts, f=f: _check_stdout(name, opts, f)

def _check_stdout( name, opts, f ):
    opts.check_stdout = f

# ----

462
def normalise_slashes( name, opts ):
463
    _normalise_fun(name, opts, normalise_slashes_)
464

465
def normalise_exe( name, opts ):
466
    _normalise_fun(name, opts, normalise_exe_)
467

468 469
def normalise_fun( *fs ):
    return lambda name, opts: _normalise_fun(name, opts, fs)
470

471
def _normalise_fun( name, opts, *fs ):
472
    opts.extra_normaliser = join_normalisers(opts.extra_normaliser, fs)
473

474 475
def normalise_errmsg_fun( *fs ):
    return lambda name, opts: _normalise_errmsg_fun(name, opts, fs)
476

477
def _normalise_errmsg_fun( name, opts, *fs ):
478 479 480 481 482 483 484 485 486 487 488 489 490
    opts.extra_errmsg_normaliser =  join_normalisers(opts.extra_errmsg_normaliser, fs)

def normalise_version_( *pkgs ):
    def normalise_version__( str ):
        return re.sub('(' + '|'.join(map(re.escape,pkgs)) + ')-[0-9.]+',
                      '\\1-<VERSION>', str)
    return normalise_version__

def normalise_version( *pkgs ):
    def normalise_version__( name, opts ):
        _normalise_fun(name, opts, normalise_version_(*pkgs))
        _normalise_errmsg_fun(name, opts, normalise_version_(*pkgs))
    return normalise_version__
491

thomie's avatar
thomie committed
492 493 494 495
def normalise_drive_letter(name, opts):
    # Windows only. Change D:\\ to C:\\.
    _normalise_fun(name, opts, lambda str: re.sub(r'[A-Z]:\\', r'C:\\', str))

496 497 498 499 500 501 502
def keep_prof_callstacks(name, opts):
    """Keep profiling callstacks.

    Use together with `only_ways(prof_ways)`.
    """
    opts.keep_prof_callstacks = True

503 504
def join_normalisers(*a):
    """
505
    Compose functions, flattening sequences.
506

507
       join_normalisers(f1,[f2,f3],f4)
508 509 510

    is the same as

511
       lambda x: f1(f2(f3(f4(x))))
512 513
    """

514 515 516 517 518 519 520 521 522 523 524 525
    def flatten(l):
        """
        Taken from http://stackoverflow.com/a/2158532/946226
        """
        for el in l:
            if isinstance(el, collections.Iterable) and not isinstance(el, basestring):
                for sub in flatten(el):
                    yield sub
            else:
                yield el

    a = flatten(a)
526 527 528

    fn = lambda x:x # identity function
    for f in a:
529
        assert callable(f)
530 531 532
        fn = lambda x,f=f,fn=fn: fn(f(x))
    return fn

533 534 535
# ----
# Function for composing two opt-fns together

536
def executeSetups(fs, name, opts):
537
    if type(fs) is list:
538
        # If we have a list of setups, then execute each one
539 540
        for f in fs:
            executeSetups(f, name, opts)
541 542 543
    else:
        # fs is a single function, so just apply it
        fs(name, opts)
544

545 546 547
# -----------------------------------------------------------------------------
# The current directory of tests

548
def newTestDir(tempdir, dir):
549 550 551 552 553 554 555 556 557 558
    # opts.testdir should be quoted when used, to make sure the testsuite
    # keeps working when it contains backward slashes, for example from
    # using os.path.join. Windows native and mingw* python
    # (/mingw64/bin/python) set `os.path.sep = '\\'`, while msys2 python
    # (/bin/python, /usr/bin/python or /usr/local/bin/python) sets
    # `os.path.sep = '/'`.
    # To catch usage of unquoted opts.testdir early, insert some spaces into
    # tempdir.
    tempdir = os.path.join(tempdir, 'test   spaces')

559 560 561 562 563 564 565 566 567 568
    # Hack. A few tests depend on files in ancestor directories
    # (e.g. extra_files(['../../../../libraries/base/dist-install/haddock.t']))
    # Make sure tempdir is sufficiently "deep", such that copying/linking those
    # files won't cause any problems.
    #
    # If you received a framework failure about adding an extra level:
    #  * add one extra '../' to the startswith('../../../../../') in do_test
    #  * add one more number here:
    tempdir = os.path.join(tempdir, '1', '2', '3')

569
    global thisdir_settings
570
    # reset the options for this test directory
571 572 573
    def settings(name, opts, tempdir=tempdir, dir=dir):
        return _newTestDir(name, opts, tempdir, dir)
    thisdir_settings = settings
574

575 576 577 578

def _newTestDir(name, opts, tempdir, dir):
    opts.srcdir = os.path.join(os.getcwd(), dir)
    opts.testdir = os.path.join(tempdir, dir, name)
579
    opts.compiler_always_flags = config.compiler_always_flags
580 581 582 583

# -----------------------------------------------------------------------------
# Actually doing tests

584 585
parallelTests = []
aloneTests = []
586
allTestNames = set([])
587

588
def runTest (opts, name, func, args):
ei@vuokko.info's avatar
ei@vuokko.info committed
589 590
    ok = 0

591
    if config.use_threads:
ei@vuokko.info's avatar
ei@vuokko.info committed
592
        t.thread_pool.acquire()
593
        try:
594
            while config.threads<(t.running_threads+1):
595
                t.thread_pool.wait()
596
            t.running_threads = t.running_threads+1
597 598
            ok=1
            t.thread_pool.release()
599
            thread.start_new_thread(test_common_thread, (name, opts, func, args))
600 601 602 603 604
        except:
            if not ok:
                t.thread_pool.release()
    else:
        test_common_work (name, opts, func, args)
605

606
# name  :: String
607
# setup :: TestOpts -> IO ()
608
def test (name, setup, func, args):
609 610 611 612 613 614 615 616 617 618
    if config.run_only_some_tests:
        if name not in config.only:
            return
        else:
            # Note [Mutating config.only]
            # config.only is initiallly the set of tests requested by
            # the user (via 'make TEST='). We then remove all tests that
            # we've already seen (in .T files), so that we can later
            # report on any tests we couldn't find and error out.
            config.only.remove(name)
619

620 621
    global aloneTests
    global parallelTests
622
    global allTestNames
623
    global thisdir_settings
624 625
    if name in allTestNames:
        framework_fail(name, 'duplicate', 'There are multiple tests with this name')
626
    if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
627
        framework_fail(name, 'bad_name', 'This test has an invalid name')
628 629 630 631 632

    # Make a deep copy of the default_testopts, as we need our own copy
    # of any dictionaries etc inside it. Otherwise, if one test modifies
    # them, all tests will see the modified version!
    myTestOpts = copy.deepcopy(default_testopts)
633

634
    executeSetups([thisdir_settings, setup], name, myTestOpts)
635 636 637 638 639 640

    thisTest = lambda : runTest(myTestOpts, name, func, args)
    if myTestOpts.alone:
        aloneTests.append(thisTest)
    else:
        parallelTests.append(thisTest)
641
    allTestNames.add(name)
642

643
if config.use_threads:
644
    def test_common_thread(name, opts, func, args):
645 646 647 648 649 650
        t.lock.acquire()
        try:
            test_common_work(name,opts,func,args)
        finally:
            t.lock.release()
            t.thread_pool.acquire()
651
            t.running_threads = t.running_threads - 1
652 653
            t.thread_pool.notify()
            t.thread_pool.release()
654

655 656 657 658 659 660 661 662 663 664
def get_package_cache_timestamp():
    if config.package_conf_cache_file == '':
        return 0.0
    else:
        try:
            return os.stat(config.package_conf_cache_file).st_mtime
        except:
            return 0.0


ei@vuokko.info's avatar
ei@vuokko.info committed
665
def test_common_work (name, opts, func, args):
666 667 668 669 670 671 672 673 674
    try:
        t.total_tests = t.total_tests+1
        setLocalTestOpts(opts)

        package_conf_cache_file_start_timestamp = get_package_cache_timestamp()

        # All the ways we might run this test
        if func == compile or func == multimod_compile:
            all_ways = config.compile_ways
675
        elif func == compile_and_run or func == multimod_compile_and_run:
676 677 678 679 680 681
            all_ways = config.run_ways
        elif func == ghci_script:
            if 'ghci' in config.run_ways:
                all_ways = ['ghci']
            else:
                all_ways = []
682
        else:
683 684 685
            all_ways = ['normal']

        # A test itself can request extra ways by setting opts.extra_ways
686
        all_ways = all_ways + [way for way in opts.extra_ways if way not in all_ways]
687 688 689 690 691

        t.total_test_cases = t.total_test_cases + len(all_ways)

        ok_way = lambda way: \
            not getTestOpts().skip \
692
            and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
693
            and (config.cmdline_ways == [] or way in config.cmdline_ways) \
694
            and (not (config.skip_perf_tests and isStatsTest())) \
695 696 697
            and way not in getTestOpts().omit_ways

        # Which ways we are asked to skip
698
        do_ways = list(filter (ok_way,all_ways))
699

700 701
        # Only run all ways in slow mode.
        # See Note [validate and testsuite speed] in toplevel Makefile.
702 703
        if config.accept:
            # Only ever run one way
704
            do_ways = do_ways[:1]
705 706 707
        elif config.speed > 0:
            # However, if we EXPLICITLY asked for a way (with extra_ways)
            # please test it!
708 709
            explicit_ways = list(filter(lambda way: way in opts.extra_ways, do_ways))
            other_ways = list(filter(lambda way: way not in opts.extra_ways, do_ways))
710
            do_ways = other_ways[:1] + explicit_ways
711

712 713 714 715 716 717 718 719 720 721 722
        # Find all files in the source directory that this test
        # depends on. Do this only once for all ways.
        # Generously add all filenames that start with the name of
        # the test to this set, as a convenience to test authors.
        # They will have to use the `extra_files` setup function to
        # specify all other files that their test depends on (but
        # this seems to be necessary for only about 10% of all
        # tests).
        files = set((f for f in os.listdir(opts.srcdir)
                        if f.startswith(name)))
        for filename in (opts.extra_files + extra_src_files.get(name, [])):
723
            if filename.startswith('../../../../../../'):
724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739
                framework_fail(name, 'whole-test',
                    'add extra level to testlib.py:newTestDir for: ' + filename)

            elif filename.startswith('/'):
                framework_fail(name, 'whole-test',
                    'no absolute paths in extra_files please: ' + filename)

            elif '*' in filename:
                # Don't use wildcards in extra_files too much, as
                # globbing is slow.
                files.update((os.path.relpath(f, opts.srcdir)
                            for f in glob.iglob(in_srcdir(filename))))

            else:
                files.add(filename)

740 741 742 743 744
        # Run the required tests...
        for way in do_ways:
            if stopping():
                break
            do_test(name, way, func, args, files)
745

746 747 748
        for way in all_ways:
            if way not in do_ways:
                skiptest (name,way)
749

750
        if config.cleanup and do_ways:
751
            cleanup()
752

753
        package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
754

755 756
        if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
            framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
757

758 759 760 761 762 763 764 765 766 767
        try:
            for f in files_written[name]:
                if os.path.exists(f):
                    try:
                        if not f in files_written_not_removed[name]:
                            files_written_not_removed[name].append(f)
                    except:
                        files_written_not_removed[name] = [f]
        except:
            pass
768
    except Exception as e:
769
        framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
Ian Lynagh's avatar
Ian Lynagh committed
770

771 772 773
def do_test(name, way, func, args, files):
    opts = getTestOpts()

774 775 776
    full_name = name + '(' + way + ')'

    try:
777 778 779 780 781
        if_verbose(2, "=====> %s %d of %d %s " % \
                    (full_name, t.total_tests, len(allTestNames), \
                    [t.n_unexpected_passes, \
                     t.n_unexpected_failures, \
                     t.n_framework_failures]))
782

783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842
        # Clean up prior to the test, so that we can't spuriously conclude
        # that it passed on the basis of old run outputs.
        cleanup()

        # Link all source files for this test into a new directory in
        # /tmp, and run the test in that directory. This makes it
        # possible to run tests in parallel, without modification, that
        # would otherwise (accidentally) write to the same output file.
        # It also makes it easier to keep the testsuite clean.

        for filename in files:
            src = in_srcdir(filename)
            dst = in_testdir(filename)

            if os.path.isfile(src):
                dirname = os.path.dirname(dst)
                if dirname:
                    mkdirp(dirname)
                try:
                    link_or_copy_file(src, dst)
                except OSError as e:
                    if e.errno == errno.EEXIST and os.path.isfile(dst):
                        # Some tests depend on files from ancestor
                        # directories (e.g. '../shell.hs'). It is
                        # possible such a file was already copied over
                        # for another test, since cleanup() doesn't
                        # delete them.
                        pass
                    else:
                        raise
            elif os.path.isdir(src):
                os.makedirs(dst)
                lndir(src, dst)
            else:
                if not config.haddock and os.path.splitext(filename)[1] == '.t':
                    # When using a ghc built without haddock support, .t
                    # files are rightfully missing. Don't
                    # framework_fail. Test will be skipped later.
                    pass
                else:
                    framework_fail(name, way,
                        'extra_file does not exist: ' + filename)

        if not files:
            # Always create the testdir, even when no files were copied
            # (because user forgot to specify extra_files setup function), to
            # prevent the confusing error: can't cd to <testdir>.
            os.makedirs(opts.testdir)

        if func.__name__ == 'run_command' or opts.pre_cmd:
            # When running 'MAKE' make sure 'TOP' still points to the
            # root of the testsuite.
            src_makefile = in_srcdir('Makefile')
            dst_makefile = in_testdir('Makefile')
            if os.path.exists(src_makefile):
                with open(src_makefile, 'r') as src:
                    makefile = re.sub('TOP=.*', 'TOP=' + config.top, src.read(), 1)
                    with open(dst_makefile, 'w') as dst:
                        dst.write(makefile)

843 844
        if config.use_threads:
            t.lock.release()
845 846 847 848

        try:
            preCmd = getTestOpts().pre_cmd
            if preCmd != None:
849
                result = runCmdFor(name, 'cd "{opts.testdir}" && {preCmd}'.format(**locals()))
850 851
                if result != 0:
                    framework_fail(name, way, 'pre-command failed: ' + str(result))
852
        except:
853 854
            framework_fail(name, way, 'pre-command exception')

ei@vuokko.info's avatar
ei@vuokko.info committed
855
        try:
856
            result = func(*[name,way] + args)
ei@vuokko.info's avatar
ei@vuokko.info committed
857
        finally:
858 859
            if config.use_threads:
                t.lock.acquire()
860

861 862 863
        if getTestOpts().expect != 'pass' and \
                getTestOpts().expect != 'fail' and \
                getTestOpts().expect != 'missing-lib':
864
            framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
865

866 867 868 869 870 871
        try:
            passFail = result['passFail']
        except:
            passFail = 'No passFail found'

        if passFail == 'pass':
872
            if _expect_pass(way):
873
                t.n_expected_passes = t.n_expected_passes + 1
874 875 876 877
                if name in t.expected_passes:
                    t.expected_passes[name].append(way)
                else:
                    t.expected_passes[name] = [way]
878
            else:
879
                if_verbose(1, '*** unexpected pass for %s' % full_name)
880
                t.n_unexpected_passes = t.n_unexpected_passes + 1
881
                addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
882
        elif passFail == 'fail':
883
            if _expect_pass(way):
884
                reason = result['reason']
885 886 887 888 889 890 891 892 893
                tag = result.get('tag')
                if tag == 'stat':
                    if_verbose(1, '*** unexpected stat test failure for %s' % full_name)
                    t.n_unexpected_stat_failures = t.n_unexpected_stat_failures + 1
                    addFailingTestInfo(t.unexpected_stat_failures, getTestOpts().testdir, name, reason, way)
                else:
                    if_verbose(1, '*** unexpected failure for %s' % full_name)
                    t.n_unexpected_failures = t.n_unexpected_failures + 1
                    addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
894
            else:
895 896 897 898 899 900
                if getTestOpts().expect == 'missing-lib':
                    t.n_missing_libs = t.n_missing_libs + 1
                    if name in t.missing_libs:
                        t.missing_libs[name].append(way)
                    else:
                        t.missing_libs[name] = [way]
Simon Marlow's avatar