testlib.py 71.6 KB
Newer Older
1
# coding=utf8
2
#
3 4 5
# (c) Simon Marlow 2002
#

6
from __future__ import print_function
7

8
import io
9
import shutil
10 11
import sys
import os
12
import errno
13 14 15
import string
import re
import traceback
16 17
import time
import datetime
18
import copy
19
import glob
ian@well-typed.com's avatar
ian@well-typed.com committed
20
from math import ceil, trunc
21
import collections
22
import subprocess
23

24
from testglobals import *
25
from testutil import *
26
from extra_files import extra_src_files
27

28 29 30 31 32
try:
    basestring
except: # Python 3
    basestring = (str,bytes)

33 34
if config.use_threads:
    import threading
35 36 37 38
    try:
        import thread
    except ImportError: # Python 3
        import _thread as thread
39

40 41 42 43 44 45 46 47
global wantToStop
wantToStop = False
def stopNow():
    global wantToStop
    wantToStop = True
def stopping():
    return wantToStop

48 49
# Options valid for the current test only (these get reset to
# testdir_testopts after each test).
50

ei@vuokko.info's avatar
ei@vuokko.info committed
51
global testopts_local
52 53 54 55 56 57
if config.use_threads:
    testopts_local = threading.local()
else:
    class TestOpts_Local:
        pass
    testopts_local = TestOpts_Local()
58 59

def getTestOpts():
ei@vuokko.info's avatar
ei@vuokko.info committed
60
    return testopts_local.x
61

ei@vuokko.info's avatar
ei@vuokko.info committed
62 63 64
def setLocalTestOpts(opts):
    global testopts_local
    testopts_local.x=opts
65

66 67 68 69 70
def isStatsTest():
    opts = getTestOpts()
    return len(opts.compiler_stats_range_fields) > 0 or len(opts.stats_range_fields) > 0


71 72 73
# This can be called at the top of a file of tests, to set default test options
# for the following tests.
def setTestOpts( f ):
74
    global thisdir_settings
75
    thisdir_settings = [thisdir_settings, f]
76 77 78 79 80 81 82 83 84 85 86 87

# -----------------------------------------------------------------------------
# Canned setup functions for common cases.  eg. for a test you might say
#
#      test('test001', normal, compile, [''])
#
# to run it without any options, but change it to
#
#      test('test001', expect_fail, compile, [''])
#
# to expect failure for this test.

88
def normal( name, opts ):
89 90
    return;

91
def skip( name, opts ):
92 93
    opts.skip = 1

94
def expect_fail( name, opts ):
95 96 97
    # The compiler, testdriver, OS or platform is missing a certain
    # feature, and we don't plan to or can't fix it now or in the
    # future.
98 99
    opts.expect = 'fail';

100
def reqlib( lib ):
101
    return lambda name, opts, l=lib: _reqlib (name, opts, l )
102

103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132
def stage1(name, opts):
    # See Note [Why is there no stage1 setup function?]
    framework_fail(name, 'stage1 setup function does not exist',
                   'add your test to testsuite/tests/stage1 instead')

# Note [Why is there no stage1 setup function?]
#
# Presumably a stage1 setup function would signal that the stage1
# compiler should be used to compile a test.
#
# Trouble is, the path to the compiler + the `ghc --info` settings for
# that compiler are currently passed in from the `make` part of the
# testsuite driver.
#
# Switching compilers in the Python part would be entirely too late, as
# all ghc_with_* settings would be wrong. See config/ghc for possible
# consequences (for example, config.run_ways would still be
# based on the default compiler, quite likely causing ./validate --slow
# to fail).
#
# It would be possible to let the Python part of the testsuite driver
# make the call to `ghc --info`, but doing so would require quite some
# work. Care has to be taken to not affect the run_command tests for
# example, as they also use the `ghc --info` settings:
#     quasiquotation/qq007/Makefile:ifeq "$(GhcDynamic)" "YES"
#
# If you want a test to run using the stage1 compiler, add it to the
# testsuite/tests/stage1 directory. Validate runs the tests in that
# directory with `make stage=1`.

133 134 135 136
# Cache the results of looking to see if we have a library or not.
# This makes quite a difference, especially on Windows.
have_lib = {}

137
def _reqlib( name, opts, lib ):
138
    if lib in have_lib:
139
        got_it = have_lib[lib]
140
    else:
141 142 143 144 145 146 147 148
        cmd = strip_quotes(config.ghc_pkg)
        p = subprocess.Popen([cmd, '--no-user-package-db', 'describe', lib],
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
        # read from stdout and stderr to avoid blocking due to
        # buffers filling
        p.communicate()
        r = p.wait()
149 150 151 152
        got_it = r == 0
        have_lib[lib] = got_it

    if not got_it:
153
        opts.expect = 'missing-lib'
154

155 156 157 158
def req_haddock( name, opts ):
    if not config.haddock:
        opts.expect = 'missing-lib'

159
def req_profiling( name, opts ):
160
    '''Require the profiling libraries (add 'GhcLibWays += p' to mk/build.mk)'''
161 162 163
    if not config.have_profiling:
        opts.expect = 'fail'

164
def req_shared_libs( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
165 166 167
    if not config.have_shared_libs:
        opts.expect = 'fail'

168
def req_interp( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
169 170 171
    if not config.have_interp:
        opts.expect = 'fail'

172
def req_smp( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
173 174 175
    if not config.have_smp:
        opts.expect = 'fail'

176
def ignore_output( name, opts ):
177 178
    opts.ignore_output = 1

179
def no_stdin( name, opts ):
180 181
    opts.no_stdin = 1

182
def combined_output( name, opts ):
pcapriotti's avatar
pcapriotti committed
183 184
    opts.combined_output = True

185 186 187
# -----

def expect_fail_for( ways ):
188
    return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
189

190
def _expect_fail_for( name, opts, ways ):
191 192
    opts.expect_fail_for = ways

193
def expect_broken( bug ):
194 195
    # This test is a expected not to work due to the indicated trac bug
    # number.
196 197 198
    return lambda name, opts, b=bug: _expect_broken (name, opts, b )

def _expect_broken( name, opts, bug ):
199
    record_broken(name, opts, bug)
200 201
    opts.expect = 'fail';

Ian Lynagh's avatar
Ian Lynagh committed
202
def expect_broken_for( bug, ways ):
203
    return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
Ian Lynagh's avatar
Ian Lynagh committed
204

205
def _expect_broken_for( name, opts, bug, ways ):
206
    record_broken(name, opts, bug)
Ian Lynagh's avatar
Ian Lynagh committed
207
    opts.expect_fail_for = ways
Ian Lynagh's avatar
Ian Lynagh committed
208

209
def record_broken(name, opts, bug):
210
    global brokens
211
    me = (bug, opts.testdir, name)
212 213 214
    if not me in brokens:
        brokens.append(me)

215 216 217 218 219
def _expect_pass(way):
    # Helper function. Not intended for use in .T files.
    opts = getTestOpts()
    return opts.expect == 'pass' and way not in opts.expect_fail_for

220 221 222
# -----

def omit_ways( ways ):
223
    return lambda name, opts, w=ways: _omit_ways( name, opts, w )
224

225
def _omit_ways( name, opts, ways ):
226 227 228 229
    opts.omit_ways = ways

# -----

230
def only_ways( ways ):
231
    return lambda name, opts, w=ways: _only_ways( name, opts, w )
232

233
def _only_ways( name, opts, ways ):
234 235 236 237
    opts.only_ways = ways

# -----

238
def extra_ways( ways ):
239
    return lambda name, opts, w=ways: _extra_ways( name, opts, w )
240

241
def _extra_ways( name, opts, ways ):
242 243 244 245
    opts.extra_ways = ways

# -----

246
def set_stdin( file ):
247
   return lambda name, opts, f=file: _set_stdin(name, opts, f);
248

249
def _set_stdin( name, opts, f ):
250 251 252 253 254
   opts.stdin = f

# -----

def exit_code( val ):
255
    return lambda name, opts, v=val: _exit_code(name, opts, v);
256

257
def _exit_code( name, opts, v ):
258 259
    opts.exit_code = v

260 261 262 263 264 265 266 267 268 269 270
def signal_exit_code( val ):
    if opsys('solaris2'):
        return exit_code( val );
    else:
        # When application running on Linux receives fatal error
        # signal, then its exit code is encoded as 128 + signal
        # value. See http://www.tldp.org/LDP/abs/html/exitcodes.html
        # I assume that Mac OS X behaves in the same way at least Mac
        # OS X builder behavior suggests this.
        return exit_code( val+128 );

271 272
# -----

273 274
def compile_timeout_multiplier( val ):
    return lambda name, opts, v=val: _compile_timeout_multiplier(name, opts, v)
275

276 277 278 279 280 281 282 283
def _compile_timeout_multiplier( name, opts, v ):
    opts.compile_timeout_multiplier = v

def run_timeout_multiplier( val ):
    return lambda name, opts, v=val: _run_timeout_multiplier(name, opts, v)

def _run_timeout_multiplier( name, opts, v ):
    opts.run_timeout_multiplier = v
284 285 286

# -----

287
def extra_run_opts( val ):
288
    return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
289

290
def _extra_run_opts( name, opts, v ):
291 292
    opts.extra_run_opts = v

293 294
# -----

Simon Marlow's avatar
Simon Marlow committed
295
def extra_hc_opts( val ):
296
    return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
Simon Marlow's avatar
Simon Marlow committed
297

298
def _extra_hc_opts( name, opts, v ):
Simon Marlow's avatar
Simon Marlow committed
299 300 301 302
    opts.extra_hc_opts = v

# -----

303
def extra_clean( files ):
304 305
    # TODO. Remove all calls to extra_clean.
    return lambda _name, _opts: None
306

307 308 309 310 311 312
def extra_files(files):
    return lambda name, opts: _extra_files(name, opts, files)

def _extra_files(name, opts, files):
    opts.extra_files.extend(files)

313 314
# -----

315 316 317 318 319 320 321
def stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);

def _stats_num_field( name, opts, field, expecteds ):
    if field in opts.stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

322
    if type(expecteds) is list:
323 324 325 326 327
        for (b, expected, dev) in expecteds:
            if b:
                opts.stats_range_fields[field] = (expected, dev)
                return
        framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
328

329 330 331
    else:
        (expected, dev) = expecteds
        opts.stats_range_fields[field] = (expected, dev)
332 333 334

def compiler_stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
335

336 337 338 339
def _compiler_stats_num_field( name, opts, field, expecteds ):
    if field in opts.compiler_stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

340 341
    # Compiler performance numbers change when debugging is on, making the results
    # useless and confusing. Therefore, skip if debugging is on.
342 343 344
    if compiler_debugged():
        skip(name, opts)

345 346 347 348
    for (b, expected, dev) in expecteds:
        if b:
            opts.compiler_stats_range_fields[field] = (expected, dev)
            return
349

350 351
    framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')

352 353
# -----

354
def when(b, f):
ian@well-typed.com's avatar
ian@well-typed.com committed
355 356 357
    # When list_brokens is on, we want to see all expect_broken calls,
    # so we always do f
    if b or config.list_broken:
358 359 360 361 362 363 364
        return f
    else:
        return normal

def unless(b, f):
    return when(not b, f)

ian@well-typed.com's avatar
ian@well-typed.com committed
365 366 367
def doing_ghci():
    return 'ghci' in config.run_ways

368
def ghc_dynamic():
ian@well-typed.com's avatar
ian@well-typed.com committed
369
    return config.ghc_dynamic
370 371

def fast():
372
    return config.speed == 2
373

374 375
def platform( plat ):
    return config.platform == plat
Ian Lynagh's avatar
Ian Lynagh committed
376

377 378
def opsys( os ):
    return config.os == os
Ian Lynagh's avatar
Ian Lynagh committed
379

380 381
def arch( arch ):
    return config.arch == arch
382

383 384
def wordsize( ws ):
    return config.wordsize == str(ws)
tibbe's avatar
tibbe committed
385

386 387
def msys( ):
    return config.msys
ian@well-typed.com's avatar
ian@well-typed.com committed
388

389 390
def cygwin( ):
    return config.cygwin
Ian Lynagh's avatar
Ian Lynagh committed
391

392 393
def have_vanilla( ):
    return config.have_vanilla
394

395 396
def have_dynamic( ):
    return config.have_dynamic
397

398 399
def have_profiling( ):
    return config.have_profiling
400

401 402
def in_tree_compiler( ):
    return config.in_tree_compiler
403

404 405 406 407 408 409
def unregisterised( ):
    return config.unregisterised

def compiler_profiled( ):
    return config.compiler_profiled

410 411
def compiler_debugged( ):
    return config.compiler_debugged
412

413 414
# ---

415
def high_memory_usage(name, opts):
416 417
    opts.alone = True

418 419 420 421 422
# If a test is for a multi-CPU race, then running the test alone
# increases the chance that we'll actually see it.
def multi_cpu_race(name, opts):
    opts.alone = True

Ian Lynagh's avatar
Ian Lynagh committed
423
# ---
424
def literate( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
425 426
    opts.literate = 1;

427
def c_src( name, opts ):
428 429
    opts.c_src = 1;

430
def objc_src( name, opts ):
Austin Seipp's avatar
Austin Seipp committed
431 432
    opts.objc_src = 1;

433
def objcpp_src( name, opts ):
434 435
    opts.objcpp_src = 1;

436
def cmm_src( name, opts ):
437 438
    opts.cmm_src = 1;

439
def outputdir( odir ):
440
    return lambda name, opts, d=odir: _outputdir(name, opts, d)
441

442
def _outputdir( name, opts, odir ):
443 444
    opts.outputdir = odir;

445 446
# ----

447
def pre_cmd( cmd ):
448
    return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
449

450
def _pre_cmd( name, opts, cmd ):
451 452 453 454
    opts.pre_cmd = cmd

# ----

455
def clean_cmd( cmd ):
456 457
    # TODO. Remove all calls to clean_cmd.
    return lambda _name, _opts: None
458 459 460

# ----

461
def cmd_prefix( prefix ):
462
    return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
463

464
def _cmd_prefix( name, opts, prefix ):
465 466 467 468 469
    opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;

# ----

def cmd_wrapper( fun ):
470
    return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
471

472
def _cmd_wrapper( name, opts, fun ):
473
    opts.cmd_wrapper = fun
474

475 476
# ----

Ian Lynagh's avatar
Ian Lynagh committed
477
def compile_cmd_prefix( prefix ):
478
    return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
Ian Lynagh's avatar
Ian Lynagh committed
479

480
def _compile_cmd_prefix( name, opts, prefix ):
Ian Lynagh's avatar
Ian Lynagh committed
481 482 483 484
    opts.compile_cmd_prefix = prefix

# ----

485 486 487 488 489 490 491 492
def check_stdout( f ):
    return lambda name, opts, f=f: _check_stdout(name, opts, f)

def _check_stdout( name, opts, f ):
    opts.check_stdout = f

# ----

493
def normalise_slashes( name, opts ):
494
    _normalise_fun(name, opts, normalise_slashes_)
495

496
def normalise_exe( name, opts ):
497
    _normalise_fun(name, opts, normalise_exe_)
498

499 500
def normalise_fun( *fs ):
    return lambda name, opts: _normalise_fun(name, opts, fs)
501

502
def _normalise_fun( name, opts, *fs ):
503
    opts.extra_normaliser = join_normalisers(opts.extra_normaliser, fs)
504

505 506
def normalise_errmsg_fun( *fs ):
    return lambda name, opts: _normalise_errmsg_fun(name, opts, fs)
507

508
def _normalise_errmsg_fun( name, opts, *fs ):
509 510 511 512 513 514 515 516 517 518 519 520 521
    opts.extra_errmsg_normaliser =  join_normalisers(opts.extra_errmsg_normaliser, fs)

def normalise_version_( *pkgs ):
    def normalise_version__( str ):
        return re.sub('(' + '|'.join(map(re.escape,pkgs)) + ')-[0-9.]+',
                      '\\1-<VERSION>', str)
    return normalise_version__

def normalise_version( *pkgs ):
    def normalise_version__( name, opts ):
        _normalise_fun(name, opts, normalise_version_(*pkgs))
        _normalise_errmsg_fun(name, opts, normalise_version_(*pkgs))
    return normalise_version__
522

thomie's avatar
thomie committed
523 524 525 526
def normalise_drive_letter(name, opts):
    # Windows only. Change D:\\ to C:\\.
    _normalise_fun(name, opts, lambda str: re.sub(r'[A-Z]:\\', r'C:\\', str))

527 528 529 530 531 532 533
def keep_prof_callstacks(name, opts):
    """Keep profiling callstacks.

    Use together with `only_ways(prof_ways)`.
    """
    opts.keep_prof_callstacks = True

534 535
def join_normalisers(*a):
    """
536
    Compose functions, flattening sequences.
537

538
       join_normalisers(f1,[f2,f3],f4)
539 540 541

    is the same as

542
       lambda x: f1(f2(f3(f4(x))))
543 544
    """

545 546 547 548 549 550 551 552 553 554 555 556
    def flatten(l):
        """
        Taken from http://stackoverflow.com/a/2158532/946226
        """
        for el in l:
            if isinstance(el, collections.Iterable) and not isinstance(el, basestring):
                for sub in flatten(el):
                    yield sub
            else:
                yield el

    a = flatten(a)
557 558 559

    fn = lambda x:x # identity function
    for f in a:
560
        assert callable(f)
561 562 563
        fn = lambda x,f=f,fn=fn: fn(f(x))
    return fn

564 565 566
# ----
# Function for composing two opt-fns together

567
def executeSetups(fs, name, opts):
568
    if type(fs) is list:
569
        # If we have a list of setups, then execute each one
570 571
        for f in fs:
            executeSetups(f, name, opts)
572 573 574
    else:
        # fs is a single function, so just apply it
        fs(name, opts)
575

576 577 578
# -----------------------------------------------------------------------------
# The current directory of tests

579 580
def newTestDir(tempdir, dir):

581
    global thisdir_settings
582
    # reset the options for this test directory
583 584 585
    def settings(name, opts, tempdir=tempdir, dir=dir):
        return _newTestDir(name, opts, tempdir, dir)
    thisdir_settings = settings
586

587 588
# Should be equal to entry in toplevel .gitignore.
testdir_suffix = '.run'
589 590 591

def _newTestDir(name, opts, tempdir, dir):
    opts.srcdir = os.path.join(os.getcwd(), dir)
592
    opts.testdir = os.path.join(tempdir, dir, name + testdir_suffix)
593
    opts.compiler_always_flags = config.compiler_always_flags
594 595 596 597

# -----------------------------------------------------------------------------
# Actually doing tests

598 599
parallelTests = []
aloneTests = []
600
allTestNames = set([])
601

602
def runTest (opts, name, func, args):
ei@vuokko.info's avatar
ei@vuokko.info committed
603 604
    ok = 0

605
    if config.use_threads:
ei@vuokko.info's avatar
ei@vuokko.info committed
606
        t.thread_pool.acquire()
607
        try:
608
            while config.threads<(t.running_threads+1):
609
                t.thread_pool.wait()
610
            t.running_threads = t.running_threads+1
611 612
            ok=1
            t.thread_pool.release()
613
            thread.start_new_thread(test_common_thread, (name, opts, func, args))
614 615 616 617 618
        except:
            if not ok:
                t.thread_pool.release()
    else:
        test_common_work (name, opts, func, args)
619

620
# name  :: String
621
# setup :: TestOpts -> IO ()
622
def test (name, setup, func, args):
623 624 625 626 627 628 629 630 631 632
    if config.run_only_some_tests:
        if name not in config.only:
            return
        else:
            # Note [Mutating config.only]
            # config.only is initiallly the set of tests requested by
            # the user (via 'make TEST='). We then remove all tests that
            # we've already seen (in .T files), so that we can later
            # report on any tests we couldn't find and error out.
            config.only.remove(name)
633

634 635
    global aloneTests
    global parallelTests
636
    global allTestNames
637
    global thisdir_settings
638 639
    if name in allTestNames:
        framework_fail(name, 'duplicate', 'There are multiple tests with this name')
640
    if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
641
        framework_fail(name, 'bad_name', 'This test has an invalid name')
642 643 644 645 646

    # Make a deep copy of the default_testopts, as we need our own copy
    # of any dictionaries etc inside it. Otherwise, if one test modifies
    # them, all tests will see the modified version!
    myTestOpts = copy.deepcopy(default_testopts)
647

648
    executeSetups([thisdir_settings, setup], name, myTestOpts)
649 650 651 652 653 654

    thisTest = lambda : runTest(myTestOpts, name, func, args)
    if myTestOpts.alone:
        aloneTests.append(thisTest)
    else:
        parallelTests.append(thisTest)
655
    allTestNames.add(name)
656

657
if config.use_threads:
658
    def test_common_thread(name, opts, func, args):
659 660 661 662 663 664
        t.lock.acquire()
        try:
            test_common_work(name,opts,func,args)
        finally:
            t.lock.release()
            t.thread_pool.acquire()
665
            t.running_threads = t.running_threads - 1
666 667
            t.thread_pool.notify()
            t.thread_pool.release()
668

669 670 671 672 673 674 675 676 677 678
def get_package_cache_timestamp():
    if config.package_conf_cache_file == '':
        return 0.0
    else:
        try:
            return os.stat(config.package_conf_cache_file).st_mtime
        except:
            return 0.0


ei@vuokko.info's avatar
ei@vuokko.info committed
679
def test_common_work (name, opts, func, args):
680 681 682 683 684 685 686 687 688
    try:
        t.total_tests = t.total_tests+1
        setLocalTestOpts(opts)

        package_conf_cache_file_start_timestamp = get_package_cache_timestamp()

        # All the ways we might run this test
        if func == compile or func == multimod_compile:
            all_ways = config.compile_ways
689
        elif func == compile_and_run or func == multimod_compile_and_run:
690 691 692 693 694 695
            all_ways = config.run_ways
        elif func == ghci_script:
            if 'ghci' in config.run_ways:
                all_ways = ['ghci']
            else:
                all_ways = []
696
        else:
697 698 699
            all_ways = ['normal']

        # A test itself can request extra ways by setting opts.extra_ways
700
        all_ways = all_ways + [way for way in opts.extra_ways if way not in all_ways]
701 702 703 704 705

        t.total_test_cases = t.total_test_cases + len(all_ways)

        ok_way = lambda way: \
            not getTestOpts().skip \
706
            and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
707
            and (config.cmdline_ways == [] or way in config.cmdline_ways) \
708
            and (not (config.skip_perf_tests and isStatsTest())) \
709 710 711
            and way not in getTestOpts().omit_ways

        # Which ways we are asked to skip
712
        do_ways = list(filter (ok_way,all_ways))
713

714 715
        # Only run all ways in slow mode.
        # See Note [validate and testsuite speed] in toplevel Makefile.
716 717
        if config.accept:
            # Only ever run one way
718
            do_ways = do_ways[:1]
719 720 721
        elif config.speed > 0:
            # However, if we EXPLICITLY asked for a way (with extra_ways)
            # please test it!
722 723
            explicit_ways = list(filter(lambda way: way in opts.extra_ways, do_ways))
            other_ways = list(filter(lambda way: way not in opts.extra_ways, do_ways))
724
            do_ways = other_ways[:1] + explicit_ways
725

726 727 728 729 730 731 732 733 734
        # Find all files in the source directory that this test
        # depends on. Do this only once for all ways.
        # Generously add all filenames that start with the name of
        # the test to this set, as a convenience to test authors.
        # They will have to use the `extra_files` setup function to
        # specify all other files that their test depends on (but
        # this seems to be necessary for only about 10% of all
        # tests).
        files = set((f for f in os.listdir(opts.srcdir)
735 736
                        if f.startswith(name) and
                           not f.endswith(testdir_suffix)))
737
        for filename in (opts.extra_files + extra_src_files.get(name, [])):
738
            if filename.startswith('/'):
739 740 741 742 743 744 745 746 747 748 749 750
                framework_fail(name, 'whole-test',
                    'no absolute paths in extra_files please: ' + filename)

            elif '*' in filename:
                # Don't use wildcards in extra_files too much, as
                # globbing is slow.
                files.update((os.path.relpath(f, opts.srcdir)
                            for f in glob.iglob(in_srcdir(filename))))

            else:
                files.add(filename)

751 752 753 754 755
        # Run the required tests...
        for way in do_ways:
            if stopping():
                break
            do_test(name, way, func, args, files)
756

757 758 759
        for way in all_ways:
            if way not in do_ways:
                skiptest (name,way)
760

761
        if config.cleanup and do_ways:
762
            cleanup()
763

764
        package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
765

766 767
        if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
            framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
768

769 770 771 772 773 774 775 776 777 778
        try:
            for f in files_written[name]:
                if os.path.exists(f):
                    try:
                        if not f in files_written_not_removed[name]:
                            files_written_not_removed[name].append(f)
                    except:
                        files_written_not_removed[name] = [f]
        except:
            pass
779
    except Exception as e:
780
        framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
Ian Lynagh's avatar
Ian Lynagh committed
781

782 783 784
def do_test(name, way, func, args, files):
    opts = getTestOpts()

785 786 787
    full_name = name + '(' + way + ')'

    try:
788 789 790 791 792
        if_verbose(2, "=====> %s %d of %d %s " % \
                    (full_name, t.total_tests, len(allTestNames), \
                    [t.n_unexpected_passes, \
                     t.n_unexpected_failures, \
                     t.n_framework_failures]))
793

794 795 796 797 798 799 800 801 802 803
        # Clean up prior to the test, so that we can't spuriously conclude
        # that it passed on the basis of old run outputs.
        cleanup()

        # Link all source files for this test into a new directory in
        # /tmp, and run the test in that directory. This makes it
        # possible to run tests in parallel, without modification, that
        # would otherwise (accidentally) write to the same output file.
        # It also makes it easier to keep the testsuite clean.

804 805 806 807 808 809 810 811 812 813 814 815
        for extra_file in files:
            src = in_srcdir(extra_file)
            if extra_file.startswith('..'):
                # In case the extra_file is a file in an ancestor
                # directory (e.g. extra_files(['../shell.hs'])), make
                # sure it is copied to the test directory
                # (testdir/shell.hs), instead of ending up somewhere
                # else in the tree (testdir/../shell.hs)
                filename = os.path.basename(extra_file)
            else:
                filename = extra_file
            assert not '..' in filename # no funny stuff (foo/../../bar)
816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844
            dst = in_testdir(filename)

            if os.path.isfile(src):
                dirname = os.path.dirname(dst)
                if dirname:
                    mkdirp(dirname)
                try:
                    link_or_copy_file(src, dst)
                except OSError as e:
                    if e.errno == errno.EEXIST and os.path.isfile(dst):
                        # Some tests depend on files from ancestor
                        # directories (e.g. '../shell.hs'). It is
                        # possible such a file was already copied over
                        # for another test, since cleanup() doesn't
                        # delete them.
                        pass
                    else:
                        raise
            elif os.path.isdir(src):
                os.makedirs(dst)
                lndir(src, dst)
            else:
                if not config.haddock and os.path.splitext(filename)[1] == '.t':
                    # When using a ghc built without haddock support, .t
                    # files are rightfully missing. Don't
                    # framework_fail. Test will be skipped later.
                    pass
                else:
                    framework_fail(name, way,
845
                        'extra_file does not exist: ' + extra_file)
846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863

        if not files:
            # Always create the testdir, even when no files were copied
            # (because user forgot to specify extra_files setup function), to
            # prevent the confusing error: can't cd to <testdir>.
            os.makedirs(opts.testdir)

        if func.__name__ == 'run_command' or opts.pre_cmd:
            # When running 'MAKE' make sure 'TOP' still points to the
            # root of the testsuite.
            src_makefile = in_srcdir('Makefile')
            dst_makefile = in_testdir('Makefile')
            if os.path.exists(src_makefile):
                with open(src_makefile, 'r') as src:
                    makefile = re.sub('TOP=.*', 'TOP=' + config.top, src.read(), 1)
                    with open(dst_makefile, 'w') as dst:
                        dst.write(makefile)

864 865
        if config.use_threads:
            t.lock.release()
866 867 868 869

        try:
            preCmd = getTestOpts().pre_cmd
            if preCmd != None:
870
                result = runCmdFor(name, 'cd "{opts.testdir}" && {preCmd}'.format(**locals()))
871 872
                if result != 0:
                    framework_fail(name, way, 'pre-command failed: ' + str(result))
873
        except:
874 875
            framework_fail(name, way, 'pre-command exception')

ei@vuokko.info's avatar
ei@vuokko.info committed
876
        try:
877
            result = func(*[name,way] + args)
ei@vuokko.info's avatar
ei@vuokko.info committed
878
        finally:
879 880
            if config.use_threads:
                t.lock.acquire()
881

882 883 884
        if getTestOpts().expect != 'pass' and \
                getTestOpts().expect != 'fail' and \
                getTestOpts().expect != 'missing-lib':
885
            framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
886

887 888 889 890 891 892
        try:
            passFail = result['passFail']
        except:
            passFail = 'No passFail found'

        if passFail == 'pass':
893
            if _expect_pass(way):
894
                t.n_expected_passes = t.n_expected_passes + 1
895 896 897 898
                if name in t.expected_passes:
                    t.expected_passes[name].append(way)
                else:
                    t.expected_passes[name] = [way]
899
            else:
900
                if_verbose(1, '*** unexpected pass for %s' % full_name)
901
                t.n_unexpected_passes = t.n_unexpected_passes + 1
902