testlib.py 71.5 KB
Newer Older
1
# coding=utf8
2
#
3 4 5
# (c) Simon Marlow 2002
#

6
from __future__ import print_function
7

8
import io
9
import shutil
10 11
import sys
import os
12
import errno
13 14 15
import string
import re
import traceback
16 17
import time
import datetime
18
import copy
19
import glob
ian@well-typed.com's avatar
ian@well-typed.com committed
20
from math import ceil, trunc
21
import collections
22
import subprocess
23

24
from testglobals import *
25
from testutil import *
26
from extra_files import extra_src_files
27

28 29 30 31 32
try:
    basestring
except: # Python 3
    basestring = (str,bytes)

33 34
if config.use_threads:
    import threading
35 36 37 38
    try:
        import thread
    except ImportError: # Python 3
        import _thread as thread
39

40 41 42 43 44 45 46 47
global wantToStop
wantToStop = False
def stopNow():
    global wantToStop
    wantToStop = True
def stopping():
    return wantToStop

48 49
# Options valid for the current test only (these get reset to
# testdir_testopts after each test).
50

ei@vuokko.info's avatar
ei@vuokko.info committed
51
global testopts_local
52 53 54 55 56 57
if config.use_threads:
    testopts_local = threading.local()
else:
    class TestOpts_Local:
        pass
    testopts_local = TestOpts_Local()
58 59

def getTestOpts():
ei@vuokko.info's avatar
ei@vuokko.info committed
60
    return testopts_local.x
61

ei@vuokko.info's avatar
ei@vuokko.info committed
62 63 64
def setLocalTestOpts(opts):
    global testopts_local
    testopts_local.x=opts
65

66 67 68 69 70
def isStatsTest():
    opts = getTestOpts()
    return len(opts.compiler_stats_range_fields) > 0 or len(opts.stats_range_fields) > 0


71 72 73
# This can be called at the top of a file of tests, to set default test options
# for the following tests.
def setTestOpts( f ):
74
    global thisdir_settings
75
    thisdir_settings = [thisdir_settings, f]
76 77 78 79 80 81 82 83 84 85 86 87

# -----------------------------------------------------------------------------
# Canned setup functions for common cases.  eg. for a test you might say
#
#      test('test001', normal, compile, [''])
#
# to run it without any options, but change it to
#
#      test('test001', expect_fail, compile, [''])
#
# to expect failure for this test.

88
def normal( name, opts ):
89 90
    return;

91
def skip( name, opts ):
92 93
    opts.skip = 1

94
def expect_fail( name, opts ):
95 96 97
    # The compiler, testdriver, OS or platform is missing a certain
    # feature, and we don't plan to or can't fix it now or in the
    # future.
98 99
    opts.expect = 'fail';

100
def reqlib( lib ):
101
    return lambda name, opts, l=lib: _reqlib (name, opts, l )
102

103 104 105 106
# Cache the results of looking to see if we have a library or not.
# This makes quite a difference, especially on Windows.
have_lib = {}

107
def _reqlib( name, opts, lib ):
108
    if lib in have_lib:
109
        got_it = have_lib[lib]
110
    else:
111 112 113 114 115 116 117 118
        cmd = strip_quotes(config.ghc_pkg)
        p = subprocess.Popen([cmd, '--no-user-package-db', 'describe', lib],
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
        # read from stdout and stderr to avoid blocking due to
        # buffers filling
        p.communicate()
        r = p.wait()
119 120 121 122
        got_it = r == 0
        have_lib[lib] = got_it

    if not got_it:
123
        opts.expect = 'missing-lib'
124

125 126 127 128
def req_haddock( name, opts ):
    if not config.haddock:
        opts.expect = 'missing-lib'

129
def req_profiling( name, opts ):
130
    '''Require the profiling libraries (add 'GhcLibWays += p' to mk/build.mk)'''
131 132 133
    if not config.have_profiling:
        opts.expect = 'fail'

134
def req_shared_libs( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
135 136 137
    if not config.have_shared_libs:
        opts.expect = 'fail'

138
def req_interp( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
139 140 141
    if not config.have_interp:
        opts.expect = 'fail'

142
def req_smp( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
143 144 145
    if not config.have_smp:
        opts.expect = 'fail'

146
def ignore_output( name, opts ):
147 148
    opts.ignore_output = 1

149
def no_stdin( name, opts ):
150 151
    opts.no_stdin = 1

152
def combined_output( name, opts ):
pcapriotti's avatar
pcapriotti committed
153 154
    opts.combined_output = True

155 156 157
# -----

def expect_fail_for( ways ):
158
    return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
159

160
def _expect_fail_for( name, opts, ways ):
161 162
    opts.expect_fail_for = ways

163
def expect_broken( bug ):
164 165
    # This test is a expected not to work due to the indicated trac bug
    # number.
166 167 168
    return lambda name, opts, b=bug: _expect_broken (name, opts, b )

def _expect_broken( name, opts, bug ):
169
    record_broken(name, opts, bug)
170 171
    opts.expect = 'fail';

Ian Lynagh's avatar
Ian Lynagh committed
172
def expect_broken_for( bug, ways ):
173
    return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
Ian Lynagh's avatar
Ian Lynagh committed
174

175
def _expect_broken_for( name, opts, bug, ways ):
176
    record_broken(name, opts, bug)
Ian Lynagh's avatar
Ian Lynagh committed
177
    opts.expect_fail_for = ways
Ian Lynagh's avatar
Ian Lynagh committed
178

179
def record_broken(name, opts, bug):
180
    global brokens
181
    me = (bug, opts.testdir, name)
182 183 184
    if not me in brokens:
        brokens.append(me)

185 186 187 188 189
def _expect_pass(way):
    # Helper function. Not intended for use in .T files.
    opts = getTestOpts()
    return opts.expect == 'pass' and way not in opts.expect_fail_for

190 191 192
# -----

def omit_ways( ways ):
193
    return lambda name, opts, w=ways: _omit_ways( name, opts, w )
194

195
def _omit_ways( name, opts, ways ):
196 197 198 199
    opts.omit_ways = ways

# -----

200
def only_ways( ways ):
201
    return lambda name, opts, w=ways: _only_ways( name, opts, w )
202

203
def _only_ways( name, opts, ways ):
204 205 206 207
    opts.only_ways = ways

# -----

208
def extra_ways( ways ):
209
    return lambda name, opts, w=ways: _extra_ways( name, opts, w )
210

211
def _extra_ways( name, opts, ways ):
212 213 214 215
    opts.extra_ways = ways

# -----

216
def set_stdin( file ):
217
   return lambda name, opts, f=file: _set_stdin(name, opts, f);
218

219
def _set_stdin( name, opts, f ):
220 221 222 223 224
   opts.stdin = f

# -----

def exit_code( val ):
225
    return lambda name, opts, v=val: _exit_code(name, opts, v);
226

227
def _exit_code( name, opts, v ):
228 229
    opts.exit_code = v

230 231 232 233 234 235 236 237 238 239 240
def signal_exit_code( val ):
    if opsys('solaris2'):
        return exit_code( val );
    else:
        # When application running on Linux receives fatal error
        # signal, then its exit code is encoded as 128 + signal
        # value. See http://www.tldp.org/LDP/abs/html/exitcodes.html
        # I assume that Mac OS X behaves in the same way at least Mac
        # OS X builder behavior suggests this.
        return exit_code( val+128 );

241 242
# -----

243 244
def compile_timeout_multiplier( val ):
    return lambda name, opts, v=val: _compile_timeout_multiplier(name, opts, v)
245

246 247 248 249 250 251 252 253
def _compile_timeout_multiplier( name, opts, v ):
    opts.compile_timeout_multiplier = v

def run_timeout_multiplier( val ):
    return lambda name, opts, v=val: _run_timeout_multiplier(name, opts, v)

def _run_timeout_multiplier( name, opts, v ):
    opts.run_timeout_multiplier = v
254 255 256

# -----

257
def extra_run_opts( val ):
258
    return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
259

260
def _extra_run_opts( name, opts, v ):
261 262
    opts.extra_run_opts = v

263 264
# -----

Simon Marlow's avatar
Simon Marlow committed
265
def extra_hc_opts( val ):
266
    return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
Simon Marlow's avatar
Simon Marlow committed
267

268
def _extra_hc_opts( name, opts, v ):
Simon Marlow's avatar
Simon Marlow committed
269 270 271 272
    opts.extra_hc_opts = v

# -----

273
def extra_clean( files ):
274 275
    # TODO. Remove all calls to extra_clean.
    return lambda _name, _opts: None
276

277 278 279 280 281 282
def extra_files(files):
    return lambda name, opts: _extra_files(name, opts, files)

def _extra_files(name, opts, files):
    opts.extra_files.extend(files)

283 284
# -----

285 286 287 288 289 290 291
def stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);

def _stats_num_field( name, opts, field, expecteds ):
    if field in opts.stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

292
    if type(expecteds) is list:
293 294 295 296 297
        for (b, expected, dev) in expecteds:
            if b:
                opts.stats_range_fields[field] = (expected, dev)
                return
        framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
298

299 300 301
    else:
        (expected, dev) = expecteds
        opts.stats_range_fields[field] = (expected, dev)
302 303 304

def compiler_stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
305

306 307 308 309
def _compiler_stats_num_field( name, opts, field, expecteds ):
    if field in opts.compiler_stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

310 311
    # Compiler performance numbers change when debugging is on, making the results
    # useless and confusing. Therefore, skip if debugging is on.
312 313 314
    if compiler_debugged():
        skip(name, opts)

315 316 317 318
    for (b, expected, dev) in expecteds:
        if b:
            opts.compiler_stats_range_fields[field] = (expected, dev)
            return
319

320 321
    framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')

322 323
# -----

324
def when(b, f):
ian@well-typed.com's avatar
ian@well-typed.com committed
325 326 327
    # When list_brokens is on, we want to see all expect_broken calls,
    # so we always do f
    if b or config.list_broken:
328 329 330 331 332 333 334
        return f
    else:
        return normal

def unless(b, f):
    return when(not b, f)

ian@well-typed.com's avatar
ian@well-typed.com committed
335 336 337
def doing_ghci():
    return 'ghci' in config.run_ways

338
def ghc_dynamic():
ian@well-typed.com's avatar
ian@well-typed.com committed
339
    return config.ghc_dynamic
340 341

def fast():
342
    return config.speed == 2
343

344 345
def platform( plat ):
    return config.platform == plat
Ian Lynagh's avatar
Ian Lynagh committed
346

347 348
def opsys( os ):
    return config.os == os
Ian Lynagh's avatar
Ian Lynagh committed
349

350 351
def arch( arch ):
    return config.arch == arch
352

353 354
def wordsize( ws ):
    return config.wordsize == str(ws)
tibbe's avatar
tibbe committed
355

356 357
def msys( ):
    return config.msys
ian@well-typed.com's avatar
ian@well-typed.com committed
358

359 360
def cygwin( ):
    return config.cygwin
Ian Lynagh's avatar
Ian Lynagh committed
361

362 363
def have_vanilla( ):
    return config.have_vanilla
364

365 366
def have_dynamic( ):
    return config.have_dynamic
367

368 369
def have_profiling( ):
    return config.have_profiling
370

371 372
def in_tree_compiler( ):
    return config.in_tree_compiler
373

374 375 376 377 378 379
def unregisterised( ):
    return config.unregisterised

def compiler_profiled( ):
    return config.compiler_profiled

380 381
def compiler_debugged( ):
    return config.compiler_debugged
382

383 384
# ---

385
def high_memory_usage(name, opts):
386 387
    opts.alone = True

388 389 390 391 392
# If a test is for a multi-CPU race, then running the test alone
# increases the chance that we'll actually see it.
def multi_cpu_race(name, opts):
    opts.alone = True

Ian Lynagh's avatar
Ian Lynagh committed
393
# ---
394
def literate( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
395 396
    opts.literate = 1;

397
def c_src( name, opts ):
398 399
    opts.c_src = 1;

400
def objc_src( name, opts ):
Austin Seipp's avatar
Austin Seipp committed
401 402
    opts.objc_src = 1;

403
def objcpp_src( name, opts ):
404 405
    opts.objcpp_src = 1;

406
def cmm_src( name, opts ):
407 408
    opts.cmm_src = 1;

409
def outputdir( odir ):
410
    return lambda name, opts, d=odir: _outputdir(name, opts, d)
411

412
def _outputdir( name, opts, odir ):
413 414
    opts.outputdir = odir;

415 416
# ----

417
def pre_cmd( cmd ):
418
    return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
419

420
def _pre_cmd( name, opts, cmd ):
421 422 423 424
    opts.pre_cmd = cmd

# ----

425
def clean_cmd( cmd ):
426 427
    # TODO. Remove all calls to clean_cmd.
    return lambda _name, _opts: None
428 429 430

# ----

431
def cmd_prefix( prefix ):
432
    return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
433

434
def _cmd_prefix( name, opts, prefix ):
435 436 437 438 439
    opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;

# ----

def cmd_wrapper( fun ):
440
    return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
441

442
def _cmd_wrapper( name, opts, fun ):
443
    opts.cmd_wrapper = fun
444

445 446
# ----

Ian Lynagh's avatar
Ian Lynagh committed
447
def compile_cmd_prefix( prefix ):
448
    return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
Ian Lynagh's avatar
Ian Lynagh committed
449

450
def _compile_cmd_prefix( name, opts, prefix ):
Ian Lynagh's avatar
Ian Lynagh committed
451 452 453 454
    opts.compile_cmd_prefix = prefix

# ----

455 456 457 458 459 460 461 462
def check_stdout( f ):
    return lambda name, opts, f=f: _check_stdout(name, opts, f)

def _check_stdout( name, opts, f ):
    opts.check_stdout = f

# ----

463
def normalise_slashes( name, opts ):
464
    _normalise_fun(name, opts, normalise_slashes_)
465

466
def normalise_exe( name, opts ):
467
    _normalise_fun(name, opts, normalise_exe_)
468

469 470
def normalise_fun( *fs ):
    return lambda name, opts: _normalise_fun(name, opts, fs)
471

472
def _normalise_fun( name, opts, *fs ):
473
    opts.extra_normaliser = join_normalisers(opts.extra_normaliser, fs)
474

475 476
def normalise_errmsg_fun( *fs ):
    return lambda name, opts: _normalise_errmsg_fun(name, opts, fs)
477

478
def _normalise_errmsg_fun( name, opts, *fs ):
479 480 481 482 483 484 485 486 487 488 489 490 491
    opts.extra_errmsg_normaliser =  join_normalisers(opts.extra_errmsg_normaliser, fs)

def normalise_version_( *pkgs ):
    def normalise_version__( str ):
        return re.sub('(' + '|'.join(map(re.escape,pkgs)) + ')-[0-9.]+',
                      '\\1-<VERSION>', str)
    return normalise_version__

def normalise_version( *pkgs ):
    def normalise_version__( name, opts ):
        _normalise_fun(name, opts, normalise_version_(*pkgs))
        _normalise_errmsg_fun(name, opts, normalise_version_(*pkgs))
    return normalise_version__
492

thomie's avatar
thomie committed
493 494 495 496
def normalise_drive_letter(name, opts):
    # Windows only. Change D:\\ to C:\\.
    _normalise_fun(name, opts, lambda str: re.sub(r'[A-Z]:\\', r'C:\\', str))

497 498 499 500 501 502 503
def keep_prof_callstacks(name, opts):
    """Keep profiling callstacks.

    Use together with `only_ways(prof_ways)`.
    """
    opts.keep_prof_callstacks = True

504 505
def join_normalisers(*a):
    """
506
    Compose functions, flattening sequences.
507

508
       join_normalisers(f1,[f2,f3],f4)
509 510 511

    is the same as

512
       lambda x: f1(f2(f3(f4(x))))
513 514
    """

515 516 517 518 519 520 521 522 523 524 525 526
    def flatten(l):
        """
        Taken from http://stackoverflow.com/a/2158532/946226
        """
        for el in l:
            if isinstance(el, collections.Iterable) and not isinstance(el, basestring):
                for sub in flatten(el):
                    yield sub
            else:
                yield el

    a = flatten(a)
527 528 529

    fn = lambda x:x # identity function
    for f in a:
530
        assert callable(f)
531 532 533
        fn = lambda x,f=f,fn=fn: fn(f(x))
    return fn

534 535 536
# ----
# Function for composing two opt-fns together

537
def executeSetups(fs, name, opts):
538
    if type(fs) is list:
539
        # If we have a list of setups, then execute each one
540 541
        for f in fs:
            executeSetups(f, name, opts)
542 543 544
    else:
        # fs is a single function, so just apply it
        fs(name, opts)
545

546 547 548
# -----------------------------------------------------------------------------
# The current directory of tests

549 550
def newTestDir(tempdir, dir):

551
    global thisdir_settings
552
    # reset the options for this test directory
553 554 555
    def settings(name, opts, tempdir=tempdir, dir=dir):
        return _newTestDir(name, opts, tempdir, dir)
    thisdir_settings = settings
556

557 558
# Should be equal to entry in toplevel .gitignore.
testdir_suffix = '.run'
559 560 561

def _newTestDir(name, opts, tempdir, dir):
    opts.srcdir = os.path.join(os.getcwd(), dir)
562
    opts.testdir = os.path.join(tempdir, dir, name + testdir_suffix)
563
    opts.compiler_always_flags = config.compiler_always_flags
564 565 566 567

# -----------------------------------------------------------------------------
# Actually doing tests

568 569
parallelTests = []
aloneTests = []
570
allTestNames = set([])
571

572
def runTest (opts, name, func, args):
ei@vuokko.info's avatar
ei@vuokko.info committed
573 574
    ok = 0

575
    if config.use_threads:
ei@vuokko.info's avatar
ei@vuokko.info committed
576
        t.thread_pool.acquire()
577
        try:
578
            while config.threads<(t.running_threads+1):
579
                t.thread_pool.wait()
580
            t.running_threads = t.running_threads+1
581 582
            ok=1
            t.thread_pool.release()
583
            thread.start_new_thread(test_common_thread, (name, opts, func, args))
584 585 586 587 588
        except:
            if not ok:
                t.thread_pool.release()
    else:
        test_common_work (name, opts, func, args)
589

590
# name  :: String
591
# setup :: TestOpts -> IO ()
592
def test (name, setup, func, args):
593 594 595 596 597 598 599 600 601 602
    if config.run_only_some_tests:
        if name not in config.only:
            return
        else:
            # Note [Mutating config.only]
            # config.only is initiallly the set of tests requested by
            # the user (via 'make TEST='). We then remove all tests that
            # we've already seen (in .T files), so that we can later
            # report on any tests we couldn't find and error out.
            config.only.remove(name)
603

604 605
    global aloneTests
    global parallelTests
606
    global allTestNames
607
    global thisdir_settings
608 609
    if name in allTestNames:
        framework_fail(name, 'duplicate', 'There are multiple tests with this name')
610
    if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
611
        framework_fail(name, 'bad_name', 'This test has an invalid name')
612 613 614 615 616

    # Make a deep copy of the default_testopts, as we need our own copy
    # of any dictionaries etc inside it. Otherwise, if one test modifies
    # them, all tests will see the modified version!
    myTestOpts = copy.deepcopy(default_testopts)
617

618
    executeSetups([thisdir_settings, setup], name, myTestOpts)
619 620 621 622 623 624

    thisTest = lambda : runTest(myTestOpts, name, func, args)
    if myTestOpts.alone:
        aloneTests.append(thisTest)
    else:
        parallelTests.append(thisTest)
625
    allTestNames.add(name)
626

627
if config.use_threads:
628
    def test_common_thread(name, opts, func, args):
629 630 631 632 633 634
        t.lock.acquire()
        try:
            test_common_work(name,opts,func,args)
        finally:
            t.lock.release()
            t.thread_pool.acquire()
635
            t.running_threads = t.running_threads - 1
636 637
            t.thread_pool.notify()
            t.thread_pool.release()
638

639 640 641 642 643 644 645 646 647 648
def get_package_cache_timestamp():
    if config.package_conf_cache_file == '':
        return 0.0
    else:
        try:
            return os.stat(config.package_conf_cache_file).st_mtime
        except:
            return 0.0


ei@vuokko.info's avatar
ei@vuokko.info committed
649
def test_common_work (name, opts, func, args):
650 651 652 653 654 655 656 657 658
    try:
        t.total_tests = t.total_tests+1
        setLocalTestOpts(opts)

        package_conf_cache_file_start_timestamp = get_package_cache_timestamp()

        # All the ways we might run this test
        if func == compile or func == multimod_compile:
            all_ways = config.compile_ways
659
        elif func == compile_and_run or func == multimod_compile_and_run:
660 661 662 663 664 665
            all_ways = config.run_ways
        elif func == ghci_script:
            if 'ghci' in config.run_ways:
                all_ways = ['ghci']
            else:
                all_ways = []
666
        else:
667 668 669
            all_ways = ['normal']

        # A test itself can request extra ways by setting opts.extra_ways
670
        all_ways = all_ways + [way for way in opts.extra_ways if way not in all_ways]
671 672 673 674 675

        t.total_test_cases = t.total_test_cases + len(all_ways)

        ok_way = lambda way: \
            not getTestOpts().skip \
676
            and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
677
            and (config.cmdline_ways == [] or way in config.cmdline_ways) \
678
            and (not (config.skip_perf_tests and isStatsTest())) \
679 680 681
            and way not in getTestOpts().omit_ways

        # Which ways we are asked to skip
682
        do_ways = list(filter (ok_way,all_ways))
683

684 685
        # Only run all ways in slow mode.
        # See Note [validate and testsuite speed] in toplevel Makefile.
686 687
        if config.accept:
            # Only ever run one way
688
            do_ways = do_ways[:1]
689 690 691
        elif config.speed > 0:
            # However, if we EXPLICITLY asked for a way (with extra_ways)
            # please test it!
692 693
            explicit_ways = list(filter(lambda way: way in opts.extra_ways, do_ways))
            other_ways = list(filter(lambda way: way not in opts.extra_ways, do_ways))
694
            do_ways = other_ways[:1] + explicit_ways
695

696 697 698 699 700 701 702 703 704
        # Find all files in the source directory that this test
        # depends on. Do this only once for all ways.
        # Generously add all filenames that start with the name of
        # the test to this set, as a convenience to test authors.
        # They will have to use the `extra_files` setup function to
        # specify all other files that their test depends on (but
        # this seems to be necessary for only about 10% of all
        # tests).
        files = set((f for f in os.listdir(opts.srcdir)
705 706
                        if f.startswith(name) and
                           not f.endswith(testdir_suffix)))
707
        for filename in (opts.extra_files + extra_src_files.get(name, [])):
708
            if filename.startswith('/'):
709 710 711 712 713 714 715 716 717 718 719 720
                framework_fail(name, 'whole-test',
                    'no absolute paths in extra_files please: ' + filename)

            elif '*' in filename:
                # Don't use wildcards in extra_files too much, as
                # globbing is slow.
                files.update((os.path.relpath(f, opts.srcdir)
                            for f in glob.iglob(in_srcdir(filename))))

            else:
                files.add(filename)

721 722 723 724 725
        # Run the required tests...
        for way in do_ways:
            if stopping():
                break
            do_test(name, way, func, args, files)
726

727 728 729
        for way in all_ways:
            if way not in do_ways:
                skiptest (name,way)
730

731
        if config.cleanup and do_ways:
732
            cleanup()
733

734
        package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
735

736 737
        if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
            framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
738

739 740 741 742 743 744 745 746 747 748
        try:
            for f in files_written[name]:
                if os.path.exists(f):
                    try:
                        if not f in files_written_not_removed[name]:
                            files_written_not_removed[name].append(f)
                    except:
                        files_written_not_removed[name] = [f]
        except:
            pass
749
    except Exception as e:
750
        framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
Ian Lynagh's avatar
Ian Lynagh committed
751

752 753 754
def do_test(name, way, func, args, files):
    opts = getTestOpts()

755 756 757
    full_name = name + '(' + way + ')'

    try:
758 759 760 761 762
        if_verbose(2, "=====> %s %d of %d %s " % \
                    (full_name, t.total_tests, len(allTestNames), \
                    [t.n_unexpected_passes, \
                     t.n_unexpected_failures, \
                     t.n_framework_failures]))
763

764 765 766 767 768 769 770 771 772 773
        # Clean up prior to the test, so that we can't spuriously conclude
        # that it passed on the basis of old run outputs.
        cleanup()

        # Link all source files for this test into a new directory in
        # /tmp, and run the test in that directory. This makes it
        # possible to run tests in parallel, without modification, that
        # would otherwise (accidentally) write to the same output file.
        # It also makes it easier to keep the testsuite clean.

774 775 776 777 778 779 780 781 782 783 784 785
        for extra_file in files:
            src = in_srcdir(extra_file)
            if extra_file.startswith('..'):
                # In case the extra_file is a file in an ancestor
                # directory (e.g. extra_files(['../shell.hs'])), make
                # sure it is copied to the test directory
                # (testdir/shell.hs), instead of ending up somewhere
                # else in the tree (testdir/../shell.hs)
                filename = os.path.basename(extra_file)
            else:
                filename = extra_file
            assert not '..' in filename # no funny stuff (foo/../../bar)
786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814
            dst = in_testdir(filename)

            if os.path.isfile(src):
                dirname = os.path.dirname(dst)
                if dirname:
                    mkdirp(dirname)
                try:
                    link_or_copy_file(src, dst)
                except OSError as e:
                    if e.errno == errno.EEXIST and os.path.isfile(dst):
                        # Some tests depend on files from ancestor
                        # directories (e.g. '../shell.hs'). It is
                        # possible such a file was already copied over
                        # for another test, since cleanup() doesn't
                        # delete them.
                        pass
                    else:
                        raise
            elif os.path.isdir(src):
                os.makedirs(dst)
                lndir(src, dst)
            else:
                if not config.haddock and os.path.splitext(filename)[1] == '.t':
                    # When using a ghc built without haddock support, .t
                    # files are rightfully missing. Don't
                    # framework_fail. Test will be skipped later.
                    pass
                else:
                    framework_fail(name, way,
815
                        'extra_file does not exist: ' + extra_file)
816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833

        if not files:
            # Always create the testdir, even when no files were copied
            # (because user forgot to specify extra_files setup function), to
            # prevent the confusing error: can't cd to <testdir>.
            os.makedirs(opts.testdir)

        if func.__name__ == 'run_command' or opts.pre_cmd:
            # When running 'MAKE' make sure 'TOP' still points to the
            # root of the testsuite.
            src_makefile = in_srcdir('Makefile')
            dst_makefile = in_testdir('Makefile')
            if os.path.exists(src_makefile):
                with open(src_makefile, 'r') as src:
                    makefile = re.sub('TOP=.*', 'TOP=' + config.top, src.read(), 1)
                    with open(dst_makefile, 'w') as dst:
                        dst.write(makefile)

834 835
        if config.use_threads:
            t.lock.release()
836 837 838 839

        try:
            preCmd = getTestOpts().pre_cmd
            if preCmd != None:
840
                result = runCmdFor(name, 'cd "{opts.testdir}" && {preCmd}'.format(**locals()))
841 842
                if result != 0:
                    framework_fail(name, way, 'pre-command failed: ' + str(result))
843
        except:
844 845
            framework_fail(name, way, 'pre-command exception')

ei@vuokko.info's avatar
ei@vuokko.info committed
846
        try:
847
            result = func(*[name,way] + args)
ei@vuokko.info's avatar
ei@vuokko.info committed
848
        finally:
849 850
            if config.use_threads:
                t.lock.acquire()
851

852 853 854
        if getTestOpts().expect != 'pass' and \
                getTestOpts().expect != 'fail' and \
                getTestOpts().expect != 'missing-lib':
855
            framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
856

857 858 859 860 861 862
        try:
            passFail = result['passFail']
        except:
            passFail = 'No passFail found'

        if passFail == 'pass':
863
            if _expect_pass(way):
864
                t.n_expected_passes = t.n_expected_passes + 1
865 866 867 868
                if name in t.expected_passes:
                    t.expected_passes[name].append(way)
                else:
                    t.expected_passes[name] = [way]
869
            else:
870
                if_verbose(1, '*** unexpected pass for %s' % full_name)
871
                t.n_unexpected_passes = t.n_unexpected_passes + 1
872
                addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
873
        elif passFail == 'fail':
874
            if _expect_pass(way):
875
                reason = result['reason']
876 877 878 879 880 881 882 883 884
                tag = result.get('tag')
                if tag == 'stat':
                    if_verbose(1, '*** unexpected stat test failure for %s' % full_name)
                    t.n_unexpected_stat_failures = t.n_unexpected_stat_failures + 1
                    addFailingTestInfo(t.unexpected_stat_failures, getTestOpts().testdir, name, reason, way)
                else:
                    if_verbose(1, '*** unexpected failure for %s' % full_name)
                    t.n_unexpected_failures = t.n_unexpected_failures