testlib.py 70.8 KB
Newer Older
1
# coding=utf8
2
#
3 4 5
# (c) Simon Marlow 2002
#

6
from __future__ import print_function
7

8
import io
9
import shutil
10
import os
11
import errno
12 13 14
import string
import re
import traceback
15 16
import time
import datetime
17
import copy
18
import glob
19
import sys
ian@well-typed.com's avatar
ian@well-typed.com committed
20
from math import ceil, trunc
21
import collections
22
import subprocess
23

24
from testglobals import *
25
from testutil import *
26
extra_src_files = {'T4198': ['exitminus1.c']} # TODO: See #12223
27

28 29
if config.use_threads:
    import threading
30 31 32 33
    try:
        import thread
    except ImportError: # Python 3
        import _thread as thread
34

35 36
global wantToStop
wantToStop = False
37 38 39 40 41

global pool_sema
if config.use_threads:
    pool_sema = threading.BoundedSemaphore(value=config.threads)

42 43 44 45 46 47
def stopNow():
    global wantToStop
    wantToStop = True
def stopping():
    return wantToStop

48 49
# Options valid for the current test only (these get reset to
# testdir_testopts after each test).
50

ei@vuokko.info's avatar
ei@vuokko.info committed
51
global testopts_local
52 53 54 55 56 57
if config.use_threads:
    testopts_local = threading.local()
else:
    class TestOpts_Local:
        pass
    testopts_local = TestOpts_Local()
58 59

def getTestOpts():
ei@vuokko.info's avatar
ei@vuokko.info committed
60
    return testopts_local.x
61

ei@vuokko.info's avatar
ei@vuokko.info committed
62 63 64
def setLocalTestOpts(opts):
    global testopts_local
    testopts_local.x=opts
65

66 67
def isStatsTest():
    opts = getTestOpts()
68
    return bool(opts.compiler_stats_range_fields or opts.stats_range_fields)
69 70


71 72 73
# This can be called at the top of a file of tests, to set default test options
# for the following tests.
def setTestOpts( f ):
74
    global thisdir_settings
75
    thisdir_settings = [thisdir_settings, f]
76 77 78 79 80 81 82 83 84 85 86

# -----------------------------------------------------------------------------
# Canned setup functions for common cases.  eg. for a test you might say
#
#      test('test001', normal, compile, [''])
#
# to run it without any options, but change it to
#
#      test('test001', expect_fail, compile, [''])
#
# to expect failure for this test.
87 88
#
# type TestOpt = (name :: String, opts :: Object) -> IO ()
89

90
def normal( name, opts ):
91 92
    return;

93
def skip( name, opts ):
94 95
    opts.skip = 1

96
def expect_fail( name, opts ):
97 98 99
    # The compiler, testdriver, OS or platform is missing a certain
    # feature, and we don't plan to or can't fix it now or in the
    # future.
100 101
    opts.expect = 'fail';

102
def reqlib( lib ):
103
    return lambda name, opts, l=lib: _reqlib (name, opts, l )
104

105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134
def stage1(name, opts):
    # See Note [Why is there no stage1 setup function?]
    framework_fail(name, 'stage1 setup function does not exist',
                   'add your test to testsuite/tests/stage1 instead')

# Note [Why is there no stage1 setup function?]
#
# Presumably a stage1 setup function would signal that the stage1
# compiler should be used to compile a test.
#
# Trouble is, the path to the compiler + the `ghc --info` settings for
# that compiler are currently passed in from the `make` part of the
# testsuite driver.
#
# Switching compilers in the Python part would be entirely too late, as
# all ghc_with_* settings would be wrong. See config/ghc for possible
# consequences (for example, config.run_ways would still be
# based on the default compiler, quite likely causing ./validate --slow
# to fail).
#
# It would be possible to let the Python part of the testsuite driver
# make the call to `ghc --info`, but doing so would require quite some
# work. Care has to be taken to not affect the run_command tests for
# example, as they also use the `ghc --info` settings:
#     quasiquotation/qq007/Makefile:ifeq "$(GhcDynamic)" "YES"
#
# If you want a test to run using the stage1 compiler, add it to the
# testsuite/tests/stage1 directory. Validate runs the tests in that
# directory with `make stage=1`.

135 136 137 138
# Cache the results of looking to see if we have a library or not.
# This makes quite a difference, especially on Windows.
have_lib = {}

139
def _reqlib( name, opts, lib ):
140
    if lib in have_lib:
141
        got_it = have_lib[lib]
142
    else:
143 144 145 146 147 148 149 150
        cmd = strip_quotes(config.ghc_pkg)
        p = subprocess.Popen([cmd, '--no-user-package-db', 'describe', lib],
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
        # read from stdout and stderr to avoid blocking due to
        # buffers filling
        p.communicate()
        r = p.wait()
151 152 153 154
        got_it = r == 0
        have_lib[lib] = got_it

    if not got_it:
155
        opts.expect = 'missing-lib'
156

157 158 159 160
def req_haddock( name, opts ):
    if not config.haddock:
        opts.expect = 'missing-lib'

161
def req_profiling( name, opts ):
162
    '''Require the profiling libraries (add 'GhcLibWays += p' to mk/build.mk)'''
163 164 165
    if not config.have_profiling:
        opts.expect = 'fail'

166
def req_shared_libs( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
167 168 169
    if not config.have_shared_libs:
        opts.expect = 'fail'

170
def req_interp( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
171 172 173
    if not config.have_interp:
        opts.expect = 'fail'

174
def req_smp( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
175 176 177
    if not config.have_smp:
        opts.expect = 'fail'

178 179 180 181 182
def ignore_stdout(name, opts):
    opts.ignore_stdout = True

def ignore_stderr(name, opts):
    opts.ignore_stderr = True
183

184
def combined_output( name, opts ):
pcapriotti's avatar
pcapriotti committed
185 186
    opts.combined_output = True

187 188 189
# -----

def expect_fail_for( ways ):
190
    return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
191

192
def _expect_fail_for( name, opts, ways ):
193 194
    opts.expect_fail_for = ways

195
def expect_broken( bug ):
196 197
    # This test is a expected not to work due to the indicated trac bug
    # number.
198 199 200
    return lambda name, opts, b=bug: _expect_broken (name, opts, b )

def _expect_broken( name, opts, bug ):
201
    record_broken(name, opts, bug)
202 203
    opts.expect = 'fail';

Ian Lynagh's avatar
Ian Lynagh committed
204
def expect_broken_for( bug, ways ):
205
    return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
Ian Lynagh's avatar
Ian Lynagh committed
206

207
def _expect_broken_for( name, opts, bug, ways ):
208
    record_broken(name, opts, bug)
Ian Lynagh's avatar
Ian Lynagh committed
209
    opts.expect_fail_for = ways
Ian Lynagh's avatar
Ian Lynagh committed
210

211
def record_broken(name, opts, bug):
212
    global brokens
213
    me = (bug, opts.testdir, name)
214 215 216
    if not me in brokens:
        brokens.append(me)

217 218 219 220 221
def _expect_pass(way):
    # Helper function. Not intended for use in .T files.
    opts = getTestOpts()
    return opts.expect == 'pass' and way not in opts.expect_fail_for

222 223 224
# -----

def omit_ways( ways ):
225
    return lambda name, opts, w=ways: _omit_ways( name, opts, w )
226

227
def _omit_ways( name, opts, ways ):
228 229 230 231
    opts.omit_ways = ways

# -----

232
def only_ways( ways ):
233
    return lambda name, opts, w=ways: _only_ways( name, opts, w )
234

235
def _only_ways( name, opts, ways ):
236 237 238 239
    opts.only_ways = ways

# -----

240
def extra_ways( ways ):
241
    return lambda name, opts, w=ways: _extra_ways( name, opts, w )
242

243
def _extra_ways( name, opts, ways ):
244 245 246 247
    opts.extra_ways = ways

# -----

248
def set_stdin( file ):
249
   return lambda name, opts, f=file: _set_stdin(name, opts, f);
250

251
def _set_stdin( name, opts, f ):
252 253 254 255 256
   opts.stdin = f

# -----

def exit_code( val ):
257
    return lambda name, opts, v=val: _exit_code(name, opts, v);
258

259
def _exit_code( name, opts, v ):
260 261
    opts.exit_code = v

262 263 264 265 266 267 268 269 270 271 272
def signal_exit_code( val ):
    if opsys('solaris2'):
        return exit_code( val );
    else:
        # When application running on Linux receives fatal error
        # signal, then its exit code is encoded as 128 + signal
        # value. See http://www.tldp.org/LDP/abs/html/exitcodes.html
        # I assume that Mac OS X behaves in the same way at least Mac
        # OS X builder behavior suggests this.
        return exit_code( val+128 );

273 274
# -----

275 276
def compile_timeout_multiplier( val ):
    return lambda name, opts, v=val: _compile_timeout_multiplier(name, opts, v)
277

278 279 280 281 282 283 284 285
def _compile_timeout_multiplier( name, opts, v ):
    opts.compile_timeout_multiplier = v

def run_timeout_multiplier( val ):
    return lambda name, opts, v=val: _run_timeout_multiplier(name, opts, v)

def _run_timeout_multiplier( name, opts, v ):
    opts.run_timeout_multiplier = v
286 287 288

# -----

289
def extra_run_opts( val ):
290
    return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
291

292
def _extra_run_opts( name, opts, v ):
293 294
    opts.extra_run_opts = v

295 296
# -----

Simon Marlow's avatar
Simon Marlow committed
297
def extra_hc_opts( val ):
298
    return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
Simon Marlow's avatar
Simon Marlow committed
299

300
def _extra_hc_opts( name, opts, v ):
Simon Marlow's avatar
Simon Marlow committed
301 302 303 304
    opts.extra_hc_opts = v

# -----

305
def extra_clean( files ):
306 307
    # TODO. Remove all calls to extra_clean.
    return lambda _name, _opts: None
308

309 310 311 312 313 314
def extra_files(files):
    return lambda name, opts: _extra_files(name, opts, files)

def _extra_files(name, opts, files):
    opts.extra_files.extend(files)

315 316
# -----

317 318 319 320 321 322 323
def stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);

def _stats_num_field( name, opts, field, expecteds ):
    if field in opts.stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

324
    if type(expecteds) is list:
325 326 327 328
        for (b, expected, dev) in expecteds:
            if b:
                opts.stats_range_fields[field] = (expected, dev)
                return
329
        framework_warn(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
330

331 332 333
    else:
        (expected, dev) = expecteds
        opts.stats_range_fields[field] = (expected, dev)
334 335 336

def compiler_stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
337

338 339 340 341
def _compiler_stats_num_field( name, opts, field, expecteds ):
    if field in opts.compiler_stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

342 343
    # Compiler performance numbers change when debugging is on, making the results
    # useless and confusing. Therefore, skip if debugging is on.
344 345 346
    if compiler_debugged():
        skip(name, opts)

347 348 349 350
    for (b, expected, dev) in expecteds:
        if b:
            opts.compiler_stats_range_fields[field] = (expected, dev)
            return
351

352
    framework_warn(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
353

354 355
# -----

356
def when(b, f):
ian@well-typed.com's avatar
ian@well-typed.com committed
357 358 359
    # When list_brokens is on, we want to see all expect_broken calls,
    # so we always do f
    if b or config.list_broken:
360 361 362 363 364 365 366
        return f
    else:
        return normal

def unless(b, f):
    return when(not b, f)

ian@well-typed.com's avatar
ian@well-typed.com committed
367 368 369
def doing_ghci():
    return 'ghci' in config.run_ways

370
def ghc_dynamic():
ian@well-typed.com's avatar
ian@well-typed.com committed
371
    return config.ghc_dynamic
372 373

def fast():
374
    return config.speed == 2
375

376 377
def platform( plat ):
    return config.platform == plat
Ian Lynagh's avatar
Ian Lynagh committed
378

379 380
def opsys( os ):
    return config.os == os
Ian Lynagh's avatar
Ian Lynagh committed
381

382 383
def arch( arch ):
    return config.arch == arch
384

385 386
def wordsize( ws ):
    return config.wordsize == str(ws)
tibbe's avatar
tibbe committed
387

388 389
def msys( ):
    return config.msys
ian@well-typed.com's avatar
ian@well-typed.com committed
390

391 392
def cygwin( ):
    return config.cygwin
Ian Lynagh's avatar
Ian Lynagh committed
393

394 395
def have_vanilla( ):
    return config.have_vanilla
396

397 398
def have_dynamic( ):
    return config.have_dynamic
399

400 401
def have_profiling( ):
    return config.have_profiling
402

403 404
def in_tree_compiler( ):
    return config.in_tree_compiler
405

406 407 408 409 410 411
def unregisterised( ):
    return config.unregisterised

def compiler_profiled( ):
    return config.compiler_profiled

412 413
def compiler_debugged( ):
    return config.compiler_debugged
414

415 416
# ---

417
def high_memory_usage(name, opts):
418 419
    opts.alone = True

420 421 422 423 424
# If a test is for a multi-CPU race, then running the test alone
# increases the chance that we'll actually see it.
def multi_cpu_race(name, opts):
    opts.alone = True

Ian Lynagh's avatar
Ian Lynagh committed
425
# ---
426
def literate( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
427 428
    opts.literate = 1;

429
def c_src( name, opts ):
430 431
    opts.c_src = 1;

432
def objc_src( name, opts ):
Austin Seipp's avatar
Austin Seipp committed
433 434
    opts.objc_src = 1;

435
def objcpp_src( name, opts ):
436 437
    opts.objcpp_src = 1;

438
def cmm_src( name, opts ):
439 440
    opts.cmm_src = 1;

441
def outputdir( odir ):
442
    return lambda name, opts, d=odir: _outputdir(name, opts, d)
443

444
def _outputdir( name, opts, odir ):
445 446
    opts.outputdir = odir;

447 448
# ----

449
def pre_cmd( cmd ):
450
    return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
451

452
def _pre_cmd( name, opts, cmd ):
453 454 455 456
    opts.pre_cmd = cmd

# ----

457
def clean_cmd( cmd ):
458 459
    # TODO. Remove all calls to clean_cmd.
    return lambda _name, _opts: None
460 461 462

# ----

463
def cmd_prefix( prefix ):
464
    return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
465

466
def _cmd_prefix( name, opts, prefix ):
467 468 469 470 471
    opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;

# ----

def cmd_wrapper( fun ):
472
    return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
473

474
def _cmd_wrapper( name, opts, fun ):
475
    opts.cmd_wrapper = fun
476

477 478
# ----

Ian Lynagh's avatar
Ian Lynagh committed
479
def compile_cmd_prefix( prefix ):
480
    return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
Ian Lynagh's avatar
Ian Lynagh committed
481

482
def _compile_cmd_prefix( name, opts, prefix ):
Ian Lynagh's avatar
Ian Lynagh committed
483 484 485 486
    opts.compile_cmd_prefix = prefix

# ----

487 488 489 490 491 492
def check_stdout( f ):
    return lambda name, opts, f=f: _check_stdout(name, opts, f)

def _check_stdout( name, opts, f ):
    opts.check_stdout = f

493 494 495
def no_check_hp(name, opts):
    opts.check_hp = False

496 497
# ----

498 499 500 501 502 503 504
def filter_stdout_lines( regex ):
    """ Filter lines of stdout with the given regular expression """
    import re
    def f( name, opts ):
        _normalise_fun(name, opts, lambda s: '\n'.join(re.findall(regex, s)))
    return f

505
def normalise_slashes( name, opts ):
506
    _normalise_fun(name, opts, normalise_slashes_)
507

508
def normalise_exe( name, opts ):
509
    _normalise_fun(name, opts, normalise_exe_)
510

511 512
def normalise_fun( *fs ):
    return lambda name, opts: _normalise_fun(name, opts, fs)
513

514
def _normalise_fun( name, opts, *fs ):
515
    opts.extra_normaliser = join_normalisers(opts.extra_normaliser, fs)
516

517 518
def normalise_errmsg_fun( *fs ):
    return lambda name, opts: _normalise_errmsg_fun(name, opts, fs)
519

520
def _normalise_errmsg_fun( name, opts, *fs ):
521 522
    opts.extra_errmsg_normaliser =  join_normalisers(opts.extra_errmsg_normaliser, fs)

Joachim Breitner's avatar
Joachim Breitner committed
523 524 525 526 527 528 529 530 531 532 533 534 535
def check_errmsg(needle):
    def norm(str):
        if needle in str:
            return "%s contained in -ddump-simpl\n" % needle
        else:
            return "%s not contained in -ddump-simpl\n" % needle
    return normalise_errmsg_fun(norm)

def grep_errmsg(needle):
    def norm(str):
        return "".join(filter(lambda l: re.search(needle, l), str.splitlines(True)))
    return normalise_errmsg_fun(norm)

536 537 538 539 540 541
def normalise_whitespace_fun(f):
    return lambda name, opts: _normalise_whitespace_fun(name, opts, f)

def _normalise_whitespace_fun(name, opts, f):
    opts.whitespace_normaliser = f

542 543 544 545 546 547 548 549 550 551 552
def normalise_version_( *pkgs ):
    def normalise_version__( str ):
        return re.sub('(' + '|'.join(map(re.escape,pkgs)) + ')-[0-9.]+',
                      '\\1-<VERSION>', str)
    return normalise_version__

def normalise_version( *pkgs ):
    def normalise_version__( name, opts ):
        _normalise_fun(name, opts, normalise_version_(*pkgs))
        _normalise_errmsg_fun(name, opts, normalise_version_(*pkgs))
    return normalise_version__
553

thomie's avatar
thomie committed
554 555 556 557
def normalise_drive_letter(name, opts):
    # Windows only. Change D:\\ to C:\\.
    _normalise_fun(name, opts, lambda str: re.sub(r'[A-Z]:\\', r'C:\\', str))

558 559 560 561 562 563 564
def keep_prof_callstacks(name, opts):
    """Keep profiling callstacks.

    Use together with `only_ways(prof_ways)`.
    """
    opts.keep_prof_callstacks = True

565 566
def join_normalisers(*a):
    """
567
    Compose functions, flattening sequences.
568

569
       join_normalisers(f1,[f2,f3],f4)
570 571 572

    is the same as

573
       lambda x: f1(f2(f3(f4(x))))
574 575
    """

576 577 578 579 580
    def flatten(l):
        """
        Taken from http://stackoverflow.com/a/2158532/946226
        """
        for el in l:
581 582
            if (isinstance(el, collections.Iterable)
                and not isinstance(el, (bytes, str))):
583 584 585 586 587 588
                for sub in flatten(el):
                    yield sub
            else:
                yield el

    a = flatten(a)
589 590 591

    fn = lambda x:x # identity function
    for f in a:
592
        assert callable(f)
593 594 595
        fn = lambda x,f=f,fn=fn: fn(f(x))
    return fn

596 597 598
# ----
# Function for composing two opt-fns together

599
def executeSetups(fs, name, opts):
600
    if type(fs) is list:
601
        # If we have a list of setups, then execute each one
602 603
        for f in fs:
            executeSetups(f, name, opts)
604 605 606
    else:
        # fs is a single function, so just apply it
        fs(name, opts)
607

608 609 610
# -----------------------------------------------------------------------------
# The current directory of tests

611 612
def newTestDir(tempdir, dir):

613
    global thisdir_settings
614
    # reset the options for this test directory
615 616 617
    def settings(name, opts, tempdir=tempdir, dir=dir):
        return _newTestDir(name, opts, tempdir, dir)
    thisdir_settings = settings
618

619 620
# Should be equal to entry in toplevel .gitignore.
testdir_suffix = '.run'
621 622 623

def _newTestDir(name, opts, tempdir, dir):
    opts.srcdir = os.path.join(os.getcwd(), dir)
624
    opts.testdir = os.path.join(tempdir, dir, name + testdir_suffix)
625
    opts.compiler_always_flags = config.compiler_always_flags
626 627 628 629

# -----------------------------------------------------------------------------
# Actually doing tests

630 631
parallelTests = []
aloneTests = []
632
allTestNames = set([])
633

634
def runTest(watcher, opts, name, func, args):
635
    if config.use_threads:
636 637 638 639 640 641
        pool_sema.acquire()
        t = threading.Thread(target=test_common_thread,
                             name=name,
                             args=(watcher, name, opts, func, args))
        t.daemon = False
        t.start()
642
    else:
643
        test_common_work(watcher, name, opts, func, args)
644

645
# name  :: String
646
# setup :: [TestOpt] -> IO ()
647
def test(name, setup, func, args):
648 649 650 651 652 653 654 655 656
    global aloneTests
    global parallelTests
    global allTestNames
    global thisdir_settings
    if name in allTestNames:
        framework_fail(name, 'duplicate', 'There are multiple tests with this name')
    if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
        framework_fail(name, 'bad_name', 'This test has an invalid name')

657 658 659 660 661
    if config.run_only_some_tests:
        if name not in config.only:
            return
        else:
            # Note [Mutating config.only]
Gabor Greif's avatar
Gabor Greif committed
662
            # config.only is initially the set of tests requested by
663 664 665 666
            # the user (via 'make TEST='). We then remove all tests that
            # we've already seen (in .T files), so that we can later
            # report on any tests we couldn't find and error out.
            config.only.remove(name)
667

668 669 670 671
    # Make a deep copy of the default_testopts, as we need our own copy
    # of any dictionaries etc inside it. Otherwise, if one test modifies
    # them, all tests will see the modified version!
    myTestOpts = copy.deepcopy(default_testopts)
672

673
    executeSetups([thisdir_settings, setup], name, myTestOpts)
674

675
    thisTest = lambda watcher: runTest(watcher, myTestOpts, name, func, args)
676 677 678 679
    if myTestOpts.alone:
        aloneTests.append(thisTest)
    else:
        parallelTests.append(thisTest)
680
    allTestNames.add(name)
681

682
if config.use_threads:
683 684 685 686 687
    def test_common_thread(watcher, name, opts, func, args):
            try:
                test_common_work(watcher, name, opts, func, args)
            finally:
                pool_sema.release()
688

689 690 691 692 693 694 695 696 697
def get_package_cache_timestamp():
    if config.package_conf_cache_file == '':
        return 0.0
    else:
        try:
            return os.stat(config.package_conf_cache_file).st_mtime
        except:
            return 0.0

Edward Z. Yang's avatar
Edward Z. Yang committed
698
do_not_copy = ('.hi', '.o', '.dyn_hi', '.dyn_o', '.out') # 12112
699

700
def test_common_work(watcher, name, opts, func, args):
701
    try:
702
        t.total_tests += 1
703 704 705 706 707 708 709
        setLocalTestOpts(opts)

        package_conf_cache_file_start_timestamp = get_package_cache_timestamp()

        # All the ways we might run this test
        if func == compile or func == multimod_compile:
            all_ways = config.compile_ways
710
        elif func == compile_and_run or func == multimod_compile_and_run:
711 712 713 714 715 716
            all_ways = config.run_ways
        elif func == ghci_script:
            if 'ghci' in config.run_ways:
                all_ways = ['ghci']
            else:
                all_ways = []
717
        else:
718 719 720
            all_ways = ['normal']

        # A test itself can request extra ways by setting opts.extra_ways
721
        all_ways = all_ways + [way for way in opts.extra_ways if way not in all_ways]
722

723
        t.total_test_cases += len(all_ways)
724 725 726

        ok_way = lambda way: \
            not getTestOpts().skip \
727
            and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
728
            and (config.cmdline_ways == [] or way in config.cmdline_ways) \
729
            and (not (config.skip_perf_tests and isStatsTest())) \
730 731 732
            and way not in getTestOpts().omit_ways

        # Which ways we are asked to skip
733
        do_ways = list(filter (ok_way,all_ways))
734

735 736
        # Only run all ways in slow mode.
        # See Note [validate and testsuite speed] in toplevel Makefile.
737 738
        if config.accept:
            # Only ever run one way
739
            do_ways = do_ways[:1]
740 741 742
        elif config.speed > 0:
            # However, if we EXPLICITLY asked for a way (with extra_ways)
            # please test it!
743 744
            explicit_ways = list(filter(lambda way: way in opts.extra_ways, do_ways))
            other_ways = list(filter(lambda way: way not in opts.extra_ways, do_ways))
745
            do_ways = other_ways[:1] + explicit_ways
746

747 748 749 750 751 752 753 754
        # Find all files in the source directory that this test
        # depends on. Do this only once for all ways.
        # Generously add all filenames that start with the name of
        # the test to this set, as a convenience to test authors.
        # They will have to use the `extra_files` setup function to
        # specify all other files that their test depends on (but
        # this seems to be necessary for only about 10% of all
        # tests).
755 756 757 758
        files = set(f for f in os.listdir(opts.srcdir)
                       if f.startswith(name) and not f == name and
                          not f.endswith(testdir_suffix) and
                          not os.path.splitext(f)[1] in do_not_copy)
759
        for filename in (opts.extra_files + extra_src_files.get(name, [])):
760
            if filename.startswith('/'):
761 762 763 764 765 766 767 768 769
                framework_fail(name, 'whole-test',
                    'no absolute paths in extra_files please: ' + filename)

            elif '*' in filename:
                # Don't use wildcards in extra_files too much, as
                # globbing is slow.
                files.update((os.path.relpath(f, opts.srcdir)
                            for f in glob.iglob(in_srcdir(filename))))

thomie's avatar
thomie committed
770
            elif filename:
771 772
                files.add(filename)

thomie's avatar
thomie committed
773 774 775
            else:
                framework_fail(name, 'whole-test', 'extra_file is empty string')

776 777 778 779
        # Run the required tests...
        for way in do_ways:
            if stopping():
                break
780 781 782 783 784 785 786
            try:
                do_test(name, way, func, args, files)
            except KeyboardInterrupt:
                stopNow()
            except Exception as e:
                framework_fail(name, way, str(e))
                traceback.print_exc()
787

788
        t.n_tests_skipped += len(set(all_ways) - set(do_ways))
789

790
        if config.cleanup and do_ways:
791 792 793 794
            try:
                cleanup()
            except Exception as e:
                framework_fail(name, 'runTest', 'Unhandled exception during cleanup: ' + str(e))
795

796
        package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
797

798 799
        if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
            framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
800

801
    except Exception as e:
802
        framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
803 804
    finally:
        watcher.notify()
Ian Lynagh's avatar
Ian Lynagh committed
805

806 807 808
def do_test(name, way, func, args, files):
    opts = getTestOpts()

809 810
    full_name = name + '(' + way + ')'

811
    if_verbose(2, "=====> {0} {1} of {2} {3}".format(
812 813 814 815
        full_name, t.total_tests, len(allTestNames), 
        [len(t.unexpected_passes),
         len(t.unexpected_failures),
         len(t.framework_failures)]))
816 817 818 819

    # Clean up prior to the test, so that we can't spuriously conclude
    # that it passed on the basis of old run outputs.
    cleanup()
820
    os.makedirs(opts.testdir)
821 822 823 824 825 826 827 828 829

    # Link all source files for this test into a new directory in
    # /tmp, and run the test in that directory. This makes it
    # possible to run tests in parallel, without modification, that
    # would otherwise (accidentally) write to the same output file.
    # It also makes it easier to keep the testsuite clean.

    for extra_file in files:
        src = in_srcdir(extra_file)
830
        dst = in_testdir(os.path.basename(extra_file.rstrip('/\\')))
831
        if os.path.isfile(src):
832
            link_or_copy_file(src, dst)
833
        elif os.path.isdir(src):
834
            os.mkdir(dst)