testlib.py 83 KB
Newer Older
1
# coding=utf8
2
#
3 4 5
# (c) Simon Marlow 2002
#

6
from __future__ import print_function
7

8
import shutil
9 10
import sys
import os
11
import errno
12 13 14
import string
import re
import traceback
15 16
import time
import datetime
17
import copy
18
import glob
ian@well-typed.com's avatar
ian@well-typed.com committed
19
from math import ceil, trunc
20
import collections
21
import subprocess
22

23
from testglobals import *
24
from testutil import *
25
from extra_files import extra_src_files
26

27 28 29 30 31
try:
    basestring
except: # Python 3
    basestring = (str,bytes)

32 33
if config.use_threads:
    import threading
34 35 36 37
    try:
        import thread
    except ImportError: # Python 3
        import _thread as thread
38

39 40 41 42 43 44 45 46
global wantToStop
wantToStop = False
def stopNow():
    global wantToStop
    wantToStop = True
def stopping():
    return wantToStop

47 48
# Options valid for the current test only (these get reset to
# testdir_testopts after each test).
49

ei@vuokko.info's avatar
ei@vuokko.info committed
50
global testopts_local
51 52 53 54 55 56
if config.use_threads:
    testopts_local = threading.local()
else:
    class TestOpts_Local:
        pass
    testopts_local = TestOpts_Local()
57 58

def getTestOpts():
ei@vuokko.info's avatar
ei@vuokko.info committed
59
    return testopts_local.x
60

ei@vuokko.info's avatar
ei@vuokko.info committed
61 62 63
def setLocalTestOpts(opts):
    global testopts_local
    testopts_local.x=opts
64

65 66 67 68 69
def isStatsTest():
    opts = getTestOpts()
    return len(opts.compiler_stats_range_fields) > 0 or len(opts.stats_range_fields) > 0


70 71 72
# This can be called at the top of a file of tests, to set default test options
# for the following tests.
def setTestOpts( f ):
73
    global thisdir_settings
74
    thisdir_settings = [thisdir_settings, f]
75 76 77 78 79 80 81 82 83 84 85 86

# -----------------------------------------------------------------------------
# Canned setup functions for common cases.  eg. for a test you might say
#
#      test('test001', normal, compile, [''])
#
# to run it without any options, but change it to
#
#      test('test001', expect_fail, compile, [''])
#
# to expect failure for this test.

87
def normal( name, opts ):
88 89
    return;

90
def skip( name, opts ):
91 92
    opts.skip = 1

93
def expect_fail( name, opts ):
94 95 96
    # The compiler, testdriver, OS or platform is missing a certain
    # feature, and we don't plan to or can't fix it now or in the
    # future.
97 98
    opts.expect = 'fail';

99
def reqlib( lib ):
100
    return lambda name, opts, l=lib: _reqlib (name, opts, l )
101

102 103 104 105
# Cache the results of looking to see if we have a library or not.
# This makes quite a difference, especially on Windows.
have_lib = {}

106
def _reqlib( name, opts, lib ):
107
    if lib in have_lib:
108
        got_it = have_lib[lib]
109
    else:
110 111 112 113 114 115 116 117
        cmd = strip_quotes(config.ghc_pkg)
        p = subprocess.Popen([cmd, '--no-user-package-db', 'describe', lib],
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
        # read from stdout and stderr to avoid blocking due to
        # buffers filling
        p.communicate()
        r = p.wait()
118 119 120 121
        got_it = r == 0
        have_lib[lib] = got_it

    if not got_it:
122
        opts.expect = 'missing-lib'
123

124 125 126 127
def req_haddock( name, opts ):
    if not config.haddock:
        opts.expect = 'missing-lib'

128
def req_profiling( name, opts ):
129
    '''Require the profiling libraries (add 'GhcLibWays += p' to mk/build.mk)'''
130 131 132
    if not config.have_profiling:
        opts.expect = 'fail'

133
def req_shared_libs( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
134 135 136
    if not config.have_shared_libs:
        opts.expect = 'fail'

137
def req_interp( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
138 139 140
    if not config.have_interp:
        opts.expect = 'fail'

141
def req_smp( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
142 143 144
    if not config.have_smp:
        opts.expect = 'fail'

145
def ignore_output( name, opts ):
146 147
    opts.ignore_output = 1

148
def no_stdin( name, opts ):
149 150
    opts.no_stdin = 1

151
def combined_output( name, opts ):
pcapriotti's avatar
pcapriotti committed
152 153
    opts.combined_output = True

154 155 156
# -----

def expect_fail_for( ways ):
157
    return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
158

159
def _expect_fail_for( name, opts, ways ):
160 161
    opts.expect_fail_for = ways

162
def expect_broken( bug ):
163 164
    # This test is a expected not to work due to the indicated trac bug
    # number.
165 166 167
    return lambda name, opts, b=bug: _expect_broken (name, opts, b )

def _expect_broken( name, opts, bug ):
168
    record_broken(name, opts, bug)
169 170
    opts.expect = 'fail';

Ian Lynagh's avatar
Ian Lynagh committed
171
def expect_broken_for( bug, ways ):
172
    return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
Ian Lynagh's avatar
Ian Lynagh committed
173

174
def _expect_broken_for( name, opts, bug, ways ):
175
    record_broken(name, opts, bug)
Ian Lynagh's avatar
Ian Lynagh committed
176
    opts.expect_fail_for = ways
Ian Lynagh's avatar
Ian Lynagh committed
177

178
def record_broken(name, opts, bug):
179
    global brokens
180
    me = (bug, opts.testdir, name)
181 182 183
    if not me in brokens:
        brokens.append(me)

184 185 186 187 188
def _expect_pass(way):
    # Helper function. Not intended for use in .T files.
    opts = getTestOpts()
    return opts.expect == 'pass' and way not in opts.expect_fail_for

189 190 191
# -----

def omit_ways( ways ):
192
    return lambda name, opts, w=ways: _omit_ways( name, opts, w )
193

194
def _omit_ways( name, opts, ways ):
195 196 197 198
    opts.omit_ways = ways

# -----

199
def only_ways( ways ):
200
    return lambda name, opts, w=ways: _only_ways( name, opts, w )
201

202
def _only_ways( name, opts, ways ):
203 204 205 206
    opts.only_ways = ways

# -----

207
def extra_ways( ways ):
208
    return lambda name, opts, w=ways: _extra_ways( name, opts, w )
209

210
def _extra_ways( name, opts, ways ):
211 212 213 214
    opts.extra_ways = ways

# -----

215
def set_stdin( file ):
216
   return lambda name, opts, f=file: _set_stdin(name, opts, f);
217

218
def _set_stdin( name, opts, f ):
219 220 221 222 223
   opts.stdin = f

# -----

def exit_code( val ):
224
    return lambda name, opts, v=val: _exit_code(name, opts, v);
225

226
def _exit_code( name, opts, v ):
227 228
    opts.exit_code = v

229 230 231 232 233 234 235 236 237 238 239
def signal_exit_code( val ):
    if opsys('solaris2'):
        return exit_code( val );
    else:
        # When application running on Linux receives fatal error
        # signal, then its exit code is encoded as 128 + signal
        # value. See http://www.tldp.org/LDP/abs/html/exitcodes.html
        # I assume that Mac OS X behaves in the same way at least Mac
        # OS X builder behavior suggests this.
        return exit_code( val+128 );

240 241
# -----

242 243
def compile_timeout_multiplier( val ):
    return lambda name, opts, v=val: _compile_timeout_multiplier(name, opts, v)
244

245 246 247 248 249 250 251 252
def _compile_timeout_multiplier( name, opts, v ):
    opts.compile_timeout_multiplier = v

def run_timeout_multiplier( val ):
    return lambda name, opts, v=val: _run_timeout_multiplier(name, opts, v)

def _run_timeout_multiplier( name, opts, v ):
    opts.run_timeout_multiplier = v
253 254 255

# -----

256
def extra_run_opts( val ):
257
    return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
258

259
def _extra_run_opts( name, opts, v ):
260 261
    opts.extra_run_opts = v

262 263
# -----

Simon Marlow's avatar
Simon Marlow committed
264
def extra_hc_opts( val ):
265
    return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
Simon Marlow's avatar
Simon Marlow committed
266

267
def _extra_hc_opts( name, opts, v ):
Simon Marlow's avatar
Simon Marlow committed
268 269 270 271
    opts.extra_hc_opts = v

# -----

272
def extra_clean( files ):
273
    assert not isinstance(files, str), files
274
    return lambda name, opts, v=files: _extra_clean(name, opts, v);
275

276
def _extra_clean( name, opts, v ):
277 278
    opts.clean_files = v

279 280 281 282 283 284
def extra_files(files):
    return lambda name, opts: _extra_files(name, opts, files)

def _extra_files(name, opts, files):
    opts.extra_files.extend(files)

285 286
# -----

287 288 289 290 291 292 293
def stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);

def _stats_num_field( name, opts, field, expecteds ):
    if field in opts.stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

294
    if type(expecteds) is list:
295 296 297 298 299
        for (b, expected, dev) in expecteds:
            if b:
                opts.stats_range_fields[field] = (expected, dev)
                return
        framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
300

301 302 303
    else:
        (expected, dev) = expecteds
        opts.stats_range_fields[field] = (expected, dev)
304 305 306

def compiler_stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
307

308 309 310 311
def _compiler_stats_num_field( name, opts, field, expecteds ):
    if field in opts.compiler_stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

312 313
    # Compiler performance numbers change when debugging is on, making the results
    # useless and confusing. Therefore, skip if debugging is on.
314 315 316
    if compiler_debugged():
        skip(name, opts)

317 318 319 320
    for (b, expected, dev) in expecteds:
        if b:
            opts.compiler_stats_range_fields[field] = (expected, dev)
            return
321

322 323
    framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')

324 325
# -----

326
def when(b, f):
ian@well-typed.com's avatar
ian@well-typed.com committed
327 328 329
    # When list_brokens is on, we want to see all expect_broken calls,
    # so we always do f
    if b or config.list_broken:
330 331 332 333 334 335 336
        return f
    else:
        return normal

def unless(b, f):
    return when(not b, f)

ian@well-typed.com's avatar
ian@well-typed.com committed
337 338 339
def doing_ghci():
    return 'ghci' in config.run_ways

340
def ghc_dynamic():
ian@well-typed.com's avatar
ian@well-typed.com committed
341
    return config.ghc_dynamic
342 343

def fast():
344
    return config.speed == 2
345

346 347
def platform( plat ):
    return config.platform == plat
Ian Lynagh's avatar
Ian Lynagh committed
348

349 350
def opsys( os ):
    return config.os == os
Ian Lynagh's avatar
Ian Lynagh committed
351

352 353
def arch( arch ):
    return config.arch == arch
354

355 356
def wordsize( ws ):
    return config.wordsize == str(ws)
tibbe's avatar
tibbe committed
357

358 359
def msys( ):
    return config.msys
ian@well-typed.com's avatar
ian@well-typed.com committed
360

361 362
def cygwin( ):
    return config.cygwin
Ian Lynagh's avatar
Ian Lynagh committed
363

364 365
def have_vanilla( ):
    return config.have_vanilla
366

367 368
def have_dynamic( ):
    return config.have_dynamic
369

370 371
def have_profiling( ):
    return config.have_profiling
372

373 374
def in_tree_compiler( ):
    return config.in_tree_compiler
375

376 377 378 379 380 381
def unregisterised( ):
    return config.unregisterised

def compiler_profiled( ):
    return config.compiler_profiled

382 383
def compiler_debugged( ):
    return config.compiler_debugged
384

385 386
# ---

387
def high_memory_usage(name, opts):
388 389
    opts.alone = True

390 391 392 393 394
# If a test is for a multi-CPU race, then running the test alone
# increases the chance that we'll actually see it.
def multi_cpu_race(name, opts):
    opts.alone = True

Ian Lynagh's avatar
Ian Lynagh committed
395
# ---
396
def literate( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
397 398
    opts.literate = 1;

399
def c_src( name, opts ):
400 401
    opts.c_src = 1;

402
def objc_src( name, opts ):
Austin Seipp's avatar
Austin Seipp committed
403 404
    opts.objc_src = 1;

405
def objcpp_src( name, opts ):
406 407
    opts.objcpp_src = 1;

408
def cmm_src( name, opts ):
409 410
    opts.cmm_src = 1;

411
def outputdir( odir ):
412
    return lambda name, opts, d=odir: _outputdir(name, opts, d)
413

414
def _outputdir( name, opts, odir ):
415 416
    opts.outputdir = odir;

417 418
# ----

419
def pre_cmd( cmd ):
420
    return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
421

422
def _pre_cmd( name, opts, cmd ):
423 424 425 426
    opts.pre_cmd = cmd

# ----

427
def clean_cmd( cmd ):
428
    return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
429

430
def _clean_cmd( name, opts, cmd ):
431 432 433 434
    opts.clean_cmd = cmd

# ----

435
def cmd_prefix( prefix ):
436
    return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
437

438
def _cmd_prefix( name, opts, prefix ):
439 440 441 442 443
    opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;

# ----

def cmd_wrapper( fun ):
444
    return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
445

446
def _cmd_wrapper( name, opts, fun ):
447
    opts.cmd_wrapper = fun
448

449 450
# ----

Ian Lynagh's avatar
Ian Lynagh committed
451
def compile_cmd_prefix( prefix ):
452
    return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
Ian Lynagh's avatar
Ian Lynagh committed
453

454
def _compile_cmd_prefix( name, opts, prefix ):
Ian Lynagh's avatar
Ian Lynagh committed
455 456 457 458
    opts.compile_cmd_prefix = prefix

# ----

459 460 461 462 463 464 465 466
def check_stdout( f ):
    return lambda name, opts, f=f: _check_stdout(name, opts, f)

def _check_stdout( name, opts, f ):
    opts.check_stdout = f

# ----

467
def normalise_slashes( name, opts ):
468
    _normalise_fun(name, opts, normalise_slashes_)
469

470
def normalise_exe( name, opts ):
471
    _normalise_fun(name, opts, normalise_exe_)
472

473 474
def normalise_fun( *fs ):
    return lambda name, opts: _normalise_fun(name, opts, fs)
475

476
def _normalise_fun( name, opts, *fs ):
477
    opts.extra_normaliser = join_normalisers(opts.extra_normaliser, fs)
478

479 480
def normalise_errmsg_fun( *fs ):
    return lambda name, opts: _normalise_errmsg_fun(name, opts, fs)
481

482
def _normalise_errmsg_fun( name, opts, *fs ):
483 484 485 486 487 488 489 490 491 492 493 494 495
    opts.extra_errmsg_normaliser =  join_normalisers(opts.extra_errmsg_normaliser, fs)

def normalise_version_( *pkgs ):
    def normalise_version__( str ):
        return re.sub('(' + '|'.join(map(re.escape,pkgs)) + ')-[0-9.]+',
                      '\\1-<VERSION>', str)
    return normalise_version__

def normalise_version( *pkgs ):
    def normalise_version__( name, opts ):
        _normalise_fun(name, opts, normalise_version_(*pkgs))
        _normalise_errmsg_fun(name, opts, normalise_version_(*pkgs))
    return normalise_version__
496

thomie's avatar
thomie committed
497 498 499 500
def normalise_drive_letter(name, opts):
    # Windows only. Change D:\\ to C:\\.
    _normalise_fun(name, opts, lambda str: re.sub(r'[A-Z]:\\', r'C:\\', str))

501 502 503 504 505 506 507
def keep_prof_callstacks(name, opts):
    """Keep profiling callstacks.

    Use together with `only_ways(prof_ways)`.
    """
    opts.keep_prof_callstacks = True

508 509
def join_normalisers(*a):
    """
510
    Compose functions, flattening sequences.
511

512
       join_normalisers(f1,[f2,f3],f4)
513 514 515

    is the same as

516
       lambda x: f1(f2(f3(f4(x))))
517 518
    """

519 520 521 522 523 524 525 526 527 528 529 530
    def flatten(l):
        """
        Taken from http://stackoverflow.com/a/2158532/946226
        """
        for el in l:
            if isinstance(el, collections.Iterable) and not isinstance(el, basestring):
                for sub in flatten(el):
                    yield sub
            else:
                yield el

    a = flatten(a)
531 532 533

    fn = lambda x:x # identity function
    for f in a:
534
        assert callable(f)
535 536 537
        fn = lambda x,f=f,fn=fn: fn(f(x))
    return fn

538 539 540
# ----
# Function for composing two opt-fns together

541
def executeSetups(fs, name, opts):
542
    if type(fs) is list:
543
        # If we have a list of setups, then execute each one
544 545
        for f in fs:
            executeSetups(f, name, opts)
546 547 548
    else:
        # fs is a single function, so just apply it
        fs(name, opts)
549

550 551 552
# -----------------------------------------------------------------------------
# The current directory of tests

553 554 555 556 557 558 559 560 561 562 563
def newTestDir(tempdir, dir):
    # Hack. A few tests depend on files in ancestor directories
    # (e.g. extra_files(['../../../../libraries/base/dist-install/haddock.t']))
    # Make sure tempdir is sufficiently "deep", such that copying/linking those
    # files won't cause any problems.
    #
    # If you received a framework failure about adding an extra level:
    #  * add one extra '../' to the startswith('../../../../../') in do_test
    #  * add one more number here:
    tempdir = os.path.join(tempdir, '1', '2', '3')

564
    global thisdir_settings
565
    # reset the options for this test directory
566 567 568
    def settings(name, opts, tempdir=tempdir, dir=dir):
        return _newTestDir(name, opts, tempdir, dir)
    thisdir_settings = settings
569

570 571 572 573

def _newTestDir(name, opts, tempdir, dir):
    opts.srcdir = os.path.join(os.getcwd(), dir)
    opts.testdir = os.path.join(tempdir, dir, name)
574
    opts.compiler_always_flags = config.compiler_always_flags
575 576 577 578

# -----------------------------------------------------------------------------
# Actually doing tests

579 580
parallelTests = []
aloneTests = []
581
allTestNames = set([])
582

583
def runTest (opts, name, func, args):
ei@vuokko.info's avatar
ei@vuokko.info committed
584 585
    ok = 0

586
    if config.use_threads:
ei@vuokko.info's avatar
ei@vuokko.info committed
587
        t.thread_pool.acquire()
588
        try:
589
            while config.threads<(t.running_threads+1):
590
                t.thread_pool.wait()
591
            t.running_threads = t.running_threads+1
592 593
            ok=1
            t.thread_pool.release()
594
            thread.start_new_thread(test_common_thread, (name, opts, func, args))
595 596 597 598 599
        except:
            if not ok:
                t.thread_pool.release()
    else:
        test_common_work (name, opts, func, args)
600

601
# name  :: String
602
# setup :: TestOpts -> IO ()
603
def test (name, setup, func, args):
604 605 606 607 608 609 610 611 612 613
    if config.run_only_some_tests:
        if name not in config.only:
            return
        else:
            # Note [Mutating config.only]
            # config.only is initiallly the set of tests requested by
            # the user (via 'make TEST='). We then remove all tests that
            # we've already seen (in .T files), so that we can later
            # report on any tests we couldn't find and error out.
            config.only.remove(name)
614

615 616
    global aloneTests
    global parallelTests
617
    global allTestNames
618
    global thisdir_settings
619 620
    if name in allTestNames:
        framework_fail(name, 'duplicate', 'There are multiple tests with this name')
621
    if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
622
        framework_fail(name, 'bad_name', 'This test has an invalid name')
623 624 625 626 627

    # Make a deep copy of the default_testopts, as we need our own copy
    # of any dictionaries etc inside it. Otherwise, if one test modifies
    # them, all tests will see the modified version!
    myTestOpts = copy.deepcopy(default_testopts)
628

629
    executeSetups([thisdir_settings, setup], name, myTestOpts)
630 631 632 633 634 635

    thisTest = lambda : runTest(myTestOpts, name, func, args)
    if myTestOpts.alone:
        aloneTests.append(thisTest)
    else:
        parallelTests.append(thisTest)
636
    allTestNames.add(name)
637

638
if config.use_threads:
639
    def test_common_thread(name, opts, func, args):
640 641 642 643 644 645
        t.lock.acquire()
        try:
            test_common_work(name,opts,func,args)
        finally:
            t.lock.release()
            t.thread_pool.acquire()
646
            t.running_threads = t.running_threads - 1
647 648
            t.thread_pool.notify()
            t.thread_pool.release()
649

650 651 652 653 654 655 656 657 658 659
def get_package_cache_timestamp():
    if config.package_conf_cache_file == '':
        return 0.0
    else:
        try:
            return os.stat(config.package_conf_cache_file).st_mtime
        except:
            return 0.0


ei@vuokko.info's avatar
ei@vuokko.info committed
660
def test_common_work (name, opts, func, args):
661 662 663 664 665 666 667 668 669
    try:
        t.total_tests = t.total_tests+1
        setLocalTestOpts(opts)

        package_conf_cache_file_start_timestamp = get_package_cache_timestamp()

        # All the ways we might run this test
        if func == compile or func == multimod_compile:
            all_ways = config.compile_ways
670
        elif func == compile_and_run or func == multimod_compile_and_run:
671 672 673 674 675 676
            all_ways = config.run_ways
        elif func == ghci_script:
            if 'ghci' in config.run_ways:
                all_ways = ['ghci']
            else:
                all_ways = []
677
        else:
678 679 680
            all_ways = ['normal']

        # A test itself can request extra ways by setting opts.extra_ways
681
        all_ways = all_ways + [way for way in opts.extra_ways if way not in all_ways]
682 683 684 685 686

        t.total_test_cases = t.total_test_cases + len(all_ways)

        ok_way = lambda way: \
            not getTestOpts().skip \
687
            and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
688
            and (config.cmdline_ways == [] or way in config.cmdline_ways) \
689
            and (not (config.skip_perf_tests and isStatsTest())) \
690 691 692
            and way not in getTestOpts().omit_ways

        # Which ways we are asked to skip
693
        do_ways = list(filter (ok_way,all_ways))
694

695 696
        # Only run all ways in slow mode.
        # See Note [validate and testsuite speed] in toplevel Makefile.
697 698
        if config.accept:
            # Only ever run one way
699
            do_ways = do_ways[:1]
700 701 702
        elif config.speed > 0:
            # However, if we EXPLICITLY asked for a way (with extra_ways)
            # please test it!
703 704
            explicit_ways = list(filter(lambda way: way in opts.extra_ways, do_ways))
            other_ways = list(filter(lambda way: way not in opts.extra_ways, do_ways))
705
            do_ways = other_ways[:1] + explicit_ways
706

707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734
        # Find all files in the source directory that this test
        # depends on. Do this only once for all ways.
        # Generously add all filenames that start with the name of
        # the test to this set, as a convenience to test authors.
        # They will have to use the `extra_files` setup function to
        # specify all other files that their test depends on (but
        # this seems to be necessary for only about 10% of all
        # tests).
        files = set((f for f in os.listdir(opts.srcdir)
                        if f.startswith(name)))
        for filename in (opts.extra_files + extra_src_files.get(name, [])):
            if filename.startswith('../../../../../'):
                framework_fail(name, 'whole-test',
                    'add extra level to testlib.py:newTestDir for: ' + filename)

            elif filename.startswith('/'):
                framework_fail(name, 'whole-test',
                    'no absolute paths in extra_files please: ' + filename)

            elif '*' in filename:
                # Don't use wildcards in extra_files too much, as
                # globbing is slow.
                files.update((os.path.relpath(f, opts.srcdir)
                            for f in glob.iglob(in_srcdir(filename))))

            else:
                files.add(filename)

735 736 737
        if not config.clean_only:
            # Run the required tests...
            for way in do_ways:
738 739
                if stopping():
                    break
740
                do_test(name, way, func, args, files)
741 742 743 744 745

            for way in all_ways:
                if way not in do_ways:
                    skiptest (name,way)

746
        if config.cleanup and (config.clean_only or do_ways):
747 748
            cleanup()
        elif False: # TODO. Delete this code.
749
            pretest_cleanup(name)
750 751
            clean([name + suff for suff in [
                       '', '.exe', '.exe.manifest', '.genscript',
752 753 754 755 756 757 758
                       '.stderr.normalised',        '.stdout.normalised',
                       '.run.stderr.normalised',    '.run.stdout.normalised',
                       '.comp.stderr.normalised',   '.comp.stdout.normalised',
                       '.interp.stderr.normalised', '.interp.stdout.normalised',
                       '.stats', '.comp.stats',
                       '.hi', '.o', '.prof', '.exe.prof', '.hc',
                       '_stub.h', '_stub.c', '_stub.o',
759
                       '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']])
760

761
            if func == multi_compile or func == multi_compile_fail:
762
                    extra_mods = args[1]
763 764 765
                    clean([replace_suffix(fx[0],'o') for fx in extra_mods])
                    clean([replace_suffix(fx[0], 'hi') for fx in extra_mods])

766 767

            clean(getTestOpts().clean_files)
Ian Lynagh's avatar
Ian Lynagh committed
768

769 770 771 772 773 774 775
            if getTestOpts().outputdir != None:
                odir = in_testdir(getTestOpts().outputdir)
                try:
                    shutil.rmtree(odir)
                except:
                    pass

776 777 778 779 780
            try:
                shutil.rmtree(in_testdir('.hpc.' + name))
            except:
                pass

781 782 783 784 785 786
            try:
                cleanCmd = getTestOpts().clean_cmd
                if cleanCmd != None:
                    result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
                    if result != 0:
                        framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
787
            except:
788
                framework_fail(name, 'cleaning', 'clean-command exception')
789

790
        package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
791

792 793
        if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
            framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
794

795 796 797 798 799 800 801 802 803 804
        try:
            for f in files_written[name]:
                if os.path.exists(f):
                    try:
                        if not f in files_written_not_removed[name]:
                            files_written_not_removed[name].append(f)
                    except:
                        files_written_not_removed[name] = [f]
        except:
            pass
805
    except Exception as e:
806
        framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
Ian Lynagh's avatar
Ian Lynagh committed
807

808
def clean(strs):
809
    return # TODO. Delete this function.
810
    for str in strs:
811 812 813 814 815
        if (str.endswith('.package.conf') or
            str.startswith('package.conf.') and not str.endswith('/*')):
            # Package confs are directories now.
            str += '/*'

816 817
        for name in glob.glob(in_testdir(str)):
            clean_full_path(name)
818

819
def clean_full_path(name):
Ian Lynagh's avatar
Ian Lynagh committed
820 821 822
        try:
            # Remove files...
            os.remove(name)
823
        except OSError as e1:
824
            try:
Ian Lynagh's avatar
Ian Lynagh committed
825 826
                # ... and empty directories
                os.rmdir(name)
827
            except OSError as e2:
Ian Lynagh's avatar
Ian Lynagh committed
828 829 830 831 832
                # We don't want to fail here, but we do want to know
                # what went wrong, so print out the exceptions.
                # ENOENT isn't a problem, though, as we clean files
                # that don't necessarily exist.
                if e1.errno != errno.ENOENT:
833
                    print(e1)
Ian Lynagh's avatar
Ian Lynagh committed
834
                if e2.errno != errno.ENOENT:
835
                    print(e2)
836

837 838 839
def do_test(name, way, func, args, files):
    opts = getTestOpts()

840 841 842
    full_name = name + '(' + way + ')'

    try:
843 844 845 846 847
        if_verbose(2, "=====> %s %d of %d %s " % \
                    (full_name, t.total_tests, len(allTestNames), \
                    [t.n_unexpected_passes, \
                     t.n_unexpected_failures, \
                     t.n_framework_failures]))
848

849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908
        # Clean up prior to the test, so that we can't spuriously conclude
        # that it passed on the basis of old run outputs.
        cleanup()

        # Link all source files for this test into a new directory in
        # /tmp, and run the test in that directory. This makes it
        # possible to run tests in parallel, without modification, that
        # would otherwise (accidentally) write to the same output file.
        # It also makes it easier to keep the testsuite clean.

        for filename in files:
            src = in_srcdir(filename)