testlib.py 71.7 KB
Newer Older
1
#
2 3 4
# (c) Simon Marlow 2002
#

5
from __future__ import print_function
6

7
import shutil
8 9
import sys
import os
10
import errno
11 12 13
import string
import re
import traceback
14 15
import time
import datetime
16
import copy
17
import glob
ian@well-typed.com's avatar
ian@well-typed.com committed
18
from math import ceil, trunc
19

20 21 22 23 24
have_subprocess = False
try:
    import subprocess
    have_subprocess = True
except:
25
    print("Warning: subprocess not found, will fall back to spawnv")
26

27
from testglobals import *
28 29
from testutil import *

30 31
if config.use_threads:
    import threading
32 33 34 35
    try:
        import thread
    except ImportError: # Python 3
        import _thread as thread
36

37 38 39 40 41 42 43 44
global wantToStop
wantToStop = False
def stopNow():
    global wantToStop
    wantToStop = True
def stopping():
    return wantToStop

45 46
# Options valid for the current test only (these get reset to
# testdir_testopts after each test).
47

ei@vuokko.info's avatar
ei@vuokko.info committed
48
global testopts_local
49 50 51 52 53 54
if config.use_threads:
    testopts_local = threading.local()
else:
    class TestOpts_Local:
        pass
    testopts_local = TestOpts_Local()
55 56

def getTestOpts():
ei@vuokko.info's avatar
ei@vuokko.info committed
57
    return testopts_local.x
58

ei@vuokko.info's avatar
ei@vuokko.info committed
59 60 61
def setLocalTestOpts(opts):
    global testopts_local
    testopts_local.x=opts
62

63 64 65 66 67
def isStatsTest():
    opts = getTestOpts()
    return len(opts.compiler_stats_range_fields) > 0 or len(opts.stats_range_fields) > 0


68 69 70
# This can be called at the top of a file of tests, to set default test options
# for the following tests.
def setTestOpts( f ):
71
    global thisdir_settings
72
    thisdir_settings = [thisdir_settings, f]
73 74 75 76 77 78 79 80 81 82 83 84

# -----------------------------------------------------------------------------
# Canned setup functions for common cases.  eg. for a test you might say
#
#      test('test001', normal, compile, [''])
#
# to run it without any options, but change it to
#
#      test('test001', expect_fail, compile, [''])
#
# to expect failure for this test.

85
def normal( name, opts ):
86 87
    return;

88
def skip( name, opts ):
89 90
    opts.skip = 1

91
def expect_fail( name, opts ):
92 93
    opts.expect = 'fail';

94
def reqlib( lib ):
95
    return lambda name, opts, l=lib: _reqlib (name, opts, l )
96

97 98 99 100
# Cache the results of looking to see if we have a library or not.
# This makes quite a difference, especially on Windows.
have_lib = {}

101
def _reqlib( name, opts, lib ):
102
    if lib in have_lib:
103
        got_it = have_lib[lib]
104
    else:
105 106 107
        if have_subprocess:
            # By preference we use subprocess, as the alternative uses
            # /dev/null which mingw doesn't have.
108
            p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
109 110 111 112 113 114 115 116 117 118 119 120 121
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
            # read from stdout and stderr to avoid blocking due to
            # buffers filling
            p.communicate()
            r = p.wait()
        else:
            r = os.system(config.ghc_pkg + ' describe ' + lib
                                         + ' > /dev/null 2> /dev/null')
        got_it = r == 0
        have_lib[lib] = got_it

    if not got_it:
122
        opts.expect = 'missing-lib'
123

124
def req_profiling( name, opts ):
125 126 127
    if not config.have_profiling:
        opts.expect = 'fail'

128
def req_shared_libs( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
129 130 131
    if not config.have_shared_libs:
        opts.expect = 'fail'

132
def req_interp( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
133 134 135
    if not config.have_interp:
        opts.expect = 'fail'

136
def req_smp( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
137 138 139
    if not config.have_smp:
        opts.expect = 'fail'

140
def ignore_output( name, opts ):
141 142
    opts.ignore_output = 1

143
def no_stdin( name, opts ):
144 145
    opts.no_stdin = 1

146
def combined_output( name, opts ):
pcapriotti's avatar
pcapriotti committed
147 148
    opts.combined_output = True

149 150 151
# -----

def expect_fail_for( ways ):
152
    return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
153

154
def _expect_fail_for( name, opts, ways ):
155 156
    opts.expect_fail_for = ways

157 158 159 160
def expect_broken( bug ):
    return lambda name, opts, b=bug: _expect_broken (name, opts, b )

def _expect_broken( name, opts, bug ):
161
    record_broken(name, opts, bug)
162 163
    opts.expect = 'fail';

Ian Lynagh's avatar
Ian Lynagh committed
164
def expect_broken_for( bug, ways ):
165
    return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
Ian Lynagh's avatar
Ian Lynagh committed
166

167
def _expect_broken_for( name, opts, bug, ways ):
168
    record_broken(name, opts, bug)
Ian Lynagh's avatar
Ian Lynagh committed
169
    opts.expect_fail_for = ways
Ian Lynagh's avatar
Ian Lynagh committed
170

171
def record_broken(name, opts, bug):
172
    global brokens
173
    me = (bug, opts.testdir, name)
174 175 176
    if not me in brokens:
        brokens.append(me)

177 178 179
# -----

def omit_ways( ways ):
180
    return lambda name, opts, w=ways: _omit_ways( name, opts, w )
181

182
def _omit_ways( name, opts, ways ):
183 184 185 186
    opts.omit_ways = ways

# -----

187
def only_ways( ways ):
188
    return lambda name, opts, w=ways: _only_ways( name, opts, w )
189

190
def _only_ways( name, opts, ways ):
191 192 193 194
    opts.only_ways = ways

# -----

195
def extra_ways( ways ):
196
    return lambda name, opts, w=ways: _extra_ways( name, opts, w )
197

198
def _extra_ways( name, opts, ways ):
199 200 201 202
    opts.extra_ways = ways

# -----

ross's avatar
ross committed
203
def omit_compiler_types( compiler_types ):
204
   return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
ross's avatar
ross committed
205

206
def _omit_compiler_types( name, opts, compiler_types ):
ross's avatar
ross committed
207
    if config.compiler_type in compiler_types:
dterei's avatar
dterei committed
208
        opts.skip = 1
ross's avatar
ross committed
209 210 211 212

# -----

def only_compiler_types( compiler_types ):
213
   return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
ross's avatar
ross committed
214

215
def _only_compiler_types( name, opts, compiler_types ):
ross's avatar
ross committed
216
    if config.compiler_type not in compiler_types:
dterei's avatar
dterei committed
217
        opts.skip = 1
ross's avatar
ross committed
218 219 220

# -----

221
def set_stdin( file ):
222
   return lambda name, opts, f=file: _set_stdin(name, opts, f);
223

224
def _set_stdin( name, opts, f ):
225 226 227 228 229
   opts.stdin = f

# -----

def exit_code( val ):
230
    return lambda name, opts, v=val: _exit_code(name, opts, v);
231

232
def _exit_code( name, opts, v ):
233 234
    opts.exit_code = v

235 236 237 238 239 240 241 242 243 244 245
def signal_exit_code( val ):
    if opsys('solaris2'):
        return exit_code( val );
    else:
        # When application running on Linux receives fatal error
        # signal, then its exit code is encoded as 128 + signal
        # value. See http://www.tldp.org/LDP/abs/html/exitcodes.html
        # I assume that Mac OS X behaves in the same way at least Mac
        # OS X builder behavior suggests this.
        return exit_code( val+128 );

246 247
# -----

248
def timeout_multiplier( val ):
249
    return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
250

251
def _timeout_multiplier( name, opts, v ):
252 253 254 255
    opts.timeout_multiplier = v

# -----

256
def extra_run_opts( val ):
257
    return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
258

259
def _extra_run_opts( name, opts, v ):
260 261
    opts.extra_run_opts = v

262 263
# -----

Simon Marlow's avatar
Simon Marlow committed
264
def extra_hc_opts( val ):
265
    return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
Simon Marlow's avatar
Simon Marlow committed
266

267
def _extra_hc_opts( name, opts, v ):
Simon Marlow's avatar
Simon Marlow committed
268 269 270 271
    opts.extra_hc_opts = v

# -----

272
def extra_clean( files ):
273
    return lambda name, opts, v=files: _extra_clean(name, opts, v);
274

275
def _extra_clean( name, opts, v ):
276 277
    opts.clean_files = v

278 279
# -----

280 281 282 283 284 285 286
def stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);

def _stats_num_field( name, opts, field, expecteds ):
    if field in opts.stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

287
    if type(expecteds) is list:
288 289 290 291 292
        for (b, expected, dev) in expecteds:
            if b:
                opts.stats_range_fields[field] = (expected, dev)
                return
        framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
293

294 295 296
    else:
        (expected, dev) = expecteds
        opts.stats_range_fields[field] = (expected, dev)
297 298 299

def compiler_stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
300

301 302 303 304
def _compiler_stats_num_field( name, opts, field, expecteds ):
    if field in opts.compiler_stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

305 306
    # Compiler performance numbers change when debugging is on, making the results
    # useless and confusing. Therefore, skip if debugging is on.
307 308 309
    if compiler_debugged():
        skip(name, opts)

310 311 312 313
    for (b, expected, dev) in expecteds:
        if b:
            opts.compiler_stats_range_fields[field] = (expected, dev)
            return
314

315 316
    framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')

317 318
# -----

319
def when(b, f):
ian@well-typed.com's avatar
ian@well-typed.com committed
320 321 322
    # When list_brokens is on, we want to see all expect_broken calls,
    # so we always do f
    if b or config.list_broken:
323 324 325 326 327 328 329
        return f
    else:
        return normal

def unless(b, f):
    return when(not b, f)

ian@well-typed.com's avatar
ian@well-typed.com committed
330 331 332
def doing_ghci():
    return 'ghci' in config.run_ways

333
def ghci_dynamic( ):
ian@well-typed.com's avatar
ian@well-typed.com committed
334
    return config.ghc_dynamic
335 336 337 338

def fast():
    return config.fast

339 340
def platform( plat ):
    return config.platform == plat
Ian Lynagh's avatar
Ian Lynagh committed
341

342 343
def opsys( os ):
    return config.os == os
Ian Lynagh's avatar
Ian Lynagh committed
344

345 346
def arch( arch ):
    return config.arch == arch
347

348 349
def wordsize( ws ):
    return config.wordsize == str(ws)
tibbe's avatar
tibbe committed
350

351 352
def msys( ):
    return config.msys
ian@well-typed.com's avatar
ian@well-typed.com committed
353

354 355
def cygwin( ):
    return config.cygwin
Ian Lynagh's avatar
Ian Lynagh committed
356

357 358
def have_vanilla( ):
    return config.have_vanilla
359

360 361
def have_dynamic( ):
    return config.have_dynamic
362

363 364
def have_profiling( ):
    return config.have_profiling
365

366 367
def in_tree_compiler( ):
    return config.in_tree_compiler
368

369 370
def compiler_type( compiler ):
    return config.compiler_type == compiler
371

372 373 374
def compiler_lt( compiler, version ):
    return config.compiler_type == compiler and \
           version_lt(config.compiler_version, version)
375

376 377 378
def compiler_le( compiler, version ):
    return config.compiler_type == compiler and \
           version_le(config.compiler_version, version)
379

380 381 382
def compiler_gt( compiler, version ):
    return config.compiler_type == compiler and \
           version_gt(config.compiler_version, version)
383

384 385 386
def compiler_ge( compiler, version ):
    return config.compiler_type == compiler and \
           version_ge(config.compiler_version, version)
387

388 389 390 391 392 393
def unregisterised( ):
    return config.unregisterised

def compiler_profiled( ):
    return config.compiler_profiled

394 395
def compiler_debugged( ):
    return config.compiler_debugged
396

397 398 399 400 401
def tag( t ):
    return t in config.compiler_tags

# ---

Ian Lynagh's avatar
Ian Lynagh committed
402 403
def namebase( nb ):
   return lambda opts, nb=nb: _namebase(opts, nb)
404

Ian Lynagh's avatar
Ian Lynagh committed
405 406
def _namebase( opts, nb ):
    opts.with_namebase = nb
407

408 409
# ---

410
def high_memory_usage(name, opts):
411 412
    opts.alone = True

413 414 415 416 417
# If a test is for a multi-CPU race, then running the test alone
# increases the chance that we'll actually see it.
def multi_cpu_race(name, opts):
    opts.alone = True

Ian Lynagh's avatar
Ian Lynagh committed
418
# ---
419
def literate( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
420 421
    opts.literate = 1;

422
def c_src( name, opts ):
423 424
    opts.c_src = 1;

425
def objc_src( name, opts ):
Austin Seipp's avatar
Austin Seipp committed
426 427
    opts.objc_src = 1;

428
def objcpp_src( name, opts ):
429 430
    opts.objcpp_src = 1;

431
def cmm_src( name, opts ):
432 433
    opts.cmm_src = 1;

434
def outputdir( odir ):
435
    return lambda name, opts, d=odir: _outputdir(name, opts, d)
436

437
def _outputdir( name, opts, odir ):
438 439
    opts.outputdir = odir;

440 441
# ----

442
def pre_cmd( cmd ):
443
    return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
444

445
def _pre_cmd( name, opts, cmd ):
446 447 448 449
    opts.pre_cmd = cmd

# ----

450
def clean_cmd( cmd ):
451
    return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
452

453
def _clean_cmd( name, opts, cmd ):
454 455 456 457
    opts.clean_cmd = cmd

# ----

458
def cmd_prefix( prefix ):
459
    return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
460

461
def _cmd_prefix( name, opts, prefix ):
462 463 464 465 466
    opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;

# ----

def cmd_wrapper( fun ):
467
    return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
468

469
def _cmd_wrapper( name, opts, fun ):
470
    opts.cmd_wrapper = fun
471

472 473
# ----

Ian Lynagh's avatar
Ian Lynagh committed
474
def compile_cmd_prefix( prefix ):
475
    return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
Ian Lynagh's avatar
Ian Lynagh committed
476

477
def _compile_cmd_prefix( name, opts, prefix ):
Ian Lynagh's avatar
Ian Lynagh committed
478 479 480 481
    opts.compile_cmd_prefix = prefix

# ----

482 483 484 485 486 487 488 489
def check_stdout( f ):
    return lambda name, opts, f=f: _check_stdout(name, opts, f)

def _check_stdout( name, opts, f ):
    opts.check_stdout = f

# ----

490
def normalise_slashes( name, opts ):
491 492
    opts.extra_normaliser = normalise_slashes_

493
def normalise_exe( name, opts ):
494 495
    opts.extra_normaliser = normalise_exe_

496
def normalise_fun( fun ):
497
    return lambda name, opts, f=fun: _normalise_fun(name, opts, f)
498

499
def _normalise_fun( name, opts, f ):
500 501
    opts.extra_normaliser = f

502
def normalise_errmsg_fun( fun ):
503
    return lambda name, opts, f=fun: _normalise_errmsg_fun(name, opts, f)
504

505
def _normalise_errmsg_fun( name, opts, f ):
506 507 508 509 510
    opts.extra_errmsg_normaliser = f

def two_normalisers(f, g):
    return lambda x, f=f, g=g: f(g(x))

511 512 513
# ----
# Function for composing two opt-fns together

514
def executeSetups(fs, name, opts):
515
    if type(fs) is list:
516
        # If we have a list of setups, then execute each one
517 518
        for f in fs:
            executeSetups(f, name, opts)
519 520 521
    else:
        # fs is a single function, so just apply it
        fs(name, opts)
522

523 524 525 526
# -----------------------------------------------------------------------------
# The current directory of tests

def newTestDir( dir ):
527
    global thisdir_settings
528
    # reset the options for this test directory
529 530 531 532 533
    thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )

def _newTestDir( name, opts, dir ):
    opts.testdir = dir
    opts.compiler_always_flags = config.compiler_always_flags
534 535 536 537

# -----------------------------------------------------------------------------
# Actually doing tests

538 539
parallelTests = []
aloneTests = []
540
allTestNames = set([])
541

542
def runTest (opts, name, func, args):
ei@vuokko.info's avatar
ei@vuokko.info committed
543 544
    ok = 0

545
    if config.use_threads:
ei@vuokko.info's avatar
ei@vuokko.info committed
546
        t.thread_pool.acquire()
547
        try:
548
            while config.threads<(t.running_threads+1):
549
                t.thread_pool.wait()
550
            t.running_threads = t.running_threads+1
551 552
            ok=1
            t.thread_pool.release()
553
            thread.start_new_thread(test_common_thread, (name, opts, func, args))
554 555 556 557 558
        except:
            if not ok:
                t.thread_pool.release()
    else:
        test_common_work (name, opts, func, args)
559

560
# name  :: String
561
# setup :: TestOpts -> IO ()
562
def test (name, setup, func, args):
563 564
    global aloneTests
    global parallelTests
565
    global allTestNames
566
    global thisdir_settings
567 568
    if name in allTestNames:
        framework_fail(name, 'duplicate', 'There are multiple tests with this name')
569
    if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
570
        framework_fail(name, 'bad_name', 'This test has an invalid name')
571 572 573 574 575

    # Make a deep copy of the default_testopts, as we need our own copy
    # of any dictionaries etc inside it. Otherwise, if one test modifies
    # them, all tests will see the modified version!
    myTestOpts = copy.deepcopy(default_testopts)
576

577
    executeSetups([thisdir_settings, setup], name, myTestOpts)
578 579 580 581 582 583

    thisTest = lambda : runTest(myTestOpts, name, func, args)
    if myTestOpts.alone:
        aloneTests.append(thisTest)
    else:
        parallelTests.append(thisTest)
584
    allTestNames.add(name)
585

586
if config.use_threads:
587
    def test_common_thread(name, opts, func, args):
588 589 590 591 592 593
        t.lock.acquire()
        try:
            test_common_work(name,opts,func,args)
        finally:
            t.lock.release()
            t.thread_pool.acquire()
594
            t.running_threads = t.running_threads - 1
595 596
            t.thread_pool.notify()
            t.thread_pool.release()
597

598 599 600 601 602 603 604 605 606 607
def get_package_cache_timestamp():
    if config.package_conf_cache_file == '':
        return 0.0
    else:
        try:
            return os.stat(config.package_conf_cache_file).st_mtime
        except:
            return 0.0


ei@vuokko.info's avatar
ei@vuokko.info committed
608
def test_common_work (name, opts, func, args):
609 610 611 612 613 614 615 616 617
    try:
        t.total_tests = t.total_tests+1
        setLocalTestOpts(opts)

        package_conf_cache_file_start_timestamp = get_package_cache_timestamp()

        # All the ways we might run this test
        if func == compile or func == multimod_compile:
            all_ways = config.compile_ways
618
        elif func == compile_and_run or func == multimod_compile_and_run:
619 620 621 622 623 624
            all_ways = config.run_ways
        elif func == ghci_script:
            if 'ghci' in config.run_ways:
                all_ways = ['ghci']
            else:
                all_ways = []
625
        else:
626 627 628
            all_ways = ['normal']

        # A test itself can request extra ways by setting opts.extra_ways
629
        all_ways = all_ways + [way for way in opts.extra_ways if way not in all_ways]
630 631 632 633 634 635

        t.total_test_cases = t.total_test_cases + len(all_ways)

        ok_way = lambda way: \
            not getTestOpts().skip \
            and (config.only == [] or name in config.only) \
636
            and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
637
            and (config.cmdline_ways == [] or way in config.cmdline_ways) \
638
            and (not (config.skip_perf_tests and isStatsTest())) \
639 640 641
            and way not in getTestOpts().omit_ways

        # Which ways we are asked to skip
642
        do_ways = list(filter (ok_way,all_ways))
643 644 645 646 647 648 649 650

        # In fast mode, we skip all but one way
        if config.fast and len(do_ways) > 0:
            do_ways = [do_ways[0]]

        if not config.clean_only:
            # Run the required tests...
            for way in do_ways:
651 652
                if stopping():
                    break
653 654 655 656 657 658 659
                do_test (name, way, func, args)

            for way in all_ways:
                if way not in do_ways:
                    skiptest (name,way)

        if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
660
            pretest_cleanup(name)
661 662
            clean([name + suff for suff in [
                       '', '.exe', '.exe.manifest', '.genscript',
663 664 665 666 667 668 669
                       '.stderr.normalised',        '.stdout.normalised',
                       '.run.stderr.normalised',    '.run.stdout.normalised',
                       '.comp.stderr.normalised',   '.comp.stdout.normalised',
                       '.interp.stderr.normalised', '.interp.stdout.normalised',
                       '.stats', '.comp.stats',
                       '.hi', '.o', '.prof', '.exe.prof', '.hc',
                       '_stub.h', '_stub.c', '_stub.o',
670
                       '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']])
671

672
            if func == multi_compile or func == multi_compile_fail:
673
                    extra_mods = args[1]
674 675 676
                    clean([replace_suffix(fx[0],'o') for fx in extra_mods])
                    clean([replace_suffix(fx[0], 'hi') for fx in extra_mods])

677 678

            clean(getTestOpts().clean_files)
Ian Lynagh's avatar
Ian Lynagh committed
679

680 681 682 683 684 685 686
            if getTestOpts().outputdir != None:
                odir = in_testdir(getTestOpts().outputdir)
                try:
                    shutil.rmtree(odir)
                except:
                    pass

687 688 689 690 691
            try:
                shutil.rmtree(in_testdir('.hpc.' + name))
            except:
                pass

692 693 694 695 696 697
            try:
                cleanCmd = getTestOpts().clean_cmd
                if cleanCmd != None:
                    result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
                    if result != 0:
                        framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
698
            except:
699
                framework_fail(name, 'cleaning', 'clean-command exception')
700

701
        package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
702

703 704
        if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
            framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
705

706 707 708 709 710 711 712 713 714 715
        try:
            for f in files_written[name]:
                if os.path.exists(f):
                    try:
                        if not f in files_written_not_removed[name]:
                            files_written_not_removed[name].append(f)
                    except:
                        files_written_not_removed[name] = [f]
        except:
            pass
716
    except Exception as e:
717
        framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
Ian Lynagh's avatar
Ian Lynagh committed
718

719 720 721 722
def clean(strs):
    for str in strs:
        for name in glob.glob(in_testdir(str)):
            clean_full_path(name)
723

724
def clean_full_path(name):
Ian Lynagh's avatar
Ian Lynagh committed
725 726 727
        try:
            # Remove files...
            os.remove(name)
728
        except OSError as e1:
729
            try:
Ian Lynagh's avatar
Ian Lynagh committed
730 731
                # ... and empty directories
                os.rmdir(name)
732
            except OSError as e2:
Ian Lynagh's avatar
Ian Lynagh committed
733 734 735 736 737
                # We don't want to fail here, but we do want to know
                # what went wrong, so print out the exceptions.
                # ENOENT isn't a problem, though, as we clean files
                # that don't necessarily exist.
                if e1.errno != errno.ENOENT:
738
                    print(e1)
Ian Lynagh's avatar
Ian Lynagh committed
739
                if e2.errno != errno.ENOENT:
740
                    print(e2)
741

742
def do_test(name, way, func, args):
743 744 745
    full_name = name + '(' + way + ')'

    try:
746 747 748 749 750
        if_verbose(2, "=====> %s %d of %d %s " % \
                    (full_name, t.total_tests, len(allTestNames), \
                    [t.n_unexpected_passes, \
                     t.n_unexpected_failures, \
                     t.n_framework_failures]))
751

752 753
        if config.use_threads:
            t.lock.release()
754 755 756 757

        try:
            preCmd = getTestOpts().pre_cmd
            if preCmd != None:
758
                result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
759 760
                if result != 0:
                    framework_fail(name, way, 'pre-command failed: ' + str(result))
761
        except:
762 763
            framework_fail(name, way, 'pre-command exception')

ei@vuokko.info's avatar
ei@vuokko.info committed
764
        try:
765
            result = func(*[name,way] + args)
ei@vuokko.info's avatar
ei@vuokko.info committed
766
        finally:
767 768
            if config.use_threads:
                t.lock.acquire()
769

770 771 772
        if getTestOpts().expect != 'pass' and \
                getTestOpts().expect != 'fail' and \
                getTestOpts().expect != 'missing-lib':
773
            framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
774

775 776 777 778 779 780
        try:
            passFail = result['passFail']
        except:
            passFail = 'No passFail found'

        if passFail == 'pass':
ei@vuokko.info's avatar
ei@vuokko.info committed
781 782
            if getTestOpts().expect == 'pass' \
               and way not in getTestOpts().expect_fail_for:
783
                t.n_expected_passes = t.n_expected_passes + 1
784 785 786 787
                if name in t.expected_passes:
                    t.expected_passes[name].append(way)
                else:
                    t.expected_passes[name] = [way]
788
            else:
789
                if_verbose(1, '*** unexpected pass for %s' % full_name)
790
                t.n_unexpected_passes = t.n_unexpected_passes + 1
791
                addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
792
        elif passFail == 'fail':
ei@vuokko.info's avatar
ei@vuokko.info committed
793 794
            if getTestOpts().expect == 'pass' \
               and way not in getTestOpts().expect_fail_for:
795
                if_verbose(1, '*** unexpected failure for %s' % full_name)
796
                t.n_unexpected_failures = t.n_unexpected_failures + 1
797 798
                reason = result['reason']
                addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
799
            else:
800 801 802 803 804 805
                if getTestOpts().expect == 'missing-lib':
                    t.n_missing_libs = t.n_missing_libs + 1
                    if name in t.missing_libs:
                        t.missing_libs[name].append(way)
                    else:
                        t.missing_libs[name] = [way]
806
                else:
807 808 809 810 811
                    t.n_expected_failures = t.n_expected_failures + 1
                    if name in t.expected_failures:
                        t.expected_failures[name].append(way)
                    else:
                        t.expected_failures[name] = [way]
812 813
        else:
            framework_fail(name, way, 'bad result ' + passFail)
814
    except KeyboardInterrupt:
815
        stopNow()
816
    except:
817
        framework_fail(name, way, 'do_test exception')
818 819
        traceback.print_exc()

820
def addPassingTestInfo (testInfos, directory, name