testlib.py 69.7 KB
Newer Older
1
#
2 3 4
# (c) Simon Marlow 2002
#

5 6 7
# This allows us to use the "with X:" syntax with python 2.5:
from __future__ import with_statement

8
import shutil
9 10
import sys
import os
11
import errno
12 13 14
import string
import re
import traceback
15 16
import time
import datetime
17
import copy
18
import glob
19
import types
ian@well-typed.com's avatar
ian@well-typed.com committed
20
from math import ceil, trunc
21

22 23 24 25 26 27 28
have_subprocess = False
try:
    import subprocess
    have_subprocess = True
except:
    print "Warning: subprocess not found, will fall back to spawnv"

29
from string import join
30
from testglobals import *
31 32
from testutil import *

33 34 35
if config.use_threads:
    import threading
    import thread
36

37 38 39 40 41 42 43 44
global wantToStop
wantToStop = False
def stopNow():
    global wantToStop
    wantToStop = True
def stopping():
    return wantToStop

45 46
# Options valid for the current test only (these get reset to
# testdir_testopts after each test).
47

ei@vuokko.info's avatar
ei@vuokko.info committed
48
global testopts_local
49 50 51 52 53 54
if config.use_threads:
    testopts_local = threading.local()
else:
    class TestOpts_Local:
        pass
    testopts_local = TestOpts_Local()
55 56

def getTestOpts():
ei@vuokko.info's avatar
ei@vuokko.info committed
57
    return testopts_local.x
58

ei@vuokko.info's avatar
ei@vuokko.info committed
59 60 61
def setLocalTestOpts(opts):
    global testopts_local
    testopts_local.x=opts
62

63 64 65 66 67
def isStatsTest():
    opts = getTestOpts()
    return len(opts.compiler_stats_range_fields) > 0 or len(opts.stats_range_fields) > 0


68 69 70
# This can be called at the top of a file of tests, to set default test options
# for the following tests.
def setTestOpts( f ):
71
    global thisdir_settings
72
    thisdir_settings = [thisdir_settings, f]
73 74 75 76 77 78 79 80 81 82 83 84

# -----------------------------------------------------------------------------
# Canned setup functions for common cases.  eg. for a test you might say
#
#      test('test001', normal, compile, [''])
#
# to run it without any options, but change it to
#
#      test('test001', expect_fail, compile, [''])
#
# to expect failure for this test.

85
def normal( name, opts ):
86 87
    return;

88
def skip( name, opts ):
89 90
    opts.skip = 1

91
def expect_fail( name, opts ):
92 93
    opts.expect = 'fail';

94
def reqlib( lib ):
95
    return lambda name, opts, l=lib: _reqlib (name, opts, l )
96

97 98 99 100
# Cache the results of looking to see if we have a library or not.
# This makes quite a difference, especially on Windows.
have_lib = {}

101
def _reqlib( name, opts, lib ):
102 103
    if have_lib.has_key(lib):
        got_it = have_lib[lib]
104
    else:
105 106 107
        if have_subprocess:
            # By preference we use subprocess, as the alternative uses
            # /dev/null which mingw doesn't have.
108
            p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
109 110 111 112 113 114 115 116 117 118 119 120 121
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
            # read from stdout and stderr to avoid blocking due to
            # buffers filling
            p.communicate()
            r = p.wait()
        else:
            r = os.system(config.ghc_pkg + ' describe ' + lib
                                         + ' > /dev/null 2> /dev/null')
        got_it = r == 0
        have_lib[lib] = got_it

    if not got_it:
122
        opts.expect = 'missing-lib'
123

124
def req_profiling( name, opts ):
125 126 127
    if not config.have_profiling:
        opts.expect = 'fail'

128
def req_shared_libs( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
129 130 131
    if not config.have_shared_libs:
        opts.expect = 'fail'

132
def req_interp( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
133 134 135
    if not config.have_interp:
        opts.expect = 'fail'

136
def req_smp( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
137 138 139
    if not config.have_smp:
        opts.expect = 'fail'

140
def ignore_output( name, opts ):
141 142
    opts.ignore_output = 1

143
def no_stdin( name, opts ):
144 145
    opts.no_stdin = 1

146
def combined_output( name, opts ):
pcapriotti's avatar
pcapriotti committed
147 148
    opts.combined_output = True

149 150 151
# -----

def expect_fail_for( ways ):
152
    return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
153

154
def _expect_fail_for( name, opts, ways ):
155 156
    opts.expect_fail_for = ways

157 158 159 160
def expect_broken( bug ):
    return lambda name, opts, b=bug: _expect_broken (name, opts, b )

def _expect_broken( name, opts, bug ):
161
    record_broken(name, opts, bug)
162 163
    opts.expect = 'fail';

Ian Lynagh's avatar
Ian Lynagh committed
164
def expect_broken_for( bug, ways ):
165
    return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
Ian Lynagh's avatar
Ian Lynagh committed
166

167
def _expect_broken_for( name, opts, bug, ways ):
168
    record_broken(name, opts, bug)
Ian Lynagh's avatar
Ian Lynagh committed
169
    opts.expect_fail_for = ways
Ian Lynagh's avatar
Ian Lynagh committed
170

171
def record_broken(name, opts, bug):
172
    global brokens
173
    me = (bug, opts.testdir, name)
174 175 176
    if not me in brokens:
        brokens.append(me)

177 178 179
# -----

def omit_ways( ways ):
180
    return lambda name, opts, w=ways: _omit_ways( name, opts, w )
181

182
def _omit_ways( name, opts, ways ):
183 184 185 186
    opts.omit_ways = ways

# -----

187
def only_ways( ways ):
188
    return lambda name, opts, w=ways: _only_ways( name, opts, w )
189

190
def _only_ways( name, opts, ways ):
191 192 193 194
    opts.only_ways = ways

# -----

195
def extra_ways( ways ):
196
    return lambda name, opts, w=ways: _extra_ways( name, opts, w )
197

198
def _extra_ways( name, opts, ways ):
199 200 201 202
    opts.extra_ways = ways

# -----

ross's avatar
ross committed
203
def omit_compiler_types( compiler_types ):
204
   return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
ross's avatar
ross committed
205

206
def _omit_compiler_types( name, opts, compiler_types ):
ross's avatar
ross committed
207
    if config.compiler_type in compiler_types:
dterei's avatar
dterei committed
208
        opts.skip = 1
ross's avatar
ross committed
209 210 211 212

# -----

def only_compiler_types( compiler_types ):
213
   return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
ross's avatar
ross committed
214

215
def _only_compiler_types( name, opts, compiler_types ):
ross's avatar
ross committed
216
    if config.compiler_type not in compiler_types:
dterei's avatar
dterei committed
217
        opts.skip = 1
ross's avatar
ross committed
218 219 220

# -----

221
def set_stdin( file ):
222
   return lambda name, opts, f=file: _set_stdin(name, opts, f);
223

224
def _set_stdin( name, opts, f ):
225 226 227 228 229
   opts.stdin = f

# -----

def exit_code( val ):
230
    return lambda name, opts, v=val: _exit_code(name, opts, v);
231

232
def _exit_code( name, opts, v ):
233 234 235 236
    opts.exit_code = v

# -----

237
def timeout_multiplier( val ):
238
    return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
239

240
def _timeout_multiplier( name, opts, v ):
241 242 243 244
    opts.timeout_multiplier = v

# -----

245
def extra_run_opts( val ):
246
    return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
247

248
def _extra_run_opts( name, opts, v ):
249 250
    opts.extra_run_opts = v

251 252
# -----

Simon Marlow's avatar
Simon Marlow committed
253
def extra_hc_opts( val ):
254
    return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
Simon Marlow's avatar
Simon Marlow committed
255

256
def _extra_hc_opts( name, opts, v ):
Simon Marlow's avatar
Simon Marlow committed
257 258 259 260
    opts.extra_hc_opts = v

# -----

261
def extra_clean( files ):
262
    return lambda name, opts, v=files: _extra_clean(name, opts, v);
263

264
def _extra_clean( name, opts, v ):
265 266
    opts.clean_files = v

267 268
# -----

269 270 271 272 273 274 275
def stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);

def _stats_num_field( name, opts, field, expecteds ):
    if field in opts.stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

276 277 278 279 280 281
    if type(expecteds) is types.ListType:
        for (b, expected, dev) in expecteds:
            if b:
                opts.stats_range_fields[field] = (expected, dev)
                return
        framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
282

283 284 285
    else:
        (expected, dev) = expecteds
        opts.stats_range_fields[field] = (expected, dev)
286 287 288

def compiler_stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
289

290 291 292 293
def _compiler_stats_num_field( name, opts, field, expecteds ):
    if field in opts.compiler_stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

294 295
    # Compiler performance numbers change when debugging is on, making the results
    # useless and confusing. Therefore, skip if debugging is on.
296 297 298
    if compiler_debugged():
        skip(name, opts)

299 300 301 302
    for (b, expected, dev) in expecteds:
        if b:
            opts.compiler_stats_range_fields[field] = (expected, dev)
            return
303

304 305
    framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')

306 307
# -----

308
def when(b, f):
ian@well-typed.com's avatar
ian@well-typed.com committed
309 310 311
    # When list_brokens is on, we want to see all expect_broken calls,
    # so we always do f
    if b or config.list_broken:
312 313 314 315 316 317 318
        return f
    else:
        return normal

def unless(b, f):
    return when(not b, f)

ian@well-typed.com's avatar
ian@well-typed.com committed
319 320 321
def doing_ghci():
    return 'ghci' in config.run_ways

322
def ghci_dynamic( ):
ian@well-typed.com's avatar
ian@well-typed.com committed
323
    return config.ghc_dynamic
324 325 326 327

def fast():
    return config.fast

328 329
def platform( plat ):
    return config.platform == plat
Ian Lynagh's avatar
Ian Lynagh committed
330

331 332
def opsys( os ):
    return config.os == os
Ian Lynagh's avatar
Ian Lynagh committed
333

334 335
def arch( arch ):
    return config.arch == arch
336

337 338
def wordsize( ws ):
    return config.wordsize == str(ws)
tibbe's avatar
tibbe committed
339

340 341
def msys( ):
    return config.msys
ian@well-typed.com's avatar
ian@well-typed.com committed
342

343 344
def cygwin( ):
    return config.cygwin
Ian Lynagh's avatar
Ian Lynagh committed
345

346 347
def have_vanilla( ):
    return config.have_vanilla
348

349 350
def have_dynamic( ):
    return config.have_dynamic
351

352 353
def have_profiling( ):
    return config.have_profiling
354

355 356
def in_tree_compiler( ):
    return config.in_tree_compiler
357

358 359
def compiler_type( compiler ):
    return config.compiler_type == compiler
360

361 362 363
def compiler_lt( compiler, version ):
    return config.compiler_type == compiler and \
           version_lt(config.compiler_version, version)
364

365 366 367
def compiler_le( compiler, version ):
    return config.compiler_type == compiler and \
           version_le(config.compiler_version, version)
368

369 370 371
def compiler_gt( compiler, version ):
    return config.compiler_type == compiler and \
           version_gt(config.compiler_version, version)
372

373 374 375
def compiler_ge( compiler, version ):
    return config.compiler_type == compiler and \
           version_ge(config.compiler_version, version)
376

377 378 379 380 381 382
def unregisterised( ):
    return config.unregisterised

def compiler_profiled( ):
    return config.compiler_profiled

383 384
def compiler_debugged( ):
    return config.compiler_debugged
385

386 387 388 389 390
def tag( t ):
    return t in config.compiler_tags

# ---

Ian Lynagh's avatar
Ian Lynagh committed
391 392
def namebase( nb ):
   return lambda opts, nb=nb: _namebase(opts, nb)
393

Ian Lynagh's avatar
Ian Lynagh committed
394 395
def _namebase( opts, nb ):
    opts.with_namebase = nb
396

397 398
# ---

399
def high_memory_usage(name, opts):
400 401
    opts.alone = True

402 403 404 405 406
# If a test is for a multi-CPU race, then running the test alone
# increases the chance that we'll actually see it.
def multi_cpu_race(name, opts):
    opts.alone = True

Ian Lynagh's avatar
Ian Lynagh committed
407
# ---
408
def literate( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
409 410
    opts.literate = 1;

411
def c_src( name, opts ):
412 413
    opts.c_src = 1;

414
def objc_src( name, opts ):
Austin Seipp's avatar
Austin Seipp committed
415 416
    opts.objc_src = 1;

417
def objcpp_src( name, opts ):
418 419
    opts.objcpp_src = 1;

420
def cmm_src( name, opts ):
421 422
    opts.cmm_src = 1;

423
def outputdir( odir ):
424
    return lambda name, opts, d=odir: _outputdir(name, opts, d)
425

426
def _outputdir( name, opts, odir ):
427 428
    opts.outputdir = odir;

429 430
# ----

431
def pre_cmd( cmd ):
432
    return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
433

434
def _pre_cmd( name, opts, cmd ):
435 436 437 438
    opts.pre_cmd = cmd

# ----

439
def clean_cmd( cmd ):
440
    return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
441

442
def _clean_cmd( name, opts, cmd ):
443 444 445 446
    opts.clean_cmd = cmd

# ----

447
def cmd_prefix( prefix ):
448
    return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
449

450
def _cmd_prefix( name, opts, prefix ):
451 452 453 454 455
    opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;

# ----

def cmd_wrapper( fun ):
456
    return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
457

458
def _cmd_wrapper( name, opts, fun ):
459
    opts.cmd_wrapper = fun
460

461 462
# ----

Ian Lynagh's avatar
Ian Lynagh committed
463
def compile_cmd_prefix( prefix ):
464
    return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
Ian Lynagh's avatar
Ian Lynagh committed
465

466
def _compile_cmd_prefix( name, opts, prefix ):
Ian Lynagh's avatar
Ian Lynagh committed
467 468 469 470
    opts.compile_cmd_prefix = prefix

# ----

471 472 473 474 475 476 477 478
def check_stdout( f ):
    return lambda name, opts, f=f: _check_stdout(name, opts, f)

def _check_stdout( name, opts, f ):
    opts.check_stdout = f

# ----

479
def normalise_slashes( name, opts ):
480 481
    opts.extra_normaliser = normalise_slashes_

482
def normalise_exe( name, opts ):
483 484
    opts.extra_normaliser = normalise_exe_

485
def normalise_fun( fun ):
486
    return lambda name, opts, f=fun: _normalise_fun(name, opts, f)
487

488
def _normalise_fun( name, opts, f ):
489 490
    opts.extra_normaliser = f

491
def normalise_errmsg_fun( fun ):
492
    return lambda name, opts, f=fun: _normalise_errmsg_fun(name, opts, f)
493

494
def _normalise_errmsg_fun( name, opts, f ):
495 496 497 498 499
    opts.extra_errmsg_normaliser = f

def two_normalisers(f, g):
    return lambda x, f=f, g=g: f(g(x))

500 501 502
# ----
# Function for composing two opt-fns together

503 504 505 506 507 508 509
def executeSetups(fs, name, opts):
    if type(fs) is types.ListType:
        # If we have a list of setups, then execute each one
        map (lambda f : executeSetups(f, name, opts), fs)
    else:
        # fs is a single function, so just apply it
        fs(name, opts)
510

511 512 513 514
# -----------------------------------------------------------------------------
# The current directory of tests

def newTestDir( dir ):
515
    global thisdir_settings
516
    # reset the options for this test directory
517 518 519 520 521
    thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )

def _newTestDir( name, opts, dir ):
    opts.testdir = dir
    opts.compiler_always_flags = config.compiler_always_flags
522 523 524 525

# -----------------------------------------------------------------------------
# Actually doing tests

526 527
parallelTests = []
aloneTests = []
528
allTestNames = set([])
529

530
def runTest (opts, name, func, args):
ei@vuokko.info's avatar
ei@vuokko.info committed
531 532
    ok = 0

533
    if config.use_threads:
ei@vuokko.info's avatar
ei@vuokko.info committed
534
        t.thread_pool.acquire()
535
        try:
536
            while config.threads<(t.running_threads+1):
537
                t.thread_pool.wait()
538
            t.running_threads = t.running_threads+1
539 540
            ok=1
            t.thread_pool.release()
541
            thread.start_new_thread(test_common_thread, (name, opts, func, args))
542 543 544 545 546
        except:
            if not ok:
                t.thread_pool.release()
    else:
        test_common_work (name, opts, func, args)
547

548
# name  :: String
549
# setup :: TestOpts -> IO ()
550
def test (name, setup, func, args):
551 552
    global aloneTests
    global parallelTests
553
    global allTestNames
554
    global thisdir_settings
555 556
    if name in allTestNames:
        framework_fail(name, 'duplicate', 'There are multiple tests with this name')
557
    if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
558
        framework_fail(name, 'bad_name', 'This test has an invalid name')
559 560 561 562 563

    # Make a deep copy of the default_testopts, as we need our own copy
    # of any dictionaries etc inside it. Otherwise, if one test modifies
    # them, all tests will see the modified version!
    myTestOpts = copy.deepcopy(default_testopts)
564

565
    executeSetups([thisdir_settings, setup], name, myTestOpts)
566 567 568 569 570 571

    thisTest = lambda : runTest(myTestOpts, name, func, args)
    if myTestOpts.alone:
        aloneTests.append(thisTest)
    else:
        parallelTests.append(thisTest)
572
    allTestNames.add(name)
573

574
if config.use_threads:
575
    def test_common_thread(name, opts, func, args):
576 577 578 579 580 581
        t.lock.acquire()
        try:
            test_common_work(name,opts,func,args)
        finally:
            t.lock.release()
            t.thread_pool.acquire()
582
            t.running_threads = t.running_threads - 1
583 584
            t.thread_pool.notify()
            t.thread_pool.release()
585

586 587 588 589 590 591 592 593 594 595
def get_package_cache_timestamp():
    if config.package_conf_cache_file == '':
        return 0.0
    else:
        try:
            return os.stat(config.package_conf_cache_file).st_mtime
        except:
            return 0.0


ei@vuokko.info's avatar
ei@vuokko.info committed
596
def test_common_work (name, opts, func, args):
597 598 599 600 601 602 603 604 605
    try:
        t.total_tests = t.total_tests+1
        setLocalTestOpts(opts)

        package_conf_cache_file_start_timestamp = get_package_cache_timestamp()

        # All the ways we might run this test
        if func == compile or func == multimod_compile:
            all_ways = config.compile_ways
606
        elif func == compile_and_run or func == multimod_compile_and_run:
607 608 609 610 611 612
            all_ways = config.run_ways
        elif func == ghci_script:
            if 'ghci' in config.run_ways:
                all_ways = ['ghci']
            else:
                all_ways = []
613
        else:
614 615 616 617 618 619 620 621 622 623 624
            all_ways = ['normal']

        # A test itself can request extra ways by setting opts.extra_ways
        all_ways = all_ways + filter(lambda way: way not in all_ways,
                                     opts.extra_ways)

        t.total_test_cases = t.total_test_cases + len(all_ways)

        ok_way = lambda way: \
            not getTestOpts().skip \
            and (config.only == [] or name in config.only) \
625
            and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
626
            and (config.cmdline_ways == [] or way in config.cmdline_ways) \
627
            and (not (config.skip_perf_tests and isStatsTest())) \
628 629 630 631 632 633 634 635 636 637 638 639
            and way not in getTestOpts().omit_ways

        # Which ways we are asked to skip
        do_ways = filter (ok_way,all_ways)

        # In fast mode, we skip all but one way
        if config.fast and len(do_ways) > 0:
            do_ways = [do_ways[0]]

        if not config.clean_only:
            # Run the required tests...
            for way in do_ways:
640 641
                if stopping():
                    break
642 643 644 645 646 647 648
                do_test (name, way, func, args)

            for way in all_ways:
                if way not in do_ways:
                    skiptest (name,way)

        if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
649
            pretest_cleanup(name)
650 651 652 653 654 655 656 657 658 659 660
            clean(map (lambda suff: name + suff,
                      ['', '.exe', '.exe.manifest', '.genscript',
                       '.stderr.normalised',        '.stdout.normalised',
                       '.run.stderr.normalised',    '.run.stdout.normalised',
                       '.comp.stderr.normalised',   '.comp.stdout.normalised',
                       '.interp.stderr.normalised', '.interp.stdout.normalised',
                       '.stats', '.comp.stats',
                       '.hi', '.o', '.prof', '.exe.prof', '.hc',
                       '_stub.h', '_stub.c', '_stub.o',
                       '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))

661
            if func == multi_compile or func == multi_compile_fail:
662 663 664 665 666
                    extra_mods = args[1]
                    clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
                    clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))

            clean(getTestOpts().clean_files)
Ian Lynagh's avatar
Ian Lynagh committed
667

668 669 670 671 672 673 674
            if getTestOpts().outputdir != None:
                odir = in_testdir(getTestOpts().outputdir)
                try:
                    shutil.rmtree(odir)
                except:
                    pass

675 676 677 678 679
            try:
                shutil.rmtree(in_testdir('.hpc.' + name))
            except:
                pass

680 681 682 683 684 685
            try:
                cleanCmd = getTestOpts().clean_cmd
                if cleanCmd != None:
                    result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
                    if result != 0:
                        framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
686
            except:
687
                framework_fail(name, 'cleaning', 'clean-command exception')
688

689
        package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
690

691 692
        if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
            framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
693

694 695 696 697 698 699 700 701 702 703 704 705
        try:
            for f in files_written[name]:
                if os.path.exists(f):
                    try:
                        if not f in files_written_not_removed[name]:
                            files_written_not_removed[name].append(f)
                    except:
                        files_written_not_removed[name] = [f]
        except:
            pass
    except Exception, e:
        framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
Ian Lynagh's avatar
Ian Lynagh committed
706

707 708 709 710
def clean(strs):
    for str in strs:
        for name in glob.glob(in_testdir(str)):
            clean_full_path(name)
711

712
def clean_full_path(name):
Ian Lynagh's avatar
Ian Lynagh committed
713 714 715 716
        try:
            # Remove files...
            os.remove(name)
        except OSError, e1:
717
            try:
Ian Lynagh's avatar
Ian Lynagh committed
718 719 720 721 722 723 724 725 726 727 728
                # ... and empty directories
                os.rmdir(name)
            except OSError, e2:
                # We don't want to fail here, but we do want to know
                # what went wrong, so print out the exceptions.
                # ENOENT isn't a problem, though, as we clean files
                # that don't necessarily exist.
                if e1.errno != errno.ENOENT:
                    print e1
                if e2.errno != errno.ENOENT:
                    print e2
729

730
def do_test(name, way, func, args):
731 732 733
    full_name = name + '(' + way + ')'

    try:
734 735 736 737 738
        if_verbose(2, "=====> %s %d of %d %s " % \
                    (full_name, t.total_tests, len(allTestNames), \
                    [t.n_unexpected_passes, \
                     t.n_unexpected_failures, \
                     t.n_framework_failures]))
739

740 741
        if config.use_threads:
            t.lock.release()
742 743 744 745

        try:
            preCmd = getTestOpts().pre_cmd
            if preCmd != None:
746
                result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
747 748
                if result != 0:
                    framework_fail(name, way, 'pre-command failed: ' + str(result))
749
        except:
750 751
            framework_fail(name, way, 'pre-command exception')

ei@vuokko.info's avatar
ei@vuokko.info committed
752 753 754
        try:
            result = apply(func, [name,way] + args)
        finally:
755 756
            if config.use_threads:
                t.lock.acquire()
757

758 759 760
        if getTestOpts().expect != 'pass' and \
                getTestOpts().expect != 'fail' and \
                getTestOpts().expect != 'missing-lib':
761
            framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
762

763 764 765 766 767 768
        try:
            passFail = result['passFail']
        except:
            passFail = 'No passFail found'

        if passFail == 'pass':
ei@vuokko.info's avatar
ei@vuokko.info committed
769 770
            if getTestOpts().expect == 'pass' \
               and way not in getTestOpts().expect_fail_for:
771
                t.n_expected_passes = t.n_expected_passes + 1
772 773 774 775
                if name in t.expected_passes:
                    t.expected_passes[name].append(way)
                else:
                    t.expected_passes[name] = [way]
776
            else:
777
                if_verbose(1, '*** unexpected pass for %s' % full_name)
778
                t.n_unexpected_passes = t.n_unexpected_passes + 1
779
                addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
780
        elif passFail == 'fail':
ei@vuokko.info's avatar
ei@vuokko.info committed
781 782
            if getTestOpts().expect == 'pass' \
               and way not in getTestOpts().expect_fail_for:
783
                if_verbose(1, '*** unexpected failure for %s' % full_name)
784
                t.n_unexpected_failures = t.n_unexpected_failures + 1
785 786
                reason = result['reason']
                addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
787
            else:
788 789 790 791 792 793
                if getTestOpts().expect == 'missing-lib':
                    t.n_missing_libs = t.n_missing_libs + 1
                    if name in t.missing_libs:
                        t.missing_libs[name].append(way)
                    else:
                        t.missing_libs[name] = [way]
794
                else:
795 796 797 798 799
                    t.n_expected_failures = t.n_expected_failures + 1
                    if name in t.expected_failures:
                        t.expected_failures[name].append(way)
                    else:
                        t.expected_failures[name] = [way]
800 801
        else:
            framework_fail(name, way, 'bad result ' + passFail)
802
    except KeyboardInterrupt:
803
        stopNow()
804
    except:
805
        framework_fail(name, way, 'do_test exception')
806 807
        traceback.print_exc()

808
def addPassingTestInfo (testInfos, directory, name, way):
809 810 811 812 813 814 815 816 817 818
    directory = re.sub('^\\.[/\\\\]', '', directory)

    if not directory in testInfos:
        testInfos[directory] = {}

    if not name in testInfos[directory]:
        testInfos[directory][name] = []

    testInfos[directory][name].append(way)

819 820 821 822 823 824 825 826 827 828 829 830 831 832
def addFailingTestInfo (testInfos, directory, name, reason, way):
    directory = re.sub('^\\.[/\\\\]', '', directory)

    if not directory in testInfos:
        testInfos[directory] = {}

    if not name in testInfos[directory]:
        testInfos[directory][name] = {}

    if not reason in testInfos[directory][name]:
        testInfos[directory][name][reason] = []

    testInfos[directory][name][reason].append(way)

833
def skiptest (name, way):
834 835
    # print 'Skipping test \"', name, '\"'
    t.n_tests_skipped = t.n_tests_skipped + 1
836 837 838 839
    if name in t.tests_skipped:
        t.tests_skipped[name].append(way)
    else:
        t.tests_skipped[name] = [way]
840

841 842
def framework_fail( name, way, reason ):
    full_name = name + '(' + way + ')'
Joachim Breitner's avatar
Joachim Breitner committed
843
    if_verbose(1, '*** framework failure for %s %s ' % (full_name, reason))
844
    t.n_framework_failures = t.n_framework_failures + 1
845 846 847 848
    if name in t.framework_failures:
        t.framework_failures[name].append(way)
    else:
        t.framework_failures[name] = [way]
849

850 851 852 853 854 855 856 857 858 859 860 861 862 863
def badResult(result):
    try:
        if result['passFail'] == 'pass':
            return False
        return True
    except:
        return True

def passed():
    return {'passFail': 'pass'}

def failBecause(reason):
    return {'passFail': 'fail', 'reason': reason}

864 865 866 867
# -----------------------------------------------------------------------------
# Generic command tests

# A generic command test is expected to run and exit successfully.
868 869 870 871 872