testlib.py 72 KB
Newer Older
1
#
2 3 4
# (c) Simon Marlow 2002
#

5 6 7
# This allows us to use the "with X:" syntax with python 2.5:
from __future__ import with_statement

8
import shutil
9 10
import sys
import os
11
import errno
12 13 14
import string
import re
import traceback
15 16
import time
import datetime
17
import copy
18
import glob
19
import types
ian@well-typed.com's avatar
ian@well-typed.com committed
20
from math import ceil, trunc
21

22 23 24 25 26 27 28
have_subprocess = False
try:
    import subprocess
    have_subprocess = True
except:
    print "Warning: subprocess not found, will fall back to spawnv"

29
from string import join
30
from testglobals import *
31 32
from testutil import *

33 34 35
if config.use_threads:
    import threading
    import thread
36

37 38 39 40 41 42 43 44
global wantToStop
wantToStop = False
def stopNow():
    global wantToStop
    wantToStop = True
def stopping():
    return wantToStop

45 46
# Options valid for the current test only (these get reset to
# testdir_testopts after each test).
47

ei@vuokko.info's avatar
ei@vuokko.info committed
48
global testopts_local
49 50 51 52 53 54
if config.use_threads:
    testopts_local = threading.local()
else:
    class TestOpts_Local:
        pass
    testopts_local = TestOpts_Local()
55 56

def getTestOpts():
ei@vuokko.info's avatar
ei@vuokko.info committed
57
    return testopts_local.x
58

ei@vuokko.info's avatar
ei@vuokko.info committed
59 60 61
def setLocalTestOpts(opts):
    global testopts_local
    testopts_local.x=opts
62 63 64 65

# This can be called at the top of a file of tests, to set default test options
# for the following tests.
def setTestOpts( f ):
66
    global thisdir_settings
67
    thisdir_settings = [thisdir_settings, f]
68 69 70 71 72 73 74 75 76 77 78 79

# -----------------------------------------------------------------------------
# Canned setup functions for common cases.  eg. for a test you might say
#
#      test('test001', normal, compile, [''])
#
# to run it without any options, but change it to
#
#      test('test001', expect_fail, compile, [''])
#
# to expect failure for this test.

80
def normal( name, opts ):
81 82
    return;

83
def skip( name, opts ):
84 85
    opts.skip = 1

86
def expect_fail( name, opts ):
87 88
    opts.expect = 'fail';

89
def reqlib( lib ):
90
    return lambda name, opts, l=lib: _reqlib (name, opts, l )
91

92 93 94 95
# Cache the results of looking to see if we have a library or not.
# This makes quite a difference, especially on Windows.
have_lib = {}

96
def _reqlib( name, opts, lib ):
97 98
    if have_lib.has_key(lib):
        got_it = have_lib[lib]
99
    else:
100 101 102
        if have_subprocess:
            # By preference we use subprocess, as the alternative uses
            # /dev/null which mingw doesn't have.
103
            p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
104 105 106 107 108 109 110 111 112 113 114 115 116
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
            # read from stdout and stderr to avoid blocking due to
            # buffers filling
            p.communicate()
            r = p.wait()
        else:
            r = os.system(config.ghc_pkg + ' describe ' + lib
                                         + ' > /dev/null 2> /dev/null')
        got_it = r == 0
        have_lib[lib] = got_it

    if not got_it:
117
        opts.expect = 'missing-lib'
118

119
def req_profiling( name, opts ):
120 121 122
    if not config.have_profiling:
        opts.expect = 'fail'

123
def req_shared_libs( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
124 125 126
    if not config.have_shared_libs:
        opts.expect = 'fail'

127
def req_interp( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
128 129 130
    if not config.have_interp:
        opts.expect = 'fail'

131
def req_smp( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
132 133 134
    if not config.have_smp:
        opts.expect = 'fail'

135
def ignore_output( name, opts ):
136 137
    opts.ignore_output = 1

138
def no_stdin( name, opts ):
139 140
    opts.no_stdin = 1

141
def combined_output( name, opts ):
pcapriotti's avatar
pcapriotti committed
142 143
    opts.combined_output = True

144 145 146
# -----

def expect_fail_for( ways ):
147
    return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
148

149
def _expect_fail_for( name, opts, ways ):
150 151
    opts.expect_fail_for = ways

152 153 154 155
def expect_broken( bug ):
    return lambda name, opts, b=bug: _expect_broken (name, opts, b )

def _expect_broken( name, opts, bug ):
156
    record_broken(name, opts, bug)
157 158
    opts.expect = 'fail';

Ian Lynagh's avatar
Ian Lynagh committed
159
def expect_broken_for( bug, ways ):
160
    return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
Ian Lynagh's avatar
Ian Lynagh committed
161

162
def _expect_broken_for( name, opts, bug, ways ):
163
    record_broken(name, opts, bug)
Ian Lynagh's avatar
Ian Lynagh committed
164
    opts.expect_fail_for = ways
Ian Lynagh's avatar
Ian Lynagh committed
165

166
def record_broken(name, opts, bug):
167
    global brokens
168
    me = (bug, opts.testdir, name)
169 170 171
    if not me in brokens:
        brokens.append(me)

172 173 174
# -----

def omit_ways( ways ):
175
    return lambda name, opts, w=ways: _omit_ways( name, opts, w )
176

177
def _omit_ways( name, opts, ways ):
178 179 180 181
    opts.omit_ways = ways

# -----

182
def only_ways( ways ):
183
    return lambda name, opts, w=ways: _only_ways( name, opts, w )
184

185
def _only_ways( name, opts, ways ):
186 187 188 189
    opts.only_ways = ways

# -----

190
def extra_ways( ways ):
191
    return lambda name, opts, w=ways: _extra_ways( name, opts, w )
192

193
def _extra_ways( name, opts, ways ):
194 195 196 197
    opts.extra_ways = ways

# -----

ross's avatar
ross committed
198
def omit_compiler_types( compiler_types ):
199
   return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
ross's avatar
ross committed
200

201
def _omit_compiler_types( name, opts, compiler_types ):
ross's avatar
ross committed
202
    if config.compiler_type in compiler_types:
dterei's avatar
dterei committed
203
        opts.skip = 1
ross's avatar
ross committed
204 205 206 207

# -----

def only_compiler_types( compiler_types ):
208
   return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
ross's avatar
ross committed
209

210
def _only_compiler_types( name, opts, compiler_types ):
ross's avatar
ross committed
211
    if config.compiler_type not in compiler_types:
dterei's avatar
dterei committed
212
        opts.skip = 1
ross's avatar
ross committed
213 214 215

# -----

216
def set_stdin( file ):
217
   return lambda name, opts, f=file: _set_stdin(name, opts, f);
218

219
def _set_stdin( name, opts, f ):
220 221 222 223 224
   opts.stdin = f

# -----

def exit_code( val ):
225
    return lambda name, opts, v=val: _exit_code(name, opts, v);
226

227
def _exit_code( name, opts, v ):
228 229 230 231
    opts.exit_code = v

# -----

232
def timeout_multiplier( val ):
233
    return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
234

235
def _timeout_multiplier( name, opts, v ):
236 237 238 239
    opts.timeout_multiplier = v

# -----

240
def extra_run_opts( val ):
241
    return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
242

243
def _extra_run_opts( name, opts, v ):
244 245
    opts.extra_run_opts = v

246 247
# -----

Simon Marlow's avatar
Simon Marlow committed
248
def extra_hc_opts( val ):
249
    return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
Simon Marlow's avatar
Simon Marlow committed
250

251
def _extra_hc_opts( name, opts, v ):
Simon Marlow's avatar
Simon Marlow committed
252 253 254 255
    opts.extra_hc_opts = v

# -----

256
def extra_clean( files ):
257
    return lambda name, opts, v=files: _extra_clean(name, opts, v);
258

259
def _extra_clean( name, opts, v ):
260 261
    opts.clean_files = v

262 263
# -----

264 265 266 267 268 269 270
def stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);

def _stats_num_field( name, opts, field, expecteds ):
    if field in opts.stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

271 272 273 274 275 276
    if type(expecteds) is types.ListType:
        for (b, expected, dev) in expecteds:
            if b:
                opts.stats_range_fields[field] = (expected, dev)
                return
        framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
277

278 279 280
    else:
        (expected, dev) = expecteds
        opts.stats_range_fields[field] = (expected, dev)
281 282 283

def compiler_stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
284

285 286 287 288 289 290 291 292
def _compiler_stats_num_field( name, opts, field, expecteds ):
    if field in opts.compiler_stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

    for (b, expected, dev) in expecteds:
        if b:
            opts.compiler_stats_range_fields[field] = (expected, dev)
            return
293

294 295
    framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')

296 297
# -----

298
def when(b, f):
ian@well-typed.com's avatar
ian@well-typed.com committed
299 300 301
    # When list_brokens is on, we want to see all expect_broken calls,
    # so we always do f
    if b or config.list_broken:
302 303 304 305 306 307 308
        return f
    else:
        return normal

def unless(b, f):
    return when(not b, f)

ian@well-typed.com's avatar
ian@well-typed.com committed
309 310 311
def doing_ghci():
    return 'ghci' in config.run_ways

312
def ghci_dynamic( ):
ian@well-typed.com's avatar
ian@well-typed.com committed
313
    return config.ghc_dynamic
314 315 316 317

def fast():
    return config.fast

318 319
def platform( plat ):
    return config.platform == plat
Ian Lynagh's avatar
Ian Lynagh committed
320

321 322
def opsys( os ):
    return config.os == os
Ian Lynagh's avatar
Ian Lynagh committed
323

324 325
def arch( arch ):
    return config.arch == arch
326

327 328
def wordsize( ws ):
    return config.wordsize == str(ws)
tibbe's avatar
tibbe committed
329

330 331
def msys( ):
    return config.msys
ian@well-typed.com's avatar
ian@well-typed.com committed
332

333 334
def cygwin( ):
    return config.cygwin
Ian Lynagh's avatar
Ian Lynagh committed
335

336 337
def have_vanilla( ):
    return config.have_vanilla
338

339 340
def have_dynamic( ):
    return config.have_dynamic
341

342 343
def have_profiling( ):
    return config.have_profiling
344

345 346
def in_tree_compiler( ):
    return config.in_tree_compiler
347

348 349
def compiler_type( compiler ):
    return config.compiler_type == compiler
350

351 352 353
def compiler_lt( compiler, version ):
    return config.compiler_type == compiler and \
           version_lt(config.compiler_version, version)
354

355 356 357
def compiler_le( compiler, version ):
    return config.compiler_type == compiler and \
           version_le(config.compiler_version, version)
358

359 360 361
def compiler_gt( compiler, version ):
    return config.compiler_type == compiler and \
           version_gt(config.compiler_version, version)
362

363 364 365
def compiler_ge( compiler, version ):
    return config.compiler_type == compiler and \
           version_ge(config.compiler_version, version)
366

367 368 369 370 371 372
def unregisterised( ):
    return config.unregisterised

def compiler_profiled( ):
    return config.compiler_profiled

373 374
def compiler_debugged( ):
    return config.compiler_debugged
375

376 377 378 379 380
def tag( t ):
    return t in config.compiler_tags

# ---

Ian Lynagh's avatar
Ian Lynagh committed
381 382
def namebase( nb ):
   return lambda opts, nb=nb: _namebase(opts, nb)
383

Ian Lynagh's avatar
Ian Lynagh committed
384 385
def _namebase( opts, nb ):
    opts.with_namebase = nb
386

387 388
# ---

389
def high_memory_usage(name, opts):
390 391
    opts.alone = True

392 393 394 395 396
# If a test is for a multi-CPU race, then running the test alone
# increases the chance that we'll actually see it.
def multi_cpu_race(name, opts):
    opts.alone = True

Ian Lynagh's avatar
Ian Lynagh committed
397
# ---
398
def literate( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
399 400
    opts.literate = 1;

401
def c_src( name, opts ):
402 403
    opts.c_src = 1;

404
def objc_src( name, opts ):
Austin Seipp's avatar
Austin Seipp committed
405 406
    opts.objc_src = 1;

407
def objcpp_src( name, opts ):
408 409
    opts.objcpp_src = 1;

410
def cmm_src( name, opts ):
411 412
    opts.cmm_src = 1;

413
def outputdir( odir ):
414
    return lambda name, opts, d=odir: _outputdir(name, opts, d)
415

416
def _outputdir( name, opts, odir ):
417 418
    opts.outputdir = odir;

419 420
# ----

421
def pre_cmd( cmd ):
422
    return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
423

424
def _pre_cmd( name, opts, cmd ):
425 426 427 428
    opts.pre_cmd = cmd

# ----

429
def clean_cmd( cmd ):
430
    return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
431

432
def _clean_cmd( name, opts, cmd ):
433 434 435 436
    opts.clean_cmd = cmd

# ----

437
def cmd_prefix( prefix ):
438
    return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
439

440
def _cmd_prefix( name, opts, prefix ):
441 442 443 444 445
    opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;

# ----

def cmd_wrapper( fun ):
446
    return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
447

448
def _cmd_wrapper( name, opts, fun ):
449
    opts.cmd_wrapper = fun
450

451 452
# ----

Ian Lynagh's avatar
Ian Lynagh committed
453
def compile_cmd_prefix( prefix ):
454
    return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
Ian Lynagh's avatar
Ian Lynagh committed
455

456
def _compile_cmd_prefix( name, opts, prefix ):
Ian Lynagh's avatar
Ian Lynagh committed
457 458 459 460
    opts.compile_cmd_prefix = prefix

# ----

461
def normalise_slashes( name, opts ):
462 463
    opts.extra_normaliser = normalise_slashes_

464
def normalise_exe( name, opts ):
465 466
    opts.extra_normaliser = normalise_exe_

467
def normalise_fun( fun ):
468
    return lambda name, opts, f=fun: _normalise_fun(name, opts, f)
469

470
def _normalise_fun( name, opts, f ):
471 472
    opts.extra_normaliser = f

473
def normalise_errmsg_fun( fun ):
474
    return lambda name, opts, f=fun: _normalise_errmsg_fun(name, opts, f)
475

476
def _normalise_errmsg_fun( name, opts, f ):
477 478 479 480 481
    opts.extra_errmsg_normaliser = f

def two_normalisers(f, g):
    return lambda x, f=f, g=g: f(g(x))

482 483 484
# ----
# Function for composing two opt-fns together

485 486 487 488 489 490 491
def executeSetups(fs, name, opts):
    if type(fs) is types.ListType:
        # If we have a list of setups, then execute each one
        map (lambda f : executeSetups(f, name, opts), fs)
    else:
        # fs is a single function, so just apply it
        fs(name, opts)
492

493 494 495 496
# -----------------------------------------------------------------------------
# The current directory of tests

def newTestDir( dir ):
497
    global thisdir_settings
498
    # reset the options for this test directory
499 500 501 502 503
    thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )

def _newTestDir( name, opts, dir ):
    opts.testdir = dir
    opts.compiler_always_flags = config.compiler_always_flags
504 505 506 507

# -----------------------------------------------------------------------------
# Actually doing tests

508 509
parallelTests = []
aloneTests = []
510
allTestNames = set([])
511

512
def runTest (opts, name, func, args):
ei@vuokko.info's avatar
ei@vuokko.info committed
513 514
    ok = 0

515
    if config.use_threads:
ei@vuokko.info's avatar
ei@vuokko.info committed
516
        t.thread_pool.acquire()
517
        try:
518
            while config.threads<(t.running_threads+1):
519
                t.thread_pool.wait()
520
            t.running_threads = t.running_threads+1
521 522
            ok=1
            t.thread_pool.release()
523
            thread.start_new_thread(test_common_thread, (name, opts, func, args))
524 525 526 527 528
        except:
            if not ok:
                t.thread_pool.release()
    else:
        test_common_work (name, opts, func, args)
529

530
# name  :: String
531
# setup :: TestOpts -> IO ()
532
def test (name, setup, func, args):
533 534
    global aloneTests
    global parallelTests
535
    global allTestNames
536
    global thisdir_settings
537 538
    if name in allTestNames:
        framework_fail(name, 'duplicate', 'There are multiple tests with this name')
539
    if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
540
        framework_fail(name, 'bad_name', 'This test has an invalid name')
541 542 543 544 545

    # Make a deep copy of the default_testopts, as we need our own copy
    # of any dictionaries etc inside it. Otherwise, if one test modifies
    # them, all tests will see the modified version!
    myTestOpts = copy.deepcopy(default_testopts)
546

547
    executeSetups([thisdir_settings, setup], name, myTestOpts)
548 549 550 551 552 553

    thisTest = lambda : runTest(myTestOpts, name, func, args)
    if myTestOpts.alone:
        aloneTests.append(thisTest)
    else:
        parallelTests.append(thisTest)
554
    allTestNames.add(name)
555

556
if config.use_threads:
557
    def test_common_thread(name, opts, func, args):
558 559 560 561 562 563
        t.lock.acquire()
        try:
            test_common_work(name,opts,func,args)
        finally:
            t.lock.release()
            t.thread_pool.acquire()
564
            t.running_threads = t.running_threads - 1
565 566
            t.thread_pool.notify()
            t.thread_pool.release()
567

568 569 570 571 572 573 574 575 576 577
def get_package_cache_timestamp():
    if config.package_conf_cache_file == '':
        return 0.0
    else:
        try:
            return os.stat(config.package_conf_cache_file).st_mtime
        except:
            return 0.0


ei@vuokko.info's avatar
ei@vuokko.info committed
578
def test_common_work (name, opts, func, args):
579 580 581 582 583 584 585 586 587
    try:
        t.total_tests = t.total_tests+1
        setLocalTestOpts(opts)

        package_conf_cache_file_start_timestamp = get_package_cache_timestamp()

        # All the ways we might run this test
        if func == compile or func == multimod_compile:
            all_ways = config.compile_ways
588
        elif func == compile_and_run or func == multimod_compile_and_run:
589 590 591 592 593 594
            all_ways = config.run_ways
        elif func == ghci_script:
            if 'ghci' in config.run_ways:
                all_ways = ['ghci']
            else:
                all_ways = []
595
        else:
596 597 598 599 600 601 602 603 604 605 606
            all_ways = ['normal']

        # A test itself can request extra ways by setting opts.extra_ways
        all_ways = all_ways + filter(lambda way: way not in all_ways,
                                     opts.extra_ways)

        t.total_test_cases = t.total_test_cases + len(all_ways)

        ok_way = lambda way: \
            not getTestOpts().skip \
            and (config.only == [] or name in config.only) \
607
            and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
608 609 610 611 612 613 614 615 616 617 618 619 620
            and (config.cmdline_ways == [] or way in config.cmdline_ways) \
            and way not in getTestOpts().omit_ways

        # Which ways we are asked to skip
        do_ways = filter (ok_way,all_ways)

        # In fast mode, we skip all but one way
        if config.fast and len(do_ways) > 0:
            do_ways = [do_ways[0]]

        if not config.clean_only:
            # Run the required tests...
            for way in do_ways:
621 622
                if stopping():
                    break
623 624 625 626 627 628 629
                do_test (name, way, func, args)

            for way in all_ways:
                if way not in do_ways:
                    skiptest (name,way)

        if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
630
            pretest_cleanup(name)
631 632 633 634 635 636 637 638 639 640 641
            clean(map (lambda suff: name + suff,
                      ['', '.exe', '.exe.manifest', '.genscript',
                       '.stderr.normalised',        '.stdout.normalised',
                       '.run.stderr.normalised',    '.run.stdout.normalised',
                       '.comp.stderr.normalised',   '.comp.stdout.normalised',
                       '.interp.stderr.normalised', '.interp.stdout.normalised',
                       '.stats', '.comp.stats',
                       '.hi', '.o', '.prof', '.exe.prof', '.hc',
                       '_stub.h', '_stub.c', '_stub.o',
                       '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))

642
            if func == multi_compile or func == multi_compile_fail:
643 644 645 646 647
                    extra_mods = args[1]
                    clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
                    clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))

            clean(getTestOpts().clean_files)
Ian Lynagh's avatar
Ian Lynagh committed
648

649 650 651 652 653 654 655
            if getTestOpts().outputdir != None:
                odir = in_testdir(getTestOpts().outputdir)
                try:
                    shutil.rmtree(odir)
                except:
                    pass

656 657 658 659 660
            try:
                shutil.rmtree(in_testdir('.hpc.' + name))
            except:
                pass

661 662 663 664 665 666 667 668
            try:
                cleanCmd = getTestOpts().clean_cmd
                if cleanCmd != None:
                    result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
                    if result != 0:
                        framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
            except e:
                framework_fail(name, 'cleaning', 'clean-command exception')
669

670
        package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
671

672 673
        if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
            framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
674

675 676 677 678 679 680 681 682 683 684 685 686
        try:
            for f in files_written[name]:
                if os.path.exists(f):
                    try:
                        if not f in files_written_not_removed[name]:
                            files_written_not_removed[name].append(f)
                    except:
                        files_written_not_removed[name] = [f]
        except:
            pass
    except Exception, e:
        framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
Ian Lynagh's avatar
Ian Lynagh committed
687

688 689 690 691
def clean(strs):
    for str in strs:
        for name in glob.glob(in_testdir(str)):
            clean_full_path(name)
692

693
def clean_full_path(name):
Ian Lynagh's avatar
Ian Lynagh committed
694 695 696 697
        try:
            # Remove files...
            os.remove(name)
        except OSError, e1:
698
            try:
Ian Lynagh's avatar
Ian Lynagh committed
699 700 701 702 703 704 705 706 707 708 709
                # ... and empty directories
                os.rmdir(name)
            except OSError, e2:
                # We don't want to fail here, but we do want to know
                # what went wrong, so print out the exceptions.
                # ENOENT isn't a problem, though, as we clean files
                # that don't necessarily exist.
                if e1.errno != errno.ENOENT:
                    print e1
                if e2.errno != errno.ENOENT:
                    print e2
710

711
def do_test(name, way, func, args):
712 713 714
    full_name = name + '(' + way + ')'

    try:
715 716 717 718 719
        if_verbose(2, "=====> %s %d of %d %s " % \
                    (full_name, t.total_tests, len(allTestNames), \
                    [t.n_unexpected_passes, \
                     t.n_unexpected_failures, \
                     t.n_framework_failures]))
720

721 722
        if config.use_threads:
            t.lock.release()
723 724 725 726

        try:
            preCmd = getTestOpts().pre_cmd
            if preCmd != None:
727
                result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
728 729 730 731 732
                if result != 0:
                    framework_fail(name, way, 'pre-command failed: ' + str(result))
        except e:
            framework_fail(name, way, 'pre-command exception')

ei@vuokko.info's avatar
ei@vuokko.info committed
733 734 735
        try:
            result = apply(func, [name,way] + args)
        finally:
736 737
            if config.use_threads:
                t.lock.acquire()
738

739 740 741
        if getTestOpts().expect != 'pass' and \
                getTestOpts().expect != 'fail' and \
                getTestOpts().expect != 'missing-lib':
742
            framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
743

744 745 746 747 748 749
        try:
            passFail = result['passFail']
        except:
            passFail = 'No passFail found'

        if passFail == 'pass':
ei@vuokko.info's avatar
ei@vuokko.info committed
750 751
            if getTestOpts().expect == 'pass' \
               and way not in getTestOpts().expect_fail_for:
752
                t.n_expected_passes = t.n_expected_passes + 1
753 754 755 756
                if name in t.expected_passes:
                    t.expected_passes[name].append(way)
                else:
                    t.expected_passes[name] = [way]
757
            else:
758
                if_verbose(1, '*** unexpected pass for %s' % full_name)
759
                t.n_unexpected_passes = t.n_unexpected_passes + 1
760
                addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
761
        elif passFail == 'fail':
ei@vuokko.info's avatar
ei@vuokko.info committed
762 763
            if getTestOpts().expect == 'pass' \
               and way not in getTestOpts().expect_fail_for:
764
                if_verbose(1, '*** unexpected failure for %s' % full_name)
765
                t.n_unexpected_failures = t.n_unexpected_failures + 1
766 767
                reason = result['reason']
                addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
768
            else:
769 770 771 772 773 774
                if getTestOpts().expect == 'missing-lib':
                    t.n_missing_libs = t.n_missing_libs + 1
                    if name in t.missing_libs:
                        t.missing_libs[name].append(way)
                    else:
                        t.missing_libs[name] = [way]
775
                else:
776 777 778 779 780
                    t.n_expected_failures = t.n_expected_failures + 1
                    if name in t.expected_failures:
                        t.expected_failures[name].append(way)
                    else:
                        t.expected_failures[name] = [way]
781 782
        else:
            framework_fail(name, way, 'bad result ' + passFail)
783
    except KeyboardInterrupt:
784
        stopNow()
785
    except:
786
        framework_fail(name, way, 'do_test exception')
787 788
        traceback.print_exc()

789
def addPassingTestInfo (testInfos, directory, name, way):
790 791 792 793 794 795 796 797 798 799
    directory = re.sub('^\\.[/\\\\]', '', directory)

    if not directory in testInfos:
        testInfos[directory] = {}

    if not name in testInfos[directory]:
        testInfos[directory][name] = []

    testInfos[directory][name].append(way)

800 801 802 803 804 805 806 807 808 809 810 811 812 813
def addFailingTestInfo (testInfos, directory, name, reason, way):
    directory = re.sub('^\\.[/\\\\]', '', directory)

    if not directory in testInfos:
        testInfos[directory] = {}

    if not name in testInfos[directory]:
        testInfos[directory][name] = {}

    if not reason in testInfos[directory][name]:
        testInfos[directory][name][reason] = []

    testInfos[directory][name][reason].append(way)

814
def skiptest (name, way):
815 816
    # print 'Skipping test \"', name, '\"'
    t.n_tests_skipped = t.n_tests_skipped + 1
817 818 819 820
    if name in t.tests_skipped:
        t.tests_skipped[name].append(way)
    else:
        t.tests_skipped[name] = [way]
821

822 823
def framework_fail( name, way, reason ):
    full_name = name + '(' + way + ')'
Joachim Breitner's avatar
Joachim Breitner committed
824
    if_verbose(1, '*** framework failure for %s %s ' % (full_name, reason))
825
    t.n_framework_failures = t.n_framework_failures + 1
826 827 828 829
    if name in t.framework_failures:
        t.framework_failures[name].append(way)
    else:
        t.framework_failures[name] = [way]
830

831 832 833 834 835 836 837 838 839 840 841 842 843 844
def badResult(result):
    try:
        if result['passFail'] == 'pass':
            return False
        return True
    except:
        return True

def passed():
    return {'passFail': 'pass'}

def failBecause(reason):
    return {'passFail': 'fail', 'reason': reason}

845 846 847 848
# -----------------------------------------------------------------------------
# Generic command tests

# A generic command test is expected to run and exit successfully.
849 850 851 852 853 854
#
# The expected exit code can be changed via exit_code() as normal, and
# the expected stdout/stderr are stored in <testname>.stdout and
# <testname>.stderr.  The output of the command can be ignored
# altogether by using run_command_ignore_output instead of
# run_command.
855

856
def run_command( name, way, cmd ):
857
    return simple_run( name, '', cmd, '' )
858

859 860 861 862
# -----------------------------------------------------------------------------
# GHCi tests

def ghci_script( name, way, script ):
863
    # filter out -fforce-recomp from compiler_always_flags, because we're
864
    # actually testing the recompilation behaviour in the GHCi tests.
865
    flags = filter(lambda f: f != '-fforce-recomp', getTestOpts().compiler_always_flags)
866
    flags.append(getTestOpts().extra_hc_opts)
867 868
    if getTestOpts().outputdir != None:
        flags.extend(["-outputdir", getTestOpts().outputdir])
869 870 871 872 873 874

    # We pass HC and HC_OPTS as environment variables, so that the
    # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
    cmd = "HC='" + config.compiler + "' " + \
          "HC_OPTS='" + join(flags,' ') + "' " + \
          "'" + config.compiler + "'" + \
</