testlib.py 71.5 KB
Newer Older
1
#
2 3 4
# (c) Simon Marlow 2002
#

5 6 7
# This allows us to use the "with X:" syntax with python 2.5:
from __future__ import with_statement

8
import shutil
9 10
import sys
import os
11
import errno
12 13 14 15
import string
import re
import traceback
import copy
16
import glob
17
import types
ian@well-typed.com's avatar
ian@well-typed.com committed
18
from math import ceil, trunc
19

20 21 22 23 24 25 26
have_subprocess = False
try:
    import subprocess
    have_subprocess = True
except:
    print "Warning: subprocess not found, will fall back to spawnv"

27
from string import join
28
from testglobals import *
29 30
from testutil import *

31 32 33
if config.use_threads:
    import threading
    import thread
34

35 36 37 38 39 40 41 42
global wantToStop
wantToStop = False
def stopNow():
    global wantToStop
    wantToStop = True
def stopping():
    return wantToStop

43 44
# Options valid for the current test only (these get reset to
# testdir_testopts after each test).
45

ei@vuokko.info's avatar
ei@vuokko.info committed
46
global testopts_local
47 48 49 50 51 52
if config.use_threads:
    testopts_local = threading.local()
else:
    class TestOpts_Local:
        pass
    testopts_local = TestOpts_Local()
53 54

def getTestOpts():
ei@vuokko.info's avatar
ei@vuokko.info committed
55
    return testopts_local.x
56

ei@vuokko.info's avatar
ei@vuokko.info committed
57 58 59
def setLocalTestOpts(opts):
    global testopts_local
    testopts_local.x=opts
60 61 62 63

# This can be called at the top of a file of tests, to set default test options
# for the following tests.
def setTestOpts( f ):
64 65
    global thisdir_settings
    thisdir_settings = compose(thisdir_settings, f)
66 67 68 69 70 71 72 73 74 75 76 77

# -----------------------------------------------------------------------------
# Canned setup functions for common cases.  eg. for a test you might say
#
#      test('test001', normal, compile, [''])
#
# to run it without any options, but change it to
#
#      test('test001', expect_fail, compile, [''])
#
# to expect failure for this test.

78
def normal( name, opts ):
79 80
    return;

81
def skip( name, opts ):
82 83
    opts.skip = 1

84
def expect_fail( name, opts ):
85 86
    opts.expect = 'fail';

87
def reqlib( lib ):
88
    return lambda name, opts, l=lib: _reqlib (name, opts, l )
89

90 91 92 93
# Cache the results of looking to see if we have a library or not.
# This makes quite a difference, especially on Windows.
have_lib = {}

94
def _reqlib( name, opts, lib ):
95 96
    if have_lib.has_key(lib):
        got_it = have_lib[lib]
97
    else:
98 99 100
        if have_subprocess:
            # By preference we use subprocess, as the alternative uses
            # /dev/null which mingw doesn't have.
101
            p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
102 103 104 105 106 107 108 109 110 111 112 113 114
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
            # read from stdout and stderr to avoid blocking due to
            # buffers filling
            p.communicate()
            r = p.wait()
        else:
            r = os.system(config.ghc_pkg + ' describe ' + lib
                                         + ' > /dev/null 2> /dev/null')
        got_it = r == 0
        have_lib[lib] = got_it

    if not got_it:
115
        opts.expect = 'missing-lib'
116

117
def req_profiling( name, opts ):
118 119 120
    if not config.have_profiling:
        opts.expect = 'fail'

121
def req_shared_libs( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
122 123 124
    if not config.have_shared_libs:
        opts.expect = 'fail'

125
def req_interp( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
126 127 128
    if not config.have_interp:
        opts.expect = 'fail'

129
def req_smp( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
130 131 132
    if not config.have_smp:
        opts.expect = 'fail'

133
def ignore_output( name, opts ):
134 135
    opts.ignore_output = 1

136
def no_stdin( name, opts ):
137 138
    opts.no_stdin = 1

139
def combined_output( name, opts ):
pcapriotti's avatar
pcapriotti committed
140 141
    opts.combined_output = True

142 143 144
# -----

def expect_fail_for( ways ):
145
    return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
146

147
def _expect_fail_for( name, opts, ways ):
148 149
    opts.expect_fail_for = ways

150 151 152 153
def expect_broken( bug ):
    return lambda name, opts, b=bug: _expect_broken (name, opts, b )

def _expect_broken( name, opts, bug ):
154
    record_broken(name, opts, bug)
155 156
    opts.expect = 'fail';

Ian Lynagh's avatar
Ian Lynagh committed
157
def expect_broken_for( bug, ways ):
158
    return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
Ian Lynagh's avatar
Ian Lynagh committed
159

160
def _expect_broken_for( name, opts, bug, ways ):
161
    record_broken(name, opts, bug)
Ian Lynagh's avatar
Ian Lynagh committed
162
    opts.expect_fail_for = ways
Ian Lynagh's avatar
Ian Lynagh committed
163

164
def record_broken(name, opts, bug):
165
    global brokens
166
    me = (bug, opts.testdir, name)
167 168 169
    if not me in brokens:
        brokens.append(me)

170 171 172
# -----

def omit_ways( ways ):
173
    return lambda name, opts, w=ways: _omit_ways( name, opts, w )
174

175
def _omit_ways( name, opts, ways ):
176 177 178 179
    opts.omit_ways = ways

# -----

180
def only_ways( ways ):
181
    return lambda name, opts, w=ways: _only_ways( name, opts, w )
182

183
def _only_ways( name, opts, ways ):
184 185 186 187
    opts.only_ways = ways

# -----

188
def extra_ways( ways ):
189
    return lambda name, opts, w=ways: _extra_ways( name, opts, w )
190

191
def _extra_ways( name, opts, ways ):
192 193 194 195
    opts.extra_ways = ways

# -----

ross's avatar
ross committed
196
def omit_compiler_types( compiler_types ):
197
   return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
ross's avatar
ross committed
198

199
def _omit_compiler_types( name, opts, compiler_types ):
ross's avatar
ross committed
200
    if config.compiler_type in compiler_types:
dterei's avatar
dterei committed
201
        opts.skip = 1
ross's avatar
ross committed
202 203 204 205

# -----

def only_compiler_types( compiler_types ):
206
   return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
ross's avatar
ross committed
207

208
def _only_compiler_types( name, opts, compiler_types ):
ross's avatar
ross committed
209
    if config.compiler_type not in compiler_types:
dterei's avatar
dterei committed
210
        opts.skip = 1
ross's avatar
ross committed
211 212 213

# -----

214
def set_stdin( file ):
215
   return lambda name, opts, f=file: _set_stdin(name, opts, f);
216

217
def _set_stdin( name, opts, f ):
218 219 220 221 222
   opts.stdin = f

# -----

def exit_code( val ):
223
    return lambda name, opts, v=val: _exit_code(name, opts, v);
224

225
def _exit_code( name, opts, v ):
226 227 228 229
    opts.exit_code = v

# -----

230
def timeout_multiplier( val ):
231
    return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
232

233
def _timeout_multiplier( name, opts, v ):
234 235 236 237
    opts.timeout_multiplier = v

# -----

238
def extra_run_opts( val ):
239
    return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
240

241
def _extra_run_opts( name, opts, v ):
242 243
    opts.extra_run_opts = v

244 245
# -----

Simon Marlow's avatar
Simon Marlow committed
246
def extra_hc_opts( val ):
247
    return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
Simon Marlow's avatar
Simon Marlow committed
248

249
def _extra_hc_opts( name, opts, v ):
Simon Marlow's avatar
Simon Marlow committed
250 251 252 253
    opts.extra_hc_opts = v

# -----

254
def extra_clean( files ):
255
    return lambda name, opts, v=files: _extra_clean(name, opts, v);
256

257
def _extra_clean( name, opts, v ):
258 259
    opts.clean_files = v

260 261
# -----

262 263 264 265 266 267 268
def stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);

def _stats_num_field( name, opts, field, expecteds ):
    if field in opts.stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

269 270 271 272 273 274
    if type(expecteds) is types.ListType:
        for (b, expected, dev) in expecteds:
            if b:
                opts.stats_range_fields[field] = (expected, dev)
                return
        framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
275

276 277 278
    else:
        (expected, dev) = expecteds
        opts.stats_range_fields[field] = (expected, dev)
279 280 281

def compiler_stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
282

283 284 285 286 287 288 289 290
def _compiler_stats_num_field( name, opts, field, expecteds ):
    if field in opts.compiler_stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

    for (b, expected, dev) in expecteds:
        if b:
            opts.compiler_stats_range_fields[field] = (expected, dev)
            return
291

292 293
    framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')

294 295
# -----

296
def when(b, f):
ian@well-typed.com's avatar
ian@well-typed.com committed
297 298 299
    # When list_brokens is on, we want to see all expect_broken calls,
    # so we always do f
    if b or config.list_broken:
300 301 302 303 304 305 306
        return f
    else:
        return normal

def unless(b, f):
    return when(not b, f)

ian@well-typed.com's avatar
ian@well-typed.com committed
307 308 309
def doing_ghci():
    return 'ghci' in config.run_ways

310 311 312 313 314 315
def ghci_dynamic( ):
    return config.ghc_dynamic_by_default

def fast():
    return config.fast

316 317
def platform( plat ):
    return config.platform == plat
Ian Lynagh's avatar
Ian Lynagh committed
318

319 320
def opsys( os ):
    return config.os == os
Ian Lynagh's avatar
Ian Lynagh committed
321

322 323
def arch( arch ):
    return config.arch == arch
324

325 326
def wordsize( ws ):
    return config.wordsize == str(ws)
tibbe's avatar
tibbe committed
327

328 329
def msys( ):
    return config.msys
ian@well-typed.com's avatar
ian@well-typed.com committed
330

331 332
def cygwin( ):
    return config.cygwin
Ian Lynagh's avatar
Ian Lynagh committed
333

334 335
def have_vanilla( ):
    return config.have_vanilla
336

337 338
def have_dynamic( ):
    return config.have_dynamic
339

340 341
def have_profiling( ):
    return config.have_profiling
342

343 344
def in_tree_compiler( ):
    return config.in_tree_compiler
345

346 347
def compiler_type( compiler ):
    return config.compiler_type == compiler
348

349 350 351
def compiler_lt( compiler, version ):
    return config.compiler_type == compiler and \
           version_lt(config.compiler_version, version)
352

353 354 355
def compiler_le( compiler, version ):
    return config.compiler_type == compiler and \
           version_le(config.compiler_version, version)
356

357 358 359
def compiler_gt( compiler, version ):
    return config.compiler_type == compiler and \
           version_gt(config.compiler_version, version)
360

361 362 363
def compiler_ge( compiler, version ):
    return config.compiler_type == compiler and \
           version_ge(config.compiler_version, version)
364

365 366 367 368 369 370
def unregisterised( ):
    return config.unregisterised

def compiler_profiled( ):
    return config.compiler_profiled

371 372
def compiler_debugged( ):
    return config.compiler_debugged
373

374 375 376 377 378
def tag( t ):
    return t in config.compiler_tags

# ---

Ian Lynagh's avatar
Ian Lynagh committed
379 380
def namebase( nb ):
   return lambda opts, nb=nb: _namebase(opts, nb)
381

Ian Lynagh's avatar
Ian Lynagh committed
382 383
def _namebase( opts, nb ):
    opts.with_namebase = nb
384

385 386
# ---

387
def high_memory_usage(name, opts):
388 389
    opts.alone = True

Ian Lynagh's avatar
Ian Lynagh committed
390
# ---
391
def literate( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
392 393
    opts.literate = 1;

394
def c_src( name, opts ):
395 396
    opts.c_src = 1;

397
def objc_src( name, opts ):
Austin Seipp's avatar
Austin Seipp committed
398 399
    opts.objc_src = 1;

400
def objcpp_src( name, opts ):
401 402
    opts.objcpp_src = 1;

403
def cmm_src( name, opts ):
404 405
    opts.cmm_src = 1;

406
def outputdir( odir ):
407
    return lambda name, opts, d=odir: _outputdir(name, opts, d)
408

409
def _outputdir( name, opts, odir ):
410 411
    opts.outputdir = odir;

412 413
# ----

414
def pre_cmd( cmd ):
415
    return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
416

417
def _pre_cmd( name, opts, cmd ):
418 419 420 421
    opts.pre_cmd = cmd

# ----

422
def clean_cmd( cmd ):
423
    return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
424

425
def _clean_cmd( name, opts, cmd ):
426 427 428 429
    opts.clean_cmd = cmd

# ----

430
def cmd_prefix( prefix ):
431
    return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
432

433
def _cmd_prefix( name, opts, prefix ):
434 435 436 437 438
    opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;

# ----

def cmd_wrapper( fun ):
439
    return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
440

441
def _cmd_wrapper( name, opts, fun ):
442
    opts.cmd_wrapper = fun
443

444 445
# ----

Ian Lynagh's avatar
Ian Lynagh committed
446
def compile_cmd_prefix( prefix ):
447
    return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
Ian Lynagh's avatar
Ian Lynagh committed
448

449
def _compile_cmd_prefix( name, opts, prefix ):
Ian Lynagh's avatar
Ian Lynagh committed
450 451 452 453
    opts.compile_cmd_prefix = prefix

# ----

454
def normalise_slashes( name, opts ):
455 456
    opts.extra_normaliser = normalise_slashes_

457
def normalise_exe( name, opts ):
458 459
    opts.extra_normaliser = normalise_exe_

460
def normalise_fun( fun ):
461
    return lambda name, opts, f=fun: _normalise_fun(name, opts, f)
462

463
def _normalise_fun( name, opts, f ):
464 465
    opts.extra_normaliser = f

466
def normalise_errmsg_fun( fun ):
467
    return lambda name, opts, f=fun: _normalise_errmsg_fun(name, opts, f)
468

469
def _normalise_errmsg_fun( name, opts, f ):
470 471 472 473 474
    opts.extra_errmsg_normaliser = f

def two_normalisers(f, g):
    return lambda x, f=f, g=g: f(g(x))

475 476 477
# ----
# Function for composing two opt-fns together

478 479 480
def composes( fs ):
    return reduce(lambda f, g: compose(f, g), fs)

481
def compose( f, g ):
482
    return lambda name, opts, f=f, g=g: _compose(name, opts, f, g)
483

484 485 486
def _compose( name, opts, f, g ):
    f(name, opts)
    g(name, opts)
487 488 489 490 491

# -----------------------------------------------------------------------------
# The current directory of tests

def newTestDir( dir ):
492
    global thisdir_settings
493
    # reset the options for this test directory
494 495 496 497 498
    thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )

def _newTestDir( name, opts, dir ):
    opts.testdir = dir
    opts.compiler_always_flags = config.compiler_always_flags
499 500 501 502

# -----------------------------------------------------------------------------
# Actually doing tests

503 504
parallelTests = []
aloneTests = []
505
allTestNames = set([])
506

507
def runTest (opts, name, func, args):
ei@vuokko.info's avatar
ei@vuokko.info committed
508 509
    ok = 0

510
    if config.use_threads:
ei@vuokko.info's avatar
ei@vuokko.info committed
511
        t.thread_pool.acquire()
512
        try:
513
            while config.threads<(t.running_threads+1):
514
                t.thread_pool.wait()
515
            t.running_threads = t.running_threads+1
516 517
            ok=1
            t.thread_pool.release()
518
            thread.start_new_thread(test_common_thread, (name, opts, func, args))
519 520 521 522 523
        except:
            if not ok:
                t.thread_pool.release()
    else:
        test_common_work (name, opts, func, args)
524

525
# name  :: String
526
# setup :: TestOpts -> IO ()
527
def test (name, setup, func, args):
528 529
    global aloneTests
    global parallelTests
530
    global allTestNames
531
    global thisdir_settings
532 533
    if name in allTestNames:
        framework_fail(name, 'duplicate', 'There are multiple tests with this name')
534
    if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
535
        framework_fail(name, 'bad_name', 'This test has an invalid name')
536 537 538 539 540

    # Make a deep copy of the default_testopts, as we need our own copy
    # of any dictionaries etc inside it. Otherwise, if one test modifies
    # them, all tests will see the modified version!
    myTestOpts = copy.deepcopy(default_testopts)
541 542 543 544

    if type(setup) is types.ListType:
       setup = composes(setup)

545 546
    setup = compose(thisdir_settings, setup)
    setup(name, myTestOpts)
547 548 549 550 551 552

    thisTest = lambda : runTest(myTestOpts, name, func, args)
    if myTestOpts.alone:
        aloneTests.append(thisTest)
    else:
        parallelTests.append(thisTest)
553
    allTestNames.add(name)
554

555
if config.use_threads:
556
    def test_common_thread(name, opts, func, args):
557 558 559 560 561 562
        t.lock.acquire()
        try:
            test_common_work(name,opts,func,args)
        finally:
            t.lock.release()
            t.thread_pool.acquire()
563
            t.running_threads = t.running_threads - 1
564 565
            t.thread_pool.notify()
            t.thread_pool.release()
566

567 568 569 570 571 572 573 574 575 576
def get_package_cache_timestamp():
    if config.package_conf_cache_file == '':
        return 0.0
    else:
        try:
            return os.stat(config.package_conf_cache_file).st_mtime
        except:
            return 0.0


ei@vuokko.info's avatar
ei@vuokko.info committed
577
def test_common_work (name, opts, func, args):
578 579 580 581 582 583 584 585 586
    try:
        t.total_tests = t.total_tests+1
        setLocalTestOpts(opts)

        package_conf_cache_file_start_timestamp = get_package_cache_timestamp()

        # All the ways we might run this test
        if func == compile or func == multimod_compile:
            all_ways = config.compile_ways
587
        elif func == compile_and_run or func == multimod_compile_and_run:
588 589 590 591 592 593
            all_ways = config.run_ways
        elif func == ghci_script:
            if 'ghci' in config.run_ways:
                all_ways = ['ghci']
            else:
                all_ways = []
594
        else:
595 596 597 598 599 600 601 602 603 604 605
            all_ways = ['normal']

        # A test itself can request extra ways by setting opts.extra_ways
        all_ways = all_ways + filter(lambda way: way not in all_ways,
                                     opts.extra_ways)

        t.total_test_cases = t.total_test_cases + len(all_ways)

        ok_way = lambda way: \
            not getTestOpts().skip \
            and (config.only == [] or name in config.only) \
606
            and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
607 608 609 610 611 612 613 614 615 616 617 618 619
            and (config.cmdline_ways == [] or way in config.cmdline_ways) \
            and way not in getTestOpts().omit_ways

        # Which ways we are asked to skip
        do_ways = filter (ok_way,all_ways)

        # In fast mode, we skip all but one way
        if config.fast and len(do_ways) > 0:
            do_ways = [do_ways[0]]

        if not config.clean_only:
            # Run the required tests...
            for way in do_ways:
620 621
                if stopping():
                    break
622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642
                do_test (name, way, func, args)

            for way in all_ways:
                if way not in do_ways:
                    skiptest (name,way)

        if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
            clean(map (lambda suff: name + suff,
                      ['', '.exe', '.exe.manifest', '.genscript',
                       '.stderr.normalised',        '.stdout.normalised',
                       '.run.stderr',               '.run.stdout',
                       '.run.stderr.normalised',    '.run.stdout.normalised',
                       '.comp.stderr',              '.comp.stdout',
                       '.comp.stderr.normalised',   '.comp.stdout.normalised',
                       '.interp.stderr',            '.interp.stdout',
                       '.interp.stderr.normalised', '.interp.stdout.normalised',
                       '.stats', '.comp.stats',
                       '.hi', '.o', '.prof', '.exe.prof', '.hc',
                       '_stub.h', '_stub.c', '_stub.o',
                       '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))

643
            if func == multi_compile or func == multi_compile_fail:
644 645 646 647 648
                    extra_mods = args[1]
                    clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
                    clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))

            clean(getTestOpts().clean_files)
Ian Lynagh's avatar
Ian Lynagh committed
649

650 651 652 653 654 655 656
            if getTestOpts().outputdir != None:
                odir = in_testdir(getTestOpts().outputdir)
                try:
                    shutil.rmtree(odir)
                except:
                    pass

657 658 659 660 661
            try:
                shutil.rmtree(in_testdir('.hpc.' + name))
            except:
                pass

662 663 664 665 666 667 668 669
            try:
                cleanCmd = getTestOpts().clean_cmd
                if cleanCmd != None:
                    result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
                    if result != 0:
                        framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
            except e:
                framework_fail(name, 'cleaning', 'clean-command exception')
670

671
        package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
672

673 674
        if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
            framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
675

676 677 678 679 680 681 682 683 684 685 686 687
        try:
            for f in files_written[name]:
                if os.path.exists(f):
                    try:
                        if not f in files_written_not_removed[name]:
                            files_written_not_removed[name].append(f)
                    except:
                        files_written_not_removed[name] = [f]
        except:
            pass
    except Exception, e:
        framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
Ian Lynagh's avatar
Ian Lynagh committed
688

689 690 691 692
def clean(strs):
    for str in strs:
        for name in glob.glob(in_testdir(str)):
            clean_full_path(name)
693

694
def clean_full_path(name):
Ian Lynagh's avatar
Ian Lynagh committed
695 696 697 698
        try:
            # Remove files...
            os.remove(name)
        except OSError, e1:
699
            try:
Ian Lynagh's avatar
Ian Lynagh committed
700 701 702 703 704 705 706 707 708 709 710
                # ... and empty directories
                os.rmdir(name)
            except OSError, e2:
                # We don't want to fail here, but we do want to know
                # what went wrong, so print out the exceptions.
                # ENOENT isn't a problem, though, as we clean files
                # that don't necessarily exist.
                if e1.errno != errno.ENOENT:
                    print e1
                if e2.errno != errno.ENOENT:
                    print e2
711

712
def do_test(name, way, func, args):
713 714 715
    full_name = name + '(' + way + ')'

    try:
716
        print '=====>', full_name, t.total_tests, 'of', len(allTestNames), \
717 718 719
                        str([t.n_unexpected_passes,   \
                             t.n_unexpected_failures, \
                             t.n_framework_failures])
720

721 722
        if config.use_threads:
            t.lock.release()
723 724 725 726

        try:
            preCmd = getTestOpts().pre_cmd
            if preCmd != None:
727
                result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
728 729 730 731 732
                if result != 0:
                    framework_fail(name, way, 'pre-command failed: ' + str(result))
        except e:
            framework_fail(name, way, 'pre-command exception')

ei@vuokko.info's avatar
ei@vuokko.info committed
733 734 735
        try:
            result = apply(func, [name,way] + args)
        finally:
736 737
            if config.use_threads:
                t.lock.acquire()
738

739 740 741
        if getTestOpts().expect != 'pass' and \
                getTestOpts().expect != 'fail' and \
                getTestOpts().expect != 'missing-lib':
742
            framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
743

744 745 746 747 748 749
        try:
            passFail = result['passFail']
        except:
            passFail = 'No passFail found'

        if passFail == 'pass':
ei@vuokko.info's avatar
ei@vuokko.info committed
750 751
            if getTestOpts().expect == 'pass' \
               and way not in getTestOpts().expect_fail_for:
752
                t.n_expected_passes = t.n_expected_passes + 1
753 754 755 756
                if name in t.expected_passes:
                    t.expected_passes[name].append(way)
                else:
                    t.expected_passes[name] = [way]
757 758 759
            else:
                print '*** unexpected pass for', full_name
                t.n_unexpected_passes = t.n_unexpected_passes + 1
760
                addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
761
        elif passFail == 'fail':
ei@vuokko.info's avatar
ei@vuokko.info committed
762 763
            if getTestOpts().expect == 'pass' \
               and way not in getTestOpts().expect_fail_for:
764 765
                print '*** unexpected failure for', full_name
                t.n_unexpected_failures = t.n_unexpected_failures + 1
766 767
                reason = result['reason']
                addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
768
            else:
769 770 771 772 773 774
                if getTestOpts().expect == 'missing-lib':
                    t.n_missing_libs = t.n_missing_libs + 1
                    if name in t.missing_libs:
                        t.missing_libs[name].append(way)
                    else:
                        t.missing_libs[name] = [way]
775
                else:
776 777 778 779 780
                    t.n_expected_failures = t.n_expected_failures + 1
                    if name in t.expected_failures:
                        t.expected_failures[name].append(way)
                    else:
                        t.expected_failures[name] = [way]
781 782
        else:
            framework_fail(name, way, 'bad result ' + passFail)
783
    except KeyboardInterrupt:
784
        stopNow()
785
    except:
786
        framework_fail(name, way, 'do_test exception')
787 788
        traceback.print_exc()

789
def addPassingTestInfo (testInfos, directory, name, way):
790 791 792 793 794 795 796 797 798 799
    directory = re.sub('^\\.[/\\\\]', '', directory)

    if not directory in testInfos:
        testInfos[directory] = {}

    if not name in testInfos[directory]:
        testInfos[directory][name] = []

    testInfos[directory][name].append(way)

800 801 802 803 804 805 806 807 808 809 810 811 812 813
def addFailingTestInfo (testInfos, directory, name, reason, way):
    directory = re.sub('^\\.[/\\\\]', '', directory)

    if not directory in testInfos:
        testInfos[directory] = {}

    if not name in testInfos[directory]:
        testInfos[directory][name] = {}

    if not reason in testInfos[directory][name]:
        testInfos[directory][name][reason] = []

    testInfos[directory][name][reason].append(way)

814
def skiptest (name, way):
815 816
    # print 'Skipping test \"', name, '\"'
    t.n_tests_skipped = t.n_tests_skipped + 1
817 818 819 820
    if name in t.tests_skipped:
        t.tests_skipped[name].append(way)
    else:
        t.tests_skipped[name] = [way]
821

822 823
def framework_fail( name, way, reason ):
    full_name = name + '(' + way + ')'
824
    print '*** framework failure for', full_name, reason
825
    t.n_framework_failures = t.n_framework_failures + 1
826 827 828 829
    if name in t.framework_failures:
        t.framework_failures[name].append(way)
    else:
        t.framework_failures[name] = [way]
830

831 832 833 834 835 836 837 838 839 840 841 842 843 844
def badResult(result):
    try:
        if result['passFail'] == 'pass':
            return False
        return True
    except:
        return True

def passed():
    return {'passFail': 'pass'}

def failBecause(reason):
    return {'passFail': 'fail', 'reason': reason}

845 846 847 848
# -----------------------------------------------------------------------------
# Generic command tests

# A generic command test is expected to run and exit successfully.
849 850 851 852 853 854
#
# The expected exit code can be changed via exit_code() as normal, and
# the expected stdout/stderr are stored in <testname>.stdout and
# <testname>.stderr.  The output of the command can be ignored
# altogether by using run_command_ignore_output instead of
# run_command.
855

856
def run_command( name, way, cmd ):
857
    return simple_run( name, '', cmd, '' )
858

859 860 861 862
# -----------------------------------------------------------------------------
# GHCi tests

def ghci_script( name, way, script ):
863
    # filter out -fforce-recomp from compiler_always_flags, because we're
864
    # actually testing the recompilation behaviour in the GHCi tests.
865
    flags = filter(lambda f: f != '-fforce-recomp', getTestOpts().compiler_always_flags)
866
    flags.append(getTestOpts().extra_hc_opts)
867 868
    if getTestOpts().outputdir != None:
        flags.extend(["-outputdir", getTestOpts().outputdir])
869 870 871 872 873 874

    # We pass HC and HC_OPTS as environment variables, so that the
    # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
    cmd = "HC='" + config.compiler + "' " + \
          "HC_OPTS='" + join(flags,' ') + "' " + \
          "'" + config.compiler + "'" + \
875
          ' --interactive -v0 -ignore-dot-ghci ' + \
876 877
          join(flags,' ')

ei@vuokko.info's avatar
ei@vuokko.info committed
878
    getTestOpts().stdin = script
879
    return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
880