testlib.py 72.2 KB
Newer Older
1
#
2 3 4
# (c) Simon Marlow 2002
#

5 6 7
# This allows us to use the "with X:" syntax with python 2.5:
from __future__ import with_statement

8
import shutil
9 10
import sys
import os
11
import errno
12 13 14
import string
import re
import traceback
15 16
import time
import datetime
17
import copy
18
import glob
19
import types
ian@well-typed.com's avatar
ian@well-typed.com committed
20
from math import ceil, trunc
21

22 23 24 25 26 27 28
have_subprocess = False
try:
    import subprocess
    have_subprocess = True
except:
    print "Warning: subprocess not found, will fall back to spawnv"

29
from string import join
30
from testglobals import *
31 32
from testutil import *

33 34 35
if config.use_threads:
    import threading
    import thread
36

37 38 39 40 41 42 43 44
global wantToStop
wantToStop = False
def stopNow():
    global wantToStop
    wantToStop = True
def stopping():
    return wantToStop

45 46
# Options valid for the current test only (these get reset to
# testdir_testopts after each test).
47

ei@vuokko.info's avatar
ei@vuokko.info committed
48
global testopts_local
49 50 51 52 53 54
if config.use_threads:
    testopts_local = threading.local()
else:
    class TestOpts_Local:
        pass
    testopts_local = TestOpts_Local()
55 56

def getTestOpts():
ei@vuokko.info's avatar
ei@vuokko.info committed
57
    return testopts_local.x
58

ei@vuokko.info's avatar
ei@vuokko.info committed
59 60 61
def setLocalTestOpts(opts):
    global testopts_local
    testopts_local.x=opts
62

63 64 65 66 67
def isStatsTest():
    opts = getTestOpts()
    return len(opts.compiler_stats_range_fields) > 0 or len(opts.stats_range_fields) > 0


68 69 70
# This can be called at the top of a file of tests, to set default test options
# for the following tests.
def setTestOpts( f ):
71
    global thisdir_settings
72
    thisdir_settings = [thisdir_settings, f]
73 74 75 76 77 78 79 80 81 82 83 84

# -----------------------------------------------------------------------------
# Canned setup functions for common cases.  eg. for a test you might say
#
#      test('test001', normal, compile, [''])
#
# to run it without any options, but change it to
#
#      test('test001', expect_fail, compile, [''])
#
# to expect failure for this test.

85
def normal( name, opts ):
86 87
    return;

88
def skip( name, opts ):
89 90
    opts.skip = 1

91
def expect_fail( name, opts ):
92 93
    opts.expect = 'fail';

94
def reqlib( lib ):
95
    return lambda name, opts, l=lib: _reqlib (name, opts, l )
96

97 98 99 100
# Cache the results of looking to see if we have a library or not.
# This makes quite a difference, especially on Windows.
have_lib = {}

101
def _reqlib( name, opts, lib ):
102 103
    if have_lib.has_key(lib):
        got_it = have_lib[lib]
104
    else:
105 106 107
        if have_subprocess:
            # By preference we use subprocess, as the alternative uses
            # /dev/null which mingw doesn't have.
108
            p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
109 110 111 112 113 114 115 116 117 118 119 120 121
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
            # read from stdout and stderr to avoid blocking due to
            # buffers filling
            p.communicate()
            r = p.wait()
        else:
            r = os.system(config.ghc_pkg + ' describe ' + lib
                                         + ' > /dev/null 2> /dev/null')
        got_it = r == 0
        have_lib[lib] = got_it

    if not got_it:
122
        opts.expect = 'missing-lib'
123

124
def req_profiling( name, opts ):
125 126 127
    if not config.have_profiling:
        opts.expect = 'fail'

128
def req_shared_libs( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
129 130 131
    if not config.have_shared_libs:
        opts.expect = 'fail'

132
def req_interp( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
133 134 135
    if not config.have_interp:
        opts.expect = 'fail'

136
def req_smp( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
137 138 139
    if not config.have_smp:
        opts.expect = 'fail'

140
def ignore_output( name, opts ):
141 142
    opts.ignore_output = 1

143
def no_stdin( name, opts ):
144 145
    opts.no_stdin = 1

146
def combined_output( name, opts ):
pcapriotti's avatar
pcapriotti committed
147 148
    opts.combined_output = True

149 150 151
# -----

def expect_fail_for( ways ):
152
    return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
153

154
def _expect_fail_for( name, opts, ways ):
155 156
    opts.expect_fail_for = ways

157 158 159 160
def expect_broken( bug ):
    return lambda name, opts, b=bug: _expect_broken (name, opts, b )

def _expect_broken( name, opts, bug ):
161
    record_broken(name, opts, bug)
162 163
    opts.expect = 'fail';

Ian Lynagh's avatar
Ian Lynagh committed
164
def expect_broken_for( bug, ways ):
165
    return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
Ian Lynagh's avatar
Ian Lynagh committed
166

167
def _expect_broken_for( name, opts, bug, ways ):
168
    record_broken(name, opts, bug)
Ian Lynagh's avatar
Ian Lynagh committed
169
    opts.expect_fail_for = ways
Ian Lynagh's avatar
Ian Lynagh committed
170

171
def record_broken(name, opts, bug):
172
    global brokens
173
    me = (bug, opts.testdir, name)
174 175 176
    if not me in brokens:
        brokens.append(me)

177 178 179
# -----

def omit_ways( ways ):
180
    return lambda name, opts, w=ways: _omit_ways( name, opts, w )
181

182
def _omit_ways( name, opts, ways ):
183 184 185 186
    opts.omit_ways = ways

# -----

187
def only_ways( ways ):
188
    return lambda name, opts, w=ways: _only_ways( name, opts, w )
189

190
def _only_ways( name, opts, ways ):
191 192 193 194
    opts.only_ways = ways

# -----

195
def extra_ways( ways ):
196
    return lambda name, opts, w=ways: _extra_ways( name, opts, w )
197

198
def _extra_ways( name, opts, ways ):
199 200 201 202
    opts.extra_ways = ways

# -----

ross's avatar
ross committed
203
def omit_compiler_types( compiler_types ):
204
   return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
ross's avatar
ross committed
205

206
def _omit_compiler_types( name, opts, compiler_types ):
ross's avatar
ross committed
207
    if config.compiler_type in compiler_types:
dterei's avatar
dterei committed
208
        opts.skip = 1
ross's avatar
ross committed
209 210 211 212

# -----

def only_compiler_types( compiler_types ):
213
   return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
ross's avatar
ross committed
214

215
def _only_compiler_types( name, opts, compiler_types ):
ross's avatar
ross committed
216
    if config.compiler_type not in compiler_types:
dterei's avatar
dterei committed
217
        opts.skip = 1
ross's avatar
ross committed
218 219 220

# -----

221
def set_stdin( file ):
222
   return lambda name, opts, f=file: _set_stdin(name, opts, f);
223

224
def _set_stdin( name, opts, f ):
225 226 227 228 229
   opts.stdin = f

# -----

def exit_code( val ):
230
    return lambda name, opts, v=val: _exit_code(name, opts, v);
231

232
def _exit_code( name, opts, v ):
233 234 235 236
    opts.exit_code = v

# -----

237
def timeout_multiplier( val ):
238
    return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
239

240
def _timeout_multiplier( name, opts, v ):
241 242 243 244
    opts.timeout_multiplier = v

# -----

245
def extra_run_opts( val ):
246
    return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
247

248
def _extra_run_opts( name, opts, v ):
249 250
    opts.extra_run_opts = v

251 252
# -----

Simon Marlow's avatar
Simon Marlow committed
253
def extra_hc_opts( val ):
254
    return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
Simon Marlow's avatar
Simon Marlow committed
255

256
def _extra_hc_opts( name, opts, v ):
Simon Marlow's avatar
Simon Marlow committed
257 258 259 260
    opts.extra_hc_opts = v

# -----

261
def extra_clean( files ):
262
    return lambda name, opts, v=files: _extra_clean(name, opts, v);
263

264
def _extra_clean( name, opts, v ):
265 266
    opts.clean_files = v

267 268
# -----

269 270 271 272 273 274 275
def stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);

def _stats_num_field( name, opts, field, expecteds ):
    if field in opts.stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

276 277 278 279 280 281
    if type(expecteds) is types.ListType:
        for (b, expected, dev) in expecteds:
            if b:
                opts.stats_range_fields[field] = (expected, dev)
                return
        framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
282

283 284 285
    else:
        (expected, dev) = expecteds
        opts.stats_range_fields[field] = (expected, dev)
286 287 288

def compiler_stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
289

290 291 292 293 294 295 296 297
def _compiler_stats_num_field( name, opts, field, expecteds ):
    if field in opts.compiler_stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

    for (b, expected, dev) in expecteds:
        if b:
            opts.compiler_stats_range_fields[field] = (expected, dev)
            return
298

299 300
    framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')

301 302
# -----

303
def when(b, f):
ian@well-typed.com's avatar
ian@well-typed.com committed
304 305 306
    # When list_brokens is on, we want to see all expect_broken calls,
    # so we always do f
    if b or config.list_broken:
307 308 309 310 311 312 313
        return f
    else:
        return normal

def unless(b, f):
    return when(not b, f)

ian@well-typed.com's avatar
ian@well-typed.com committed
314 315 316
def doing_ghci():
    return 'ghci' in config.run_ways

317
def ghci_dynamic( ):
ian@well-typed.com's avatar
ian@well-typed.com committed
318
    return config.ghc_dynamic
319 320 321 322

def fast():
    return config.fast

323 324
def platform( plat ):
    return config.platform == plat
Ian Lynagh's avatar
Ian Lynagh committed
325

326 327
def opsys( os ):
    return config.os == os
Ian Lynagh's avatar
Ian Lynagh committed
328

329 330
def arch( arch ):
    return config.arch == arch
331

332 333
def wordsize( ws ):
    return config.wordsize == str(ws)
tibbe's avatar
tibbe committed
334

335 336
def msys( ):
    return config.msys
ian@well-typed.com's avatar
ian@well-typed.com committed
337

338 339
def cygwin( ):
    return config.cygwin
Ian Lynagh's avatar
Ian Lynagh committed
340

341 342
def have_vanilla( ):
    return config.have_vanilla
343

344 345
def have_dynamic( ):
    return config.have_dynamic
346

347 348
def have_profiling( ):
    return config.have_profiling
349

350 351
def in_tree_compiler( ):
    return config.in_tree_compiler
352

353 354
def compiler_type( compiler ):
    return config.compiler_type == compiler
355

356 357 358
def compiler_lt( compiler, version ):
    return config.compiler_type == compiler and \
           version_lt(config.compiler_version, version)
359

360 361 362
def compiler_le( compiler, version ):
    return config.compiler_type == compiler and \
           version_le(config.compiler_version, version)
363

364 365 366
def compiler_gt( compiler, version ):
    return config.compiler_type == compiler and \
           version_gt(config.compiler_version, version)
367

368 369 370
def compiler_ge( compiler, version ):
    return config.compiler_type == compiler and \
           version_ge(config.compiler_version, version)
371

372 373 374 375 376 377
def unregisterised( ):
    return config.unregisterised

def compiler_profiled( ):
    return config.compiler_profiled

378 379
def compiler_debugged( ):
    return config.compiler_debugged
380

381 382 383 384 385
def tag( t ):
    return t in config.compiler_tags

# ---

Ian Lynagh's avatar
Ian Lynagh committed
386 387
def namebase( nb ):
   return lambda opts, nb=nb: _namebase(opts, nb)
388

Ian Lynagh's avatar
Ian Lynagh committed
389 390
def _namebase( opts, nb ):
    opts.with_namebase = nb
391

392 393
# ---

394
def high_memory_usage(name, opts):
395 396
    opts.alone = True

397 398 399 400 401
# If a test is for a multi-CPU race, then running the test alone
# increases the chance that we'll actually see it.
def multi_cpu_race(name, opts):
    opts.alone = True

Ian Lynagh's avatar
Ian Lynagh committed
402
# ---
403
def literate( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
404 405
    opts.literate = 1;

406
def c_src( name, opts ):
407 408
    opts.c_src = 1;

409
def objc_src( name, opts ):
Austin Seipp's avatar
Austin Seipp committed
410 411
    opts.objc_src = 1;

412
def objcpp_src( name, opts ):
413 414
    opts.objcpp_src = 1;

415
def cmm_src( name, opts ):
416 417
    opts.cmm_src = 1;

418
def outputdir( odir ):
419
    return lambda name, opts, d=odir: _outputdir(name, opts, d)
420

421
def _outputdir( name, opts, odir ):
422 423
    opts.outputdir = odir;

424 425
# ----

426
def pre_cmd( cmd ):
427
    return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
428

429
def _pre_cmd( name, opts, cmd ):
430 431 432 433
    opts.pre_cmd = cmd

# ----

434
def clean_cmd( cmd ):
435
    return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
436

437
def _clean_cmd( name, opts, cmd ):
438 439 440 441
    opts.clean_cmd = cmd

# ----

442
def cmd_prefix( prefix ):
443
    return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
444

445
def _cmd_prefix( name, opts, prefix ):
446 447 448 449 450
    opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;

# ----

def cmd_wrapper( fun ):
451
    return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
452

453
def _cmd_wrapper( name, opts, fun ):
454
    opts.cmd_wrapper = fun
455

456 457
# ----

Ian Lynagh's avatar
Ian Lynagh committed
458
def compile_cmd_prefix( prefix ):
459
    return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
Ian Lynagh's avatar
Ian Lynagh committed
460

461
def _compile_cmd_prefix( name, opts, prefix ):
Ian Lynagh's avatar
Ian Lynagh committed
462 463 464 465
    opts.compile_cmd_prefix = prefix

# ----

466
def normalise_slashes( name, opts ):
467 468
    opts.extra_normaliser = normalise_slashes_

469
def normalise_exe( name, opts ):
470 471
    opts.extra_normaliser = normalise_exe_

472
def normalise_fun( fun ):
473
    return lambda name, opts, f=fun: _normalise_fun(name, opts, f)
474

475
def _normalise_fun( name, opts, f ):
476 477
    opts.extra_normaliser = f

478
def normalise_errmsg_fun( fun ):
479
    return lambda name, opts, f=fun: _normalise_errmsg_fun(name, opts, f)
480

481
def _normalise_errmsg_fun( name, opts, f ):
482 483 484 485 486
    opts.extra_errmsg_normaliser = f

def two_normalisers(f, g):
    return lambda x, f=f, g=g: f(g(x))

487 488 489
# ----
# Function for composing two opt-fns together

490 491 492 493 494 495 496
def executeSetups(fs, name, opts):
    if type(fs) is types.ListType:
        # If we have a list of setups, then execute each one
        map (lambda f : executeSetups(f, name, opts), fs)
    else:
        # fs is a single function, so just apply it
        fs(name, opts)
497

498 499 500 501
# -----------------------------------------------------------------------------
# The current directory of tests

def newTestDir( dir ):
502
    global thisdir_settings
503
    # reset the options for this test directory
504 505 506 507 508
    thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )

def _newTestDir( name, opts, dir ):
    opts.testdir = dir
    opts.compiler_always_flags = config.compiler_always_flags
509 510 511 512

# -----------------------------------------------------------------------------
# Actually doing tests

513 514
parallelTests = []
aloneTests = []
515
allTestNames = set([])
516

517
def runTest (opts, name, func, args):
ei@vuokko.info's avatar
ei@vuokko.info committed
518 519
    ok = 0

520
    if config.use_threads:
ei@vuokko.info's avatar
ei@vuokko.info committed
521
        t.thread_pool.acquire()
522
        try:
523
            while config.threads<(t.running_threads+1):
524
                t.thread_pool.wait()
525
            t.running_threads = t.running_threads+1
526 527
            ok=1
            t.thread_pool.release()
528
            thread.start_new_thread(test_common_thread, (name, opts, func, args))
529 530 531 532 533
        except:
            if not ok:
                t.thread_pool.release()
    else:
        test_common_work (name, opts, func, args)
534

535
# name  :: String
536
# setup :: TestOpts -> IO ()
537
def test (name, setup, func, args):
538 539
    global aloneTests
    global parallelTests
540
    global allTestNames
541
    global thisdir_settings
542 543
    if name in allTestNames:
        framework_fail(name, 'duplicate', 'There are multiple tests with this name')
544
    if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
545
        framework_fail(name, 'bad_name', 'This test has an invalid name')
546 547 548 549 550

    # Make a deep copy of the default_testopts, as we need our own copy
    # of any dictionaries etc inside it. Otherwise, if one test modifies
    # them, all tests will see the modified version!
    myTestOpts = copy.deepcopy(default_testopts)
551

552
    executeSetups([thisdir_settings, setup], name, myTestOpts)
553 554 555 556 557 558

    thisTest = lambda : runTest(myTestOpts, name, func, args)
    if myTestOpts.alone:
        aloneTests.append(thisTest)
    else:
        parallelTests.append(thisTest)
559
    allTestNames.add(name)
560

561
if config.use_threads:
562
    def test_common_thread(name, opts, func, args):
563 564 565 566 567 568
        t.lock.acquire()
        try:
            test_common_work(name,opts,func,args)
        finally:
            t.lock.release()
            t.thread_pool.acquire()
569
            t.running_threads = t.running_threads - 1
570 571
            t.thread_pool.notify()
            t.thread_pool.release()
572

573 574 575 576 577 578 579 580 581 582
def get_package_cache_timestamp():
    if config.package_conf_cache_file == '':
        return 0.0
    else:
        try:
            return os.stat(config.package_conf_cache_file).st_mtime
        except:
            return 0.0


ei@vuokko.info's avatar
ei@vuokko.info committed
583
def test_common_work (name, opts, func, args):
584 585 586 587 588 589 590 591 592
    try:
        t.total_tests = t.total_tests+1
        setLocalTestOpts(opts)

        package_conf_cache_file_start_timestamp = get_package_cache_timestamp()

        # All the ways we might run this test
        if func == compile or func == multimod_compile:
            all_ways = config.compile_ways
593
        elif func == compile_and_run or func == multimod_compile_and_run:
594 595 596 597 598 599
            all_ways = config.run_ways
        elif func == ghci_script:
            if 'ghci' in config.run_ways:
                all_ways = ['ghci']
            else:
                all_ways = []
600
        else:
601 602 603 604 605 606 607 608 609 610 611
            all_ways = ['normal']

        # A test itself can request extra ways by setting opts.extra_ways
        all_ways = all_ways + filter(lambda way: way not in all_ways,
                                     opts.extra_ways)

        t.total_test_cases = t.total_test_cases + len(all_ways)

        ok_way = lambda way: \
            not getTestOpts().skip \
            and (config.only == [] or name in config.only) \
612
            and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
613
            and (config.cmdline_ways == [] or way in config.cmdline_ways) \
614
            and (not (config.skip_perf_tests and isStatsTest())) \
615 616 617 618 619 620 621 622 623 624 625 626
            and way not in getTestOpts().omit_ways

        # Which ways we are asked to skip
        do_ways = filter (ok_way,all_ways)

        # In fast mode, we skip all but one way
        if config.fast and len(do_ways) > 0:
            do_ways = [do_ways[0]]

        if not config.clean_only:
            # Run the required tests...
            for way in do_ways:
627 628
                if stopping():
                    break
629 630 631 632 633 634 635
                do_test (name, way, func, args)

            for way in all_ways:
                if way not in do_ways:
                    skiptest (name,way)

        if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
636
            pretest_cleanup(name)
637 638 639 640 641 642 643 644 645 646 647
            clean(map (lambda suff: name + suff,
                      ['', '.exe', '.exe.manifest', '.genscript',
                       '.stderr.normalised',        '.stdout.normalised',
                       '.run.stderr.normalised',    '.run.stdout.normalised',
                       '.comp.stderr.normalised',   '.comp.stdout.normalised',
                       '.interp.stderr.normalised', '.interp.stdout.normalised',
                       '.stats', '.comp.stats',
                       '.hi', '.o', '.prof', '.exe.prof', '.hc',
                       '_stub.h', '_stub.c', '_stub.o',
                       '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))

648
            if func == multi_compile or func == multi_compile_fail:
649 650 651 652 653
                    extra_mods = args[1]
                    clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
                    clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))

            clean(getTestOpts().clean_files)
Ian Lynagh's avatar
Ian Lynagh committed
654

655 656 657 658 659 660 661
            if getTestOpts().outputdir != None:
                odir = in_testdir(getTestOpts().outputdir)
                try:
                    shutil.rmtree(odir)
                except:
                    pass

662 663 664 665 666
            try:
                shutil.rmtree(in_testdir('.hpc.' + name))
            except:
                pass

667 668 669 670 671 672 673 674
            try:
                cleanCmd = getTestOpts().clean_cmd
                if cleanCmd != None:
                    result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
                    if result != 0:
                        framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
            except e:
                framework_fail(name, 'cleaning', 'clean-command exception')
675

676
        package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
677

678 679
        if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
            framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
680

681 682 683 684 685 686 687 688 689 690 691 692
        try:
            for f in files_written[name]:
                if os.path.exists(f):
                    try:
                        if not f in files_written_not_removed[name]:
                            files_written_not_removed[name].append(f)
                    except:
                        files_written_not_removed[name] = [f]
        except:
            pass
    except Exception, e:
        framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
Ian Lynagh's avatar
Ian Lynagh committed
693

694 695 696 697
def clean(strs):
    for str in strs:
        for name in glob.glob(in_testdir(str)):
            clean_full_path(name)
698

699
def clean_full_path(name):
Ian Lynagh's avatar
Ian Lynagh committed
700 701 702 703
        try:
            # Remove files...
            os.remove(name)
        except OSError, e1:
704
            try:
Ian Lynagh's avatar
Ian Lynagh committed
705 706 707 708 709 710 711 712 713 714 715
                # ... and empty directories
                os.rmdir(name)
            except OSError, e2:
                # We don't want to fail here, but we do want to know
                # what went wrong, so print out the exceptions.
                # ENOENT isn't a problem, though, as we clean files
                # that don't necessarily exist.
                if e1.errno != errno.ENOENT:
                    print e1
                if e2.errno != errno.ENOENT:
                    print e2
716

717
def do_test(name, way, func, args):
718 719 720
    full_name = name + '(' + way + ')'

    try:
721 722 723 724 725
        if_verbose(2, "=====> %s %d of %d %s " % \
                    (full_name, t.total_tests, len(allTestNames), \
                    [t.n_unexpected_passes, \
                     t.n_unexpected_failures, \
                     t.n_framework_failures]))
726

727 728
        if config.use_threads:
            t.lock.release()
729 730 731 732

        try:
            preCmd = getTestOpts().pre_cmd
            if preCmd != None:
733
                result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
734 735 736 737 738
                if result != 0:
                    framework_fail(name, way, 'pre-command failed: ' + str(result))
        except e:
            framework_fail(name, way, 'pre-command exception')

ei@vuokko.info's avatar
ei@vuokko.info committed
739 740 741
        try:
            result = apply(func, [name,way] + args)
        finally:
742 743
            if config.use_threads:
                t.lock.acquire()
744

745 746 747
        if getTestOpts().expect != 'pass' and \
                getTestOpts().expect != 'fail' and \
                getTestOpts().expect != 'missing-lib':
748
            framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
749

750 751 752 753 754 755
        try:
            passFail = result['passFail']
        except:
            passFail = 'No passFail found'

        if passFail == 'pass':
ei@vuokko.info's avatar
ei@vuokko.info committed
756 757
            if getTestOpts().expect == 'pass' \
               and way not in getTestOpts().expect_fail_for:
758
                t.n_expected_passes = t.n_expected_passes + 1
759 760 761 762
                if name in t.expected_passes:
                    t.expected_passes[name].append(way)
                else:
                    t.expected_passes[name] = [way]
763
            else:
764
                if_verbose(1, '*** unexpected pass for %s' % full_name)
765
                t.n_unexpected_passes = t.n_unexpected_passes + 1
766
                addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
767
        elif passFail == 'fail':
ei@vuokko.info's avatar
ei@vuokko.info committed
768 769
            if getTestOpts().expect == 'pass' \
               and way not in getTestOpts().expect_fail_for:
770
                if_verbose(1, '*** unexpected failure for %s' % full_name)
771
                t.n_unexpected_failures = t.n_unexpected_failures + 1
772 773
                reason = result['reason']
                addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
774
            else:
775 776 777 778 779 780
                if getTestOpts().expect == 'missing-lib':
                    t.n_missing_libs = t.n_missing_libs + 1
                    if name in t.missing_libs:
                        t.missing_libs[name].append(way)
                    else:
                        t.missing_libs[name] = [way]
781
                else:
782 783 784 785 786
                    t.n_expected_failures = t.n_expected_failures + 1
                    if name in t.expected_failures:
                        t.expected_failures[name].append(way)
                    else:
                        t.expected_failures[name] = [way]
787 788
        else:
            framework_fail(name, way, 'bad result ' + passFail)
789
    except KeyboardInterrupt:
790
        stopNow()
791
    except:
792
        framework_fail(name, way, 'do_test exception')
793 794
        traceback.print_exc()

795
def addPassingTestInfo (testInfos, directory, name, way):
796 797 798 799 800 801 802 803 804 805
    directory = re.sub('^\\.[/\\\\]', '', directory)

    if not directory in testInfos:
        testInfos[directory] = {}

    if not name in testInfos[directory]:
        testInfos[directory][name] = []

    testInfos[directory][name].append(way)

806 807 808 809 810 811 812 813 814 815 816 817 818 819
def addFailingTestInfo (testInfos, directory, name, reason, way):
    directory = re.sub('^\\.[/\\\\]', '', directory)

    if not directory in testInfos:
        testInfos[directory] = {}

    if not name in testInfos[directory]:
        testInfos[directory][name] = {}

    if not reason in testInfos[directory][name]:
        testInfos[directory][name][reason] = []

    testInfos[directory][name][reason].append(way)

820
def skiptest (name, way):
821 822
    # print 'Skipping test \"', name, '\"'
    t.n_tests_skipped = t.n_tests_skipped + 1
823 824 825 826
    if name in t.tests_skipped:
        t.tests_skipped[name].append(way)
    else:
        t.tests_skipped[name] = [way]
827

828 829
def framework_fail( name, way, reason ):
    full_name = name + '(' + way + ')'
Joachim Breitner's avatar
Joachim Breitner committed
830
    if_verbose(1, '*** framework failure for %s %s ' % (full_name, reason))
831
    t.n_framework_failures = t.n_framework_failures + 1
832 833 834 835
    if name in t.framework_failures:
        t.framework_failures[name].append(way)
    else:
        t.framework_failures[name] = [way]
836

837 838 839 840 841 842 843 844 845 846 847 848 849 850
def badResult(result):
    try:
        if result['passFail'] == 'pass':
            return False
        return True
    except:
        return True

def passed():
    return {'passFail': 'pass'}

def failBecause(reason):
    return {'passFail': 'fail', 'reason': reason}

851 852 853 854
# -----------------------------------------------------------------------------
# Generic command tests

# A generic command test is expected to run and exit successfully.
855 856 857 858 859 860
#
# The expected exit code can be changed via exit_code() as normal, and
# the expected stdout/stderr are stored in <testname>.stdout and
# <testname>.stderr.  The output of the command can be ignored
# altogether by using run_command_ignore_output instead of
# run_command.
861

862
def run_command( name, way, cmd ):
863
    return simple_run( name, '', cmd, '' )
864

865 866 867 868
# -----------------------------------------------------------------------------
# GHCi tests

def ghci_script( name, way, script ):
869
    # filter out -fforce-recomp from compiler_always_flags, because we're
870
    # actually testing the recompilation behaviour in the GHCi tests.
871
    flags = filter(lambda f: f != '-fforce-recomp', getTestOpts().compiler_always_flags)