testlib.py 72.2 KB
Newer Older
1
#
2 3 4
# (c) Simon Marlow 2002
#

5 6 7
# This allows us to use the "with X:" syntax with python 2.5:
from __future__ import with_statement

8
import shutil
9 10
import sys
import os
11
import errno
12 13 14
import string
import re
import traceback
15 16
import time
import datetime
17
import copy
18
import glob
19
import types
ian@well-typed.com's avatar
ian@well-typed.com committed
20
from math import ceil, trunc
21

22 23 24 25 26 27 28
have_subprocess = False
try:
    import subprocess
    have_subprocess = True
except:
    print "Warning: subprocess not found, will fall back to spawnv"

29
from string import join
30
from testglobals import *
31 32
from testutil import *

33 34 35
if config.use_threads:
    import threading
    import thread
36

37 38 39 40 41 42 43 44
global wantToStop
wantToStop = False
def stopNow():
    global wantToStop
    wantToStop = True
def stopping():
    return wantToStop

45 46
# Options valid for the current test only (these get reset to
# testdir_testopts after each test).
47

ei@vuokko.info's avatar
ei@vuokko.info committed
48
global testopts_local
49 50 51 52 53 54
if config.use_threads:
    testopts_local = threading.local()
else:
    class TestOpts_Local:
        pass
    testopts_local = TestOpts_Local()
55 56

def getTestOpts():
ei@vuokko.info's avatar
ei@vuokko.info committed
57
    return testopts_local.x
58

ei@vuokko.info's avatar
ei@vuokko.info committed
59 60 61
def setLocalTestOpts(opts):
    global testopts_local
    testopts_local.x=opts
62

63 64 65 66 67
def isStatsTest():
    opts = getTestOpts()
    return len(opts.compiler_stats_range_fields) > 0 or len(opts.stats_range_fields) > 0


68 69 70
# This can be called at the top of a file of tests, to set default test options
# for the following tests.
def setTestOpts( f ):
71
    global thisdir_settings
72
    thisdir_settings = [thisdir_settings, f]
73 74 75 76 77 78 79 80 81 82 83 84

# -----------------------------------------------------------------------------
# Canned setup functions for common cases.  eg. for a test you might say
#
#      test('test001', normal, compile, [''])
#
# to run it without any options, but change it to
#
#      test('test001', expect_fail, compile, [''])
#
# to expect failure for this test.

85
def normal( name, opts ):
86 87
    return;

88
def skip( name, opts ):
89 90
    opts.skip = 1

91
def expect_fail( name, opts ):
92 93
    opts.expect = 'fail';

94
def reqlib( lib ):
95
    return lambda name, opts, l=lib: _reqlib (name, opts, l )
96

97 98 99 100
# Cache the results of looking to see if we have a library or not.
# This makes quite a difference, especially on Windows.
have_lib = {}

101
def _reqlib( name, opts, lib ):
102 103
    if have_lib.has_key(lib):
        got_it = have_lib[lib]
104
    else:
105 106 107
        if have_subprocess:
            # By preference we use subprocess, as the alternative uses
            # /dev/null which mingw doesn't have.
108
            p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
109 110 111 112 113 114 115 116 117 118 119 120 121
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
            # read from stdout and stderr to avoid blocking due to
            # buffers filling
            p.communicate()
            r = p.wait()
        else:
            r = os.system(config.ghc_pkg + ' describe ' + lib
                                         + ' > /dev/null 2> /dev/null')
        got_it = r == 0
        have_lib[lib] = got_it

    if not got_it:
122
        opts.expect = 'missing-lib'
123

124
def req_profiling( name, opts ):
125 126 127
    if not config.have_profiling:
        opts.expect = 'fail'

128
def req_shared_libs( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
129 130 131
    if not config.have_shared_libs:
        opts.expect = 'fail'

132
def req_interp( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
133 134 135
    if not config.have_interp:
        opts.expect = 'fail'

136
def req_smp( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
137 138 139
    if not config.have_smp:
        opts.expect = 'fail'

140
def ignore_output( name, opts ):
141 142
    opts.ignore_output = 1

143
def no_stdin( name, opts ):
144 145
    opts.no_stdin = 1

146
def combined_output( name, opts ):
pcapriotti's avatar
pcapriotti committed
147 148
    opts.combined_output = True

149 150 151
# -----

def expect_fail_for( ways ):
152
    return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
153

154
def _expect_fail_for( name, opts, ways ):
155 156
    opts.expect_fail_for = ways

157 158 159 160
def expect_broken( bug ):
    return lambda name, opts, b=bug: _expect_broken (name, opts, b )

def _expect_broken( name, opts, bug ):
161
    record_broken(name, opts, bug)
162 163
    opts.expect = 'fail';

Ian Lynagh's avatar
Ian Lynagh committed
164
def expect_broken_for( bug, ways ):
165
    return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
Ian Lynagh's avatar
Ian Lynagh committed
166

167
def _expect_broken_for( name, opts, bug, ways ):
168
    record_broken(name, opts, bug)
Ian Lynagh's avatar
Ian Lynagh committed
169
    opts.expect_fail_for = ways
Ian Lynagh's avatar
Ian Lynagh committed
170

171
def record_broken(name, opts, bug):
172
    global brokens
173
    me = (bug, opts.testdir, name)
174 175 176
    if not me in brokens:
        brokens.append(me)

177 178 179
# -----

def omit_ways( ways ):
180
    return lambda name, opts, w=ways: _omit_ways( name, opts, w )
181

182
def _omit_ways( name, opts, ways ):
183 184 185 186
    opts.omit_ways = ways

# -----

187
def only_ways( ways ):
188
    return lambda name, opts, w=ways: _only_ways( name, opts, w )
189

190
def _only_ways( name, opts, ways ):
191 192 193 194
    opts.only_ways = ways

# -----

195
def extra_ways( ways ):
196
    return lambda name, opts, w=ways: _extra_ways( name, opts, w )
197

198
def _extra_ways( name, opts, ways ):
199 200 201 202
    opts.extra_ways = ways

# -----

ross's avatar
ross committed
203
def omit_compiler_types( compiler_types ):
204
   return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
ross's avatar
ross committed
205

206
def _omit_compiler_types( name, opts, compiler_types ):
ross's avatar
ross committed
207
    if config.compiler_type in compiler_types:
dterei's avatar
dterei committed
208
        opts.skip = 1
ross's avatar
ross committed
209 210 211 212

# -----

def only_compiler_types( compiler_types ):
213
   return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
ross's avatar
ross committed
214

215
def _only_compiler_types( name, opts, compiler_types ):
ross's avatar
ross committed
216
    if config.compiler_type not in compiler_types:
dterei's avatar
dterei committed
217
        opts.skip = 1
ross's avatar
ross committed
218 219 220

# -----

221
def set_stdin( file ):
222
   return lambda name, opts, f=file: _set_stdin(name, opts, f);
223

224
def _set_stdin( name, opts, f ):
225 226 227 228 229
   opts.stdin = f

# -----

def exit_code( val ):
230
    return lambda name, opts, v=val: _exit_code(name, opts, v);
231

232
def _exit_code( name, opts, v ):
233 234 235 236
    opts.exit_code = v

# -----

237
def timeout_multiplier( val ):
238
    return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
239

240
def _timeout_multiplier( name, opts, v ):
241 242 243 244
    opts.timeout_multiplier = v

# -----

245
def extra_run_opts( val ):
246
    return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
247

248
def _extra_run_opts( name, opts, v ):
249 250
    opts.extra_run_opts = v

251 252
# -----

Simon Marlow's avatar
Simon Marlow committed
253
def extra_hc_opts( val ):
254
    return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
Simon Marlow's avatar
Simon Marlow committed
255

256
def _extra_hc_opts( name, opts, v ):
Simon Marlow's avatar
Simon Marlow committed
257 258 259 260
    opts.extra_hc_opts = v

# -----

261
def extra_clean( files ):
262
    return lambda name, opts, v=files: _extra_clean(name, opts, v);
263

264
def _extra_clean( name, opts, v ):
265 266
    opts.clean_files = v

267 268
# -----

269 270 271 272 273 274 275
def stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);

def _stats_num_field( name, opts, field, expecteds ):
    if field in opts.stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

276 277 278 279 280 281
    if type(expecteds) is types.ListType:
        for (b, expected, dev) in expecteds:
            if b:
                opts.stats_range_fields[field] = (expected, dev)
                return
        framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
282

283 284 285
    else:
        (expected, dev) = expecteds
        opts.stats_range_fields[field] = (expected, dev)
286 287 288

def compiler_stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
289

290 291 292 293
def _compiler_stats_num_field( name, opts, field, expecteds ):
    if field in opts.compiler_stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

294 295 296
    if compiler_debugged():
        skip(name, opts)

297 298 299 300
    for (b, expected, dev) in expecteds:
        if b:
            opts.compiler_stats_range_fields[field] = (expected, dev)
            return
301

302 303
    framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')

304 305
# -----

306
def when(b, f):
ian@well-typed.com's avatar
ian@well-typed.com committed
307 308 309
    # When list_brokens is on, we want to see all expect_broken calls,
    # so we always do f
    if b or config.list_broken:
310 311 312 313 314 315 316
        return f
    else:
        return normal

def unless(b, f):
    return when(not b, f)

ian@well-typed.com's avatar
ian@well-typed.com committed
317 318 319
def doing_ghci():
    return 'ghci' in config.run_ways

320
def ghci_dynamic( ):
ian@well-typed.com's avatar
ian@well-typed.com committed
321
    return config.ghc_dynamic
322 323 324 325

def fast():
    return config.fast

326 327
def platform( plat ):
    return config.platform == plat
Ian Lynagh's avatar
Ian Lynagh committed
328

329 330
def opsys( os ):
    return config.os == os
Ian Lynagh's avatar
Ian Lynagh committed
331

332 333
def arch( arch ):
    return config.arch == arch
334

335 336
def wordsize( ws ):
    return config.wordsize == str(ws)
tibbe's avatar
tibbe committed
337

338 339
def msys( ):
    return config.msys
ian@well-typed.com's avatar
ian@well-typed.com committed
340

341 342
def cygwin( ):
    return config.cygwin
Ian Lynagh's avatar
Ian Lynagh committed
343

344 345
def have_vanilla( ):
    return config.have_vanilla
346

347 348
def have_dynamic( ):
    return config.have_dynamic
349

350 351
def have_profiling( ):
    return config.have_profiling
352

353 354
def in_tree_compiler( ):
    return config.in_tree_compiler
355

356 357
def compiler_type( compiler ):
    return config.compiler_type == compiler
358

359 360 361
def compiler_lt( compiler, version ):
    return config.compiler_type == compiler and \
           version_lt(config.compiler_version, version)
362

363 364 365
def compiler_le( compiler, version ):
    return config.compiler_type == compiler and \
           version_le(config.compiler_version, version)
366

367 368 369
def compiler_gt( compiler, version ):
    return config.compiler_type == compiler and \
           version_gt(config.compiler_version, version)
370

371 372 373
def compiler_ge( compiler, version ):
    return config.compiler_type == compiler and \
           version_ge(config.compiler_version, version)
374

375 376 377 378 379 380
def unregisterised( ):
    return config.unregisterised

def compiler_profiled( ):
    return config.compiler_profiled

381 382
def compiler_debugged( ):
    return config.compiler_debugged
383

384 385 386 387 388
def tag( t ):
    return t in config.compiler_tags

# ---

Ian Lynagh's avatar
Ian Lynagh committed
389 390
def namebase( nb ):
   return lambda opts, nb=nb: _namebase(opts, nb)
391

Ian Lynagh's avatar
Ian Lynagh committed
392 393
def _namebase( opts, nb ):
    opts.with_namebase = nb
394

395 396
# ---

397
def high_memory_usage(name, opts):
398 399
    opts.alone = True

400 401 402 403 404
# If a test is for a multi-CPU race, then running the test alone
# increases the chance that we'll actually see it.
def multi_cpu_race(name, opts):
    opts.alone = True

Ian Lynagh's avatar
Ian Lynagh committed
405
# ---
406
def literate( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
407 408
    opts.literate = 1;

409
def c_src( name, opts ):
410 411
    opts.c_src = 1;

412
def objc_src( name, opts ):
Austin Seipp's avatar
Austin Seipp committed
413 414
    opts.objc_src = 1;

415
def objcpp_src( name, opts ):
416 417
    opts.objcpp_src = 1;

418
def cmm_src( name, opts ):
419 420
    opts.cmm_src = 1;

421
def outputdir( odir ):
422
    return lambda name, opts, d=odir: _outputdir(name, opts, d)
423

424
def _outputdir( name, opts, odir ):
425 426
    opts.outputdir = odir;

427 428
# ----

429
def pre_cmd( cmd ):
430
    return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
431

432
def _pre_cmd( name, opts, cmd ):
433 434 435 436
    opts.pre_cmd = cmd

# ----

437
def clean_cmd( cmd ):
438
    return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
439

440
def _clean_cmd( name, opts, cmd ):
441 442 443 444
    opts.clean_cmd = cmd

# ----

445
def cmd_prefix( prefix ):
446
    return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
447

448
def _cmd_prefix( name, opts, prefix ):
449 450 451 452 453
    opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;

# ----

def cmd_wrapper( fun ):
454
    return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
455

456
def _cmd_wrapper( name, opts, fun ):
457
    opts.cmd_wrapper = fun
458

459 460
# ----

Ian Lynagh's avatar
Ian Lynagh committed
461
def compile_cmd_prefix( prefix ):
462
    return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
Ian Lynagh's avatar
Ian Lynagh committed
463

464
def _compile_cmd_prefix( name, opts, prefix ):
Ian Lynagh's avatar
Ian Lynagh committed
465 466 467 468
    opts.compile_cmd_prefix = prefix

# ----

469
def normalise_slashes( name, opts ):
470 471
    opts.extra_normaliser = normalise_slashes_

472
def normalise_exe( name, opts ):
473 474
    opts.extra_normaliser = normalise_exe_

475
def normalise_fun( fun ):
476
    return lambda name, opts, f=fun: _normalise_fun(name, opts, f)
477

478
def _normalise_fun( name, opts, f ):
479 480
    opts.extra_normaliser = f

481
def normalise_errmsg_fun( fun ):
482
    return lambda name, opts, f=fun: _normalise_errmsg_fun(name, opts, f)
483

484
def _normalise_errmsg_fun( name, opts, f ):
485 486 487 488 489
    opts.extra_errmsg_normaliser = f

def two_normalisers(f, g):
    return lambda x, f=f, g=g: f(g(x))

490 491 492
# ----
# Function for composing two opt-fns together

493 494 495 496 497 498 499
def executeSetups(fs, name, opts):
    if type(fs) is types.ListType:
        # If we have a list of setups, then execute each one
        map (lambda f : executeSetups(f, name, opts), fs)
    else:
        # fs is a single function, so just apply it
        fs(name, opts)
500

501 502 503 504
# -----------------------------------------------------------------------------
# The current directory of tests

def newTestDir( dir ):
505
    global thisdir_settings
506
    # reset the options for this test directory
507 508 509 510 511
    thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )

def _newTestDir( name, opts, dir ):
    opts.testdir = dir
    opts.compiler_always_flags = config.compiler_always_flags
512 513 514 515

# -----------------------------------------------------------------------------
# Actually doing tests

516 517
parallelTests = []
aloneTests = []
518
allTestNames = set([])
519

520
def runTest (opts, name, func, args):
ei@vuokko.info's avatar
ei@vuokko.info committed
521 522
    ok = 0

523
    if config.use_threads:
ei@vuokko.info's avatar
ei@vuokko.info committed
524
        t.thread_pool.acquire()
525
        try:
526
            while config.threads<(t.running_threads+1):
527
                t.thread_pool.wait()
528
            t.running_threads = t.running_threads+1
529 530
            ok=1
            t.thread_pool.release()
531
            thread.start_new_thread(test_common_thread, (name, opts, func, args))
532 533 534 535 536
        except:
            if not ok:
                t.thread_pool.release()
    else:
        test_common_work (name, opts, func, args)
537

538
# name  :: String
539
# setup :: TestOpts -> IO ()
540
def test (name, setup, func, args):
541 542
    global aloneTests
    global parallelTests
543
    global allTestNames
544
    global thisdir_settings
545 546
    if name in allTestNames:
        framework_fail(name, 'duplicate', 'There are multiple tests with this name')
547
    if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
548
        framework_fail(name, 'bad_name', 'This test has an invalid name')
549 550 551 552 553

    # Make a deep copy of the default_testopts, as we need our own copy
    # of any dictionaries etc inside it. Otherwise, if one test modifies
    # them, all tests will see the modified version!
    myTestOpts = copy.deepcopy(default_testopts)
554

555
    executeSetups([thisdir_settings, setup], name, myTestOpts)
556 557 558 559 560 561

    thisTest = lambda : runTest(myTestOpts, name, func, args)
    if myTestOpts.alone:
        aloneTests.append(thisTest)
    else:
        parallelTests.append(thisTest)
562
    allTestNames.add(name)
563

564
if config.use_threads:
565
    def test_common_thread(name, opts, func, args):
566 567 568 569 570 571
        t.lock.acquire()
        try:
            test_common_work(name,opts,func,args)
        finally:
            t.lock.release()
            t.thread_pool.acquire()
572
            t.running_threads = t.running_threads - 1
573 574
            t.thread_pool.notify()
            t.thread_pool.release()
575

576 577 578 579 580 581 582 583 584 585
def get_package_cache_timestamp():
    if config.package_conf_cache_file == '':
        return 0.0
    else:
        try:
            return os.stat(config.package_conf_cache_file).st_mtime
        except:
            return 0.0


ei@vuokko.info's avatar
ei@vuokko.info committed
586
def test_common_work (name, opts, func, args):
587 588 589 590 591 592 593 594 595
    try:
        t.total_tests = t.total_tests+1
        setLocalTestOpts(opts)

        package_conf_cache_file_start_timestamp = get_package_cache_timestamp()

        # All the ways we might run this test
        if func == compile or func == multimod_compile:
            all_ways = config.compile_ways
596
        elif func == compile_and_run or func == multimod_compile_and_run:
597 598 599 600 601 602
            all_ways = config.run_ways
        elif func == ghci_script:
            if 'ghci' in config.run_ways:
                all_ways = ['ghci']
            else:
                all_ways = []
603
        else:
604 605 606 607 608 609 610 611 612 613 614
            all_ways = ['normal']

        # A test itself can request extra ways by setting opts.extra_ways
        all_ways = all_ways + filter(lambda way: way not in all_ways,
                                     opts.extra_ways)

        t.total_test_cases = t.total_test_cases + len(all_ways)

        ok_way = lambda way: \
            not getTestOpts().skip \
            and (config.only == [] or name in config.only) \
615
            and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
616
            and (config.cmdline_ways == [] or way in config.cmdline_ways) \
617
            and (not (config.skip_perf_tests and isStatsTest())) \
618 619 620 621 622 623 624 625 626 627 628 629
            and way not in getTestOpts().omit_ways

        # Which ways we are asked to skip
        do_ways = filter (ok_way,all_ways)

        # In fast mode, we skip all but one way
        if config.fast and len(do_ways) > 0:
            do_ways = [do_ways[0]]

        if not config.clean_only:
            # Run the required tests...
            for way in do_ways:
630 631
                if stopping():
                    break
632 633 634 635 636 637 638
                do_test (name, way, func, args)

            for way in all_ways:
                if way not in do_ways:
                    skiptest (name,way)

        if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
639
            pretest_cleanup(name)
640 641 642 643 644 645 646 647 648 649 650
            clean(map (lambda suff: name + suff,
                      ['', '.exe', '.exe.manifest', '.genscript',
                       '.stderr.normalised',        '.stdout.normalised',
                       '.run.stderr.normalised',    '.run.stdout.normalised',
                       '.comp.stderr.normalised',   '.comp.stdout.normalised',
                       '.interp.stderr.normalised', '.interp.stdout.normalised',
                       '.stats', '.comp.stats',
                       '.hi', '.o', '.prof', '.exe.prof', '.hc',
                       '_stub.h', '_stub.c', '_stub.o',
                       '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))

651
            if func == multi_compile or func == multi_compile_fail:
652 653 654 655 656
                    extra_mods = args[1]
                    clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
                    clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))

            clean(getTestOpts().clean_files)
Ian Lynagh's avatar
Ian Lynagh committed
657

658 659 660 661 662 663 664
            if getTestOpts().outputdir != None:
                odir = in_testdir(getTestOpts().outputdir)
                try:
                    shutil.rmtree(odir)
                except:
                    pass

665 666 667 668 669
            try:
                shutil.rmtree(in_testdir('.hpc.' + name))
            except:
                pass

670 671 672 673 674 675 676 677
            try:
                cleanCmd = getTestOpts().clean_cmd
                if cleanCmd != None:
                    result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
                    if result != 0:
                        framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
            except e:
                framework_fail(name, 'cleaning', 'clean-command exception')
678

679
        package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
680

681 682
        if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
            framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
683

684 685 686 687 688 689 690 691 692 693 694 695
        try:
            for f in files_written[name]:
                if os.path.exists(f):
                    try:
                        if not f in files_written_not_removed[name]:
                            files_written_not_removed[name].append(f)
                    except:
                        files_written_not_removed[name] = [f]
        except:
            pass
    except Exception, e:
        framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
Ian Lynagh's avatar
Ian Lynagh committed
696

697 698 699 700
def clean(strs):
    for str in strs:
        for name in glob.glob(in_testdir(str)):
            clean_full_path(name)
701

702
def clean_full_path(name):
Ian Lynagh's avatar
Ian Lynagh committed
703 704 705 706
        try:
            # Remove files...
            os.remove(name)
        except OSError, e1:
707
            try:
Ian Lynagh's avatar
Ian Lynagh committed
708 709 710 711 712 713 714 715 716 717 718
                # ... and empty directories
                os.rmdir(name)
            except OSError, e2:
                # We don't want to fail here, but we do want to know
                # what went wrong, so print out the exceptions.
                # ENOENT isn't a problem, though, as we clean files
                # that don't necessarily exist.
                if e1.errno != errno.ENOENT:
                    print e1
                if e2.errno != errno.ENOENT:
                    print e2
719

720
def do_test(name, way, func, args):
721 722 723
    full_name = name + '(' + way + ')'

    try:
724 725 726 727 728
        if_verbose(2, "=====> %s %d of %d %s " % \
                    (full_name, t.total_tests, len(allTestNames), \
                    [t.n_unexpected_passes, \
                     t.n_unexpected_failures, \
                     t.n_framework_failures]))
729

730 731
        if config.use_threads:
            t.lock.release()
732 733 734 735

        try:
            preCmd = getTestOpts().pre_cmd
            if preCmd != None:
736
                result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
737 738 739 740 741
                if result != 0:
                    framework_fail(name, way, 'pre-command failed: ' + str(result))
        except e:
            framework_fail(name, way, 'pre-command exception')

ei@vuokko.info's avatar
ei@vuokko.info committed
742 743 744
        try:
            result = apply(func, [name,way] + args)
        finally:
745 746
            if config.use_threads:
                t.lock.acquire()
747

748 749 750
        if getTestOpts().expect != 'pass' and \
                getTestOpts().expect != 'fail' and \
                getTestOpts().expect != 'missing-lib':
751
            framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
752

753 754 755 756 757 758
        try:
            passFail = result['passFail']
        except:
            passFail = 'No passFail found'

        if passFail == 'pass':
ei@vuokko.info's avatar
ei@vuokko.info committed
759 760
            if getTestOpts().expect == 'pass' \
               and way not in getTestOpts().expect_fail_for:
761
                t.n_expected_passes = t.n_expected_passes + 1
762 763 764 765
                if name in t.expected_passes:
                    t.expected_passes[name].append(way)
                else:
                    t.expected_passes[name] = [way]
766
            else:
767
                if_verbose(1, '*** unexpected pass for %s' % full_name)
768
                t.n_unexpected_passes = t.n_unexpected_passes + 1
769
                addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
770
        elif passFail == 'fail':
ei@vuokko.info's avatar
ei@vuokko.info committed
771 772
            if getTestOpts().expect == 'pass' \
               and way not in getTestOpts().expect_fail_for:
773
                if_verbose(1, '*** unexpected failure for %s' % full_name)
774
                t.n_unexpected_failures = t.n_unexpected_failures + 1
775 776
                reason = result['reason']
                addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
777
            else:
778 779 780 781 782 783
                if getTestOpts().expect == 'missing-lib':
                    t.n_missing_libs = t.n_missing_libs + 1
                    if name in t.missing_libs:
                        t.missing_libs[name].append(way)
                    else:
                        t.missing_libs[name] = [way]
784
                else:
785 786 787 788 789
                    t.n_expected_failures = t.n_expected_failures + 1
                    if name in t.expected_failures:
                        t.expected_failures[name].append(way)
                    else:
                        t.expected_failures[name] = [way]
790 791
        else:
            framework_fail(name, way, 'bad result ' + passFail)
792
    except KeyboardInterrupt:
793
        stopNow()
794
    except:
795
        framework_fail(name, way, 'do_test exception')
796 797
        traceback.print_exc()

798
def addPassingTestInfo (testInfos, directory, name, way):
799 800 801 802 803 804 805 806 807 808
    directory = re.sub('^\\.[/\\\\]', '', directory)

    if not directory in testInfos:
        testInfos[directory] = {}

    if not name in testInfos[directory]:
        testInfos[directory][name] = []

    testInfos[directory][name].append(way)

809 810 811 812 813 814 815 816 817 818 819 820 821 822
def addFailingTestInfo (testInfos, directory, name, reason, way):
    directory = re.sub('^\\.[/\\\\]', '', directory)

    if not directory in testInfos:
        testInfos[directory] = {}

    if not name in testInfos[directory]:
        testInfos[directory][name] = {}

    if not reason in testInfos[directory][name]:
        testInfos[directory][name][reason] = []

    testInfos[directory][name][reason].append(way)

823
def skiptest (name, way):
824 825
    # print 'Skipping test \"', name, '\"'
    t.n_tests_skipped = t.n_tests_skipped + 1
826 827 828 829
    if name in t.tests_skipped:
        t.tests_skipped[name].append(way)
    else:
        t.tests_skipped[name] = [way]
830

831 832
def framework_fail( name, way, reason ):
    full_name = name + '(' + way + ')'
Joachim Breitner's avatar
Joachim Breitner committed
833
    if_verbose(1, '*** framework failure for %s %s ' % (full_name, reason))
834
    t.n_framework_failures = t.n_framework_failures + 1
835 836 837 838
    if name in t.framework_failures:
        t.framework_failures[name].append(way)
    else:
        t.framework_failures[name] = [way]
839

840 841 842 843 844 845 846 847 848 849 850 851 852 853
def badResult(result):
    try:
        if result['passFail'] == 'pass':
            return False
        return True
    except:
        return True

def passed():
    return {'passFail': 'pass'}

def failBecause(reason):
    return {'passFail': 'fail', 'reason': reason}

854 855 856 857
# -----------------------------------------------------------------------------
# Generic command tests

# A generic command test is expected to run and exit successfully.
858 859 860 861 862 863
#
# The expected exit code can be changed via exit_code() as normal, and
# the expected stdout/stderr are stored in <testname>.stdout and
# <testname>.stderr.  The output of the command can be ignored
# altogether by using run_command_ignore_output instead of
# run_command.
864

865
def run_command( name, way, cmd ):
866
    return simple_run( name, '', cmd, '' )
867

868 869 870 871
# -----------------------------------------------------------------------------
# GHCi tests

def ghci_script( name, way, script ):
872
    # filter out -fforce-recomp from compiler_always_flags, because we're