testlib.py 72.9 KB
Newer Older
1
#
2 3 4
# (c) Simon Marlow 2002
#

5 6 7
# This allows us to use the "with X:" syntax with python 2.5:
from __future__ import with_statement

8
import shutil
9 10
import sys
import os
11
import errno
12 13 14 15
import string
import re
import traceback
import copy
16
import glob
17
import types
ian@well-typed.com's avatar
ian@well-typed.com committed
18
from math import ceil, trunc
19

20 21 22 23 24 25 26
have_subprocess = False
try:
    import subprocess
    have_subprocess = True
except:
    print "Warning: subprocess not found, will fall back to spawnv"

27
from string import join
28
from testglobals import *
29 30
from testutil import *

31 32 33
if config.use_threads:
    import threading
    import thread
34

35 36 37 38 39 40 41 42
global wantToStop
wantToStop = False
def stopNow():
    global wantToStop
    wantToStop = True
def stopping():
    return wantToStop

43 44
# Options valid for the current test only (these get reset to
# testdir_testopts after each test).
45

ei@vuokko.info's avatar
ei@vuokko.info committed
46
global testopts_local
47 48 49 50 51 52
if config.use_threads:
    testopts_local = threading.local()
else:
    class TestOpts_Local:
        pass
    testopts_local = TestOpts_Local()
53 54

def getTestOpts():
ei@vuokko.info's avatar
ei@vuokko.info committed
55
    return testopts_local.x
56

ei@vuokko.info's avatar
ei@vuokko.info committed
57 58 59
def setLocalTestOpts(opts):
    global testopts_local
    testopts_local.x=opts
60 61 62 63

# This can be called at the top of a file of tests, to set default test options
# for the following tests.
def setTestOpts( f ):
64 65
    global thisdir_settings
    thisdir_settings = compose(thisdir_settings, f)
66 67 68 69 70 71 72 73 74 75 76 77

# -----------------------------------------------------------------------------
# Canned setup functions for common cases.  eg. for a test you might say
#
#      test('test001', normal, compile, [''])
#
# to run it without any options, but change it to
#
#      test('test001', expect_fail, compile, [''])
#
# to expect failure for this test.

78
def normal( name, opts ):
79 80
    return;

81
def skip( name, opts ):
82 83
    opts.skip = 1

84
def expect_fail( name, opts ):
85 86
    opts.expect = 'fail';

87
def reqlib( lib ):
88
    return lambda name, opts, l=lib: _reqlib (name, opts, l )
89

90 91 92 93
# Cache the results of looking to see if we have a library or not.
# This makes quite a difference, especially on Windows.
have_lib = {}

94
def _reqlib( name, opts, lib ):
95 96
    if have_lib.has_key(lib):
        got_it = have_lib[lib]
97
    else:
98 99 100
        if have_subprocess:
            # By preference we use subprocess, as the alternative uses
            # /dev/null which mingw doesn't have.
101
            p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
102 103 104 105 106 107 108 109 110 111 112 113 114
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
            # read from stdout and stderr to avoid blocking due to
            # buffers filling
            p.communicate()
            r = p.wait()
        else:
            r = os.system(config.ghc_pkg + ' describe ' + lib
                                         + ' > /dev/null 2> /dev/null')
        got_it = r == 0
        have_lib[lib] = got_it

    if not got_it:
115
        opts.expect = 'missing-lib'
116

117
def req_profiling( name, opts ):
118 119 120
    if not config.have_profiling:
        opts.expect = 'fail'

121
def req_shared_libs( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
122 123 124
    if not config.have_shared_libs:
        opts.expect = 'fail'

125
def req_interp( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
126 127 128
    if not config.have_interp:
        opts.expect = 'fail'

129
def req_smp( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
130 131 132
    if not config.have_smp:
        opts.expect = 'fail'

133
def ignore_output( name, opts ):
134 135
    opts.ignore_output = 1

136
def no_stdin( name, opts ):
137 138
    opts.no_stdin = 1

139
def combined_output( name, opts ):
pcapriotti's avatar
pcapriotti committed
140 141
    opts.combined_output = True

142 143 144
# -----

def expect_fail_for( ways ):
145
    return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
146

147
def _expect_fail_for( name, opts, ways ):
148 149
    opts.expect_fail_for = ways

150 151 152 153
def expect_broken( bug ):
    return lambda name, opts, b=bug: _expect_broken (name, opts, b )

def _expect_broken( name, opts, bug ):
154
    record_broken(name, bug)
155 156
    opts.expect = 'fail';

Ian Lynagh's avatar
Ian Lynagh committed
157
def expect_broken_for( bug, ways ):
158
    return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
Ian Lynagh's avatar
Ian Lynagh committed
159

160
def _expect_broken_for( name, opts, bug, ways ):
161
    record_broken(name, bug)
Ian Lynagh's avatar
Ian Lynagh committed
162
    opts.expect_fail_for = ways
Ian Lynagh's avatar
Ian Lynagh committed
163

164 165 166 167 168 169
def record_broken(name, bug):
    global brokens
    me = (bug, name)
    if not me in brokens:
        brokens.append(me)

170 171 172
# -----

def omit_ways( ways ):
173
    return lambda name, opts, w=ways: _omit_ways( name, opts, w )
174

175
def _omit_ways( name, opts, ways ):
176 177 178 179
    opts.omit_ways = ways

# -----

180
def only_ways( ways ):
181
    return lambda name, opts, w=ways: _only_ways( name, opts, w )
182

183
def _only_ways( name, opts, ways ):
184 185 186 187
    opts.only_ways = ways

# -----

188
def extra_ways( ways ):
189
    return lambda name, opts, w=ways: _extra_ways( name, opts, w )
190

191
def _extra_ways( name, opts, ways ):
192 193 194 195
    opts.extra_ways = ways

# -----

ross's avatar
ross committed
196
def omit_compiler_types( compiler_types ):
197
   return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
ross's avatar
ross committed
198

199
def _omit_compiler_types( name, opts, compiler_types ):
ross's avatar
ross committed
200
    if config.compiler_type in compiler_types:
dterei's avatar
dterei committed
201
        opts.skip = 1
ross's avatar
ross committed
202 203 204 205

# -----

def only_compiler_types( compiler_types ):
206
   return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
ross's avatar
ross committed
207

208
def _only_compiler_types( name, opts, compiler_types ):
ross's avatar
ross committed
209
    if config.compiler_type not in compiler_types:
dterei's avatar
dterei committed
210
        opts.skip = 1
ross's avatar
ross committed
211 212 213

# -----

214
def set_stdin( file ):
215
   return lambda name, opts, f=file: _set_stdin(name, opts, f);
216

217
def _set_stdin( name, opts, f ):
218 219 220 221 222
   opts.stdin = f

# -----

def exit_code( val ):
223
    return lambda name, opts, v=val: _exit_code(name, opts, v);
224

225
def _exit_code( name, opts, v ):
226 227 228 229
    opts.exit_code = v

# -----

230
def timeout_multiplier( val ):
231
    return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
232

233
def _timeout_multiplier( name, opts, v ):
234 235 236 237
    opts.timeout_multiplier = v

# -----

238
def extra_run_opts( val ):
239
    return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
240

241
def _extra_run_opts( name, opts, v ):
242 243
    opts.extra_run_opts = v

244 245
# -----

Simon Marlow's avatar
Simon Marlow committed
246
def extra_hc_opts( val ):
247
    return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
Simon Marlow's avatar
Simon Marlow committed
248

249
def _extra_hc_opts( name, opts, v ):
Simon Marlow's avatar
Simon Marlow committed
250 251 252 253
    opts.extra_hc_opts = v

# -----

254
def extra_clean( files ):
255
    return lambda name, opts, v=files: _extra_clean(name, opts, v);
256

257
def _extra_clean( name, opts, v ):
258 259
    opts.clean_files = v

260 261
# -----

262 263 264 265 266 267 268
def stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);

def _stats_num_field( name, opts, field, expecteds ):
    if field in opts.stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

269 270 271 272 273 274
    if type(expecteds) is types.ListType:
        for (b, expected, dev) in expecteds:
            if b:
                opts.stats_range_fields[field] = (expected, dev)
                return
        framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
275

276 277 278
    else:
        (expected, dev) = expecteds
        opts.stats_range_fields[field] = (expected, dev)
279 280 281

def compiler_stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
282

283 284 285 286 287 288 289 290
def _compiler_stats_num_field( name, opts, field, expecteds ):
    if field in opts.compiler_stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

    for (b, expected, dev) in expecteds:
        if b:
            opts.compiler_stats_range_fields[field] = (expected, dev)
            return
291

292 293
    framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')

294 295
# -----

296
def skip_if_no_ghci(name, opts):
dterei's avatar
dterei committed
297 298
    if not ('ghci' in config.run_ways):
        opts.skip = 1
299

300 301
# ----

302
def skip_if_fast(name, opts):
dterei's avatar
dterei committed
303 304
    if config.fast:
        opts.skip = 1
305

Simon Marlow's avatar
Simon Marlow committed
306 307
# -----

308
def when(b, f):
ian@well-typed.com's avatar
ian@well-typed.com committed
309 310 311
    # When list_brokens is on, we want to see all expect_broken calls,
    # so we always do f
    if b or config.list_broken:
312 313 314 315 316 317 318
        return f
    else:
        return normal

def unless(b, f):
    return when(not b, f)

319 320
def platform( plat ):
    return config.platform == plat
Ian Lynagh's avatar
Ian Lynagh committed
321

322 323
def opsys( os ):
    return config.os == os
Ian Lynagh's avatar
Ian Lynagh committed
324

325 326
def arch( arch ):
    return config.arch == arch
327

328 329
def wordsize( ws ):
    return config.wordsize == str(ws)
tibbe's avatar
tibbe committed
330

331 332
def unregisterised( ):
    return config.unregisterised
ian@well-typed.com's avatar
ian@well-typed.com committed
333

334 335
def msys( ):
    return config.msys
ian@well-typed.com's avatar
ian@well-typed.com committed
336

337 338
def cygwin( ):
    return config.cygwin
Ian Lynagh's avatar
Ian Lynagh committed
339

340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375
def when_have_vanilla( f ):
    if config.have_vanilla:
        return f
    else:
        return normal

def unless_have_vanilla( f ):
    if config.have_vanilla:
        return normal
    else:
        return f

def when_have_dynamic( f ):
    if config.have_dynamic:
        return f
    else:
        return normal

def unless_have_dynamic( f ):
    if config.have_dynamic:
        return normal
    else:
        return f

def when_have_profiling( f ):
    if config.have_profiling:
        return f
    else:
        return normal

def unless_have_profiling( f ):
    if config.have_profiling:
        return normal
    else:
        return f

376 377
# ---

378 379 380 381 382 383
def if_ghci_dynamic( f ):
    if config.ghc_dynamic_by_default:
        return f
    else:
        return normal

384 385 386 387 388 389 390 391 392 393 394 395
def if_in_tree_compiler( f ):
    if config.in_tree_compiler:
        return f
    else:
        return normal

def unless_in_tree_compiler( f ):
    if config.in_tree_compiler:
        return normal
    else:
        return f

396 397 398 399 400
def if_compiler_type( compiler, f ):
    if config.compiler_type == compiler:
        return f
    else:
        return normal
401 402 403 404 405 406

def if_compiler_profiled( f ):
    if config.compiler_profiled:
        return f
    else:
        return normal
407

408 409 410 411 412 413
def unless_compiler_profiled( f ):
    if config.compiler_profiled:
        return normal
    else:
        return f

Ian Lynagh's avatar
Ian Lynagh committed
414
def if_compiler_lt( compiler, version, f ):
415
    if config.compiler_type == compiler and \
Ian Lynagh's avatar
Ian Lynagh committed
416 417 418 419
       version_lt(config.compiler_version, version):
        return f
    else:
        return normal
420

Ian Lynagh's avatar
Ian Lynagh committed
421
def if_compiler_le( compiler, version, f ):
422 423
    if config.compiler_type == compiler and \
       version_le(config.compiler_version, version):
Ian Lynagh's avatar
Ian Lynagh committed
424 425 426
        return f
    else:
        return normal
427

Ian Lynagh's avatar
Ian Lynagh committed
428
def if_compiler_gt( compiler, version, f ):
429
    if config.compiler_type == compiler and \
Ian Lynagh's avatar
Ian Lynagh committed
430 431 432 433
       version_gt(config.compiler_version, version):
        return f
    else:
        return normal
434

Ian Lynagh's avatar
Ian Lynagh committed
435
def if_compiler_ge( compiler, version, f ):
436
    if config.compiler_type == compiler and \
Ian Lynagh's avatar
Ian Lynagh committed
437 438 439 440
       version_ge(config.compiler_version, version):
        return f
    else:
        return normal
441

442 443 444 445 446 447
def if_compiler_debugged( f ):
    if config.compiler_debugged:
        return f
    else:
        return normal

Ian Lynagh's avatar
Ian Lynagh committed
448 449
def namebase( nb ):
   return lambda opts, nb=nb: _namebase(opts, nb)
450

Ian Lynagh's avatar
Ian Lynagh committed
451 452
def _namebase( opts, nb ):
    opts.with_namebase = nb
453

454 455
# ---

456
def if_tag( tag, f ):
457
    if tag in config.compiler_tags:
458 459 460
        return f
    else:
        return normal
461

462
def unless_tag( tag, f ):
463
    if not (tag in config.compiler_tags):
464 465 466
        return f
    else:
        return normal
467

ei@vuokko.info's avatar
ei@vuokko.info committed
468
# ---
469
def high_memory_usage(name, opts):
470 471
    opts.alone = True

Ian Lynagh's avatar
Ian Lynagh committed
472
# ---
473
def literate( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
474 475
    opts.literate = 1;

476
def c_src( name, opts ):
477 478
    opts.c_src = 1;

479
def objc_src( name, opts ):
Austin Seipp's avatar
Austin Seipp committed
480 481
    opts.objc_src = 1;

482
def objcpp_src( name, opts ):
483 484
    opts.objcpp_src = 1;

485
def cmm_src( name, opts ):
486 487
    opts.cmm_src = 1;

488
def outputdir( odir ):
489
    return lambda name, opts, d=odir: _outputdir(name, opts, d)
490

491
def _outputdir( name, opts, odir ):
492 493
    opts.outputdir = odir;

494 495
# ----

496
def pre_cmd( cmd ):
497
    return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
498

499
def _pre_cmd( name, opts, cmd ):
500 501 502 503
    opts.pre_cmd = cmd

# ----

504
def clean_cmd( cmd ):
505
    return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
506

507
def _clean_cmd( name, opts, cmd ):
508 509 510 511
    opts.clean_cmd = cmd

# ----

512
def cmd_prefix( prefix ):
513
    return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
514

515
def _cmd_prefix( name, opts, prefix ):
516 517 518 519 520
    opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;

# ----

def cmd_wrapper( fun ):
521
    return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
522

523
def _cmd_wrapper( name, opts, fun ):
524
    opts.cmd_wrapper = fun
525

526 527
# ----

Ian Lynagh's avatar
Ian Lynagh committed
528
def compile_cmd_prefix( prefix ):
529
    return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
Ian Lynagh's avatar
Ian Lynagh committed
530

531
def _compile_cmd_prefix( name, opts, prefix ):
Ian Lynagh's avatar
Ian Lynagh committed
532 533 534 535
    opts.compile_cmd_prefix = prefix

# ----

536
def normalise_slashes( name, opts ):
537 538
    opts.extra_normaliser = normalise_slashes_

539
def normalise_exe( name, opts ):
540 541
    opts.extra_normaliser = normalise_exe_

542
def normalise_fun( fun ):
543
    return lambda name, opts, f=fun: _normalise_fun(name, opts, f)
544

545
def _normalise_fun( name, opts, f ):
546 547
    opts.extra_normaliser = f

548
def normalise_errmsg_fun( fun ):
549
    return lambda name, opts, f=fun: _normalise_errmsg_fun(name, opts, f)
550

551
def _normalise_errmsg_fun( name, opts, f ):
552 553 554 555 556
    opts.extra_errmsg_normaliser = f

def two_normalisers(f, g):
    return lambda x, f=f, g=g: f(g(x))

557 558 559
# ----
# Function for composing two opt-fns together

560 561 562
def composes( fs ):
    return reduce(lambda f, g: compose(f, g), fs)

563
def compose( f, g ):
564
    return lambda name, opts, f=f, g=g: _compose(name, opts, f, g)
565

566 567 568
def _compose( name, opts, f, g ):
    f(name, opts)
    g(name, opts)
569 570 571 572 573

# -----------------------------------------------------------------------------
# The current directory of tests

def newTestDir( dir ):
574
    global thisdir_settings
575
    # reset the options for this test directory
576 577 578 579 580
    thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )

def _newTestDir( name, opts, dir ):
    opts.testdir = dir
    opts.compiler_always_flags = config.compiler_always_flags
581 582 583 584

# -----------------------------------------------------------------------------
# Actually doing tests

585 586
parallelTests = []
aloneTests = []
587
allTestNames = set([])
588

589
def runTest (opts, name, func, args):
ei@vuokko.info's avatar
ei@vuokko.info committed
590 591
    ok = 0

592
    if config.use_threads:
ei@vuokko.info's avatar
ei@vuokko.info committed
593
        t.thread_pool.acquire()
594
        try:
595
            while config.threads<(t.running_threads+1):
596
                t.thread_pool.wait()
597
            t.running_threads = t.running_threads+1
598 599
            ok=1
            t.thread_pool.release()
600
            thread.start_new_thread(test_common_thread, (name, opts, func, args))
601 602 603 604 605
        except:
            if not ok:
                t.thread_pool.release()
    else:
        test_common_work (name, opts, func, args)
606

607
# name  :: String
608
# setup :: TestOpts -> IO ()
609
def test (name, setup, func, args):
610 611
    global aloneTests
    global parallelTests
612
    global allTestNames
613
    global thisdir_settings
614 615
    if name in allTestNames:
        framework_fail(name, 'duplicate', 'There are multiple tests with this name')
616
    if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
617
        framework_fail(name, 'bad_name', 'This test has an invalid name')
618 619 620 621 622

    # Make a deep copy of the default_testopts, as we need our own copy
    # of any dictionaries etc inside it. Otherwise, if one test modifies
    # them, all tests will see the modified version!
    myTestOpts = copy.deepcopy(default_testopts)
623 624 625 626

    if type(setup) is types.ListType:
       setup = composes(setup)

627 628
    setup = compose(thisdir_settings, setup)
    setup(name, myTestOpts)
629 630 631 632 633 634

    thisTest = lambda : runTest(myTestOpts, name, func, args)
    if myTestOpts.alone:
        aloneTests.append(thisTest)
    else:
        parallelTests.append(thisTest)
635
    allTestNames.add(name)
636

637
if config.use_threads:
638
    def test_common_thread(name, opts, func, args):
639 640 641 642 643 644
        t.lock.acquire()
        try:
            test_common_work(name,opts,func,args)
        finally:
            t.lock.release()
            t.thread_pool.acquire()
645
            t.running_threads = t.running_threads - 1
646 647
            t.thread_pool.notify()
            t.thread_pool.release()
648

649 650 651 652 653 654 655 656 657 658
def get_package_cache_timestamp():
    if config.package_conf_cache_file == '':
        return 0.0
    else:
        try:
            return os.stat(config.package_conf_cache_file).st_mtime
        except:
            return 0.0


ei@vuokko.info's avatar
ei@vuokko.info committed
659
def test_common_work (name, opts, func, args):
660 661 662 663 664 665 666 667 668
    try:
        t.total_tests = t.total_tests+1
        setLocalTestOpts(opts)

        package_conf_cache_file_start_timestamp = get_package_cache_timestamp()

        # All the ways we might run this test
        if func == compile or func == multimod_compile:
            all_ways = config.compile_ways
669
        elif func == compile_and_run or func == multimod_compile_and_run:
670 671 672 673 674 675
            all_ways = config.run_ways
        elif func == ghci_script:
            if 'ghci' in config.run_ways:
                all_ways = ['ghci']
            else:
                all_ways = []
676
        else:
677 678 679 680 681 682 683 684 685 686 687
            all_ways = ['normal']

        # A test itself can request extra ways by setting opts.extra_ways
        all_ways = all_ways + filter(lambda way: way not in all_ways,
                                     opts.extra_ways)

        t.total_test_cases = t.total_test_cases + len(all_ways)

        ok_way = lambda way: \
            not getTestOpts().skip \
            and (config.only == [] or name in config.only) \
688
            and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
689 690 691 692 693 694 695 696 697 698 699 700 701
            and (config.cmdline_ways == [] or way in config.cmdline_ways) \
            and way not in getTestOpts().omit_ways

        # Which ways we are asked to skip
        do_ways = filter (ok_way,all_ways)

        # In fast mode, we skip all but one way
        if config.fast and len(do_ways) > 0:
            do_ways = [do_ways[0]]

        if not config.clean_only:
            # Run the required tests...
            for way in do_ways:
702 703
                if stopping():
                    break
704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724
                do_test (name, way, func, args)

            for way in all_ways:
                if way not in do_ways:
                    skiptest (name,way)

        if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
            clean(map (lambda suff: name + suff,
                      ['', '.exe', '.exe.manifest', '.genscript',
                       '.stderr.normalised',        '.stdout.normalised',
                       '.run.stderr',               '.run.stdout',
                       '.run.stderr.normalised',    '.run.stdout.normalised',
                       '.comp.stderr',              '.comp.stdout',
                       '.comp.stderr.normalised',   '.comp.stdout.normalised',
                       '.interp.stderr',            '.interp.stdout',
                       '.interp.stderr.normalised', '.interp.stdout.normalised',
                       '.stats', '.comp.stats',
                       '.hi', '.o', '.prof', '.exe.prof', '.hc',
                       '_stub.h', '_stub.c', '_stub.o',
                       '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))

725
            if func == multi_compile or func == multi_compile_fail:
726 727 728 729 730
                    extra_mods = args[1]
                    clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
                    clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))

            clean(getTestOpts().clean_files)
Ian Lynagh's avatar
Ian Lynagh committed
731

732 733 734 735 736 737 738
            if getTestOpts().outputdir != None:
                odir = in_testdir(getTestOpts().outputdir)
                try:
                    shutil.rmtree(odir)
                except:
                    pass

739 740 741 742 743
            try:
                shutil.rmtree(in_testdir('.hpc.' + name))
            except:
                pass

744 745 746 747 748 749 750 751
            try:
                cleanCmd = getTestOpts().clean_cmd
                if cleanCmd != None:
                    result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
                    if result != 0:
                        framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
            except e:
                framework_fail(name, 'cleaning', 'clean-command exception')
752

753
        package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
754

755 756
        if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
            framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
757

758 759 760 761 762 763 764 765 766 767 768 769
        try:
            for f in files_written[name]:
                if os.path.exists(f):
                    try:
                        if not f in files_written_not_removed[name]:
                            files_written_not_removed[name].append(f)
                    except:
                        files_written_not_removed[name] = [f]
        except:
            pass
    except Exception, e:
        framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
Ian Lynagh's avatar
Ian Lynagh committed
770

771 772 773 774
def clean(strs):
    for str in strs:
        for name in glob.glob(in_testdir(str)):
            clean_full_path(name)
775

776
def clean_full_path(name):
Ian Lynagh's avatar
Ian Lynagh committed
777 778 779 780
        try:
            # Remove files...
            os.remove(name)
        except OSError, e1:
781
            try:
Ian Lynagh's avatar
Ian Lynagh committed
782 783 784 785 786 787 788 789 790 791 792
                # ... and empty directories
                os.rmdir(name)
            except OSError, e2:
                # We don't want to fail here, but we do want to know
                # what went wrong, so print out the exceptions.
                # ENOENT isn't a problem, though, as we clean files
                # that don't necessarily exist.
                if e1.errno != errno.ENOENT:
                    print e1
                if e2.errno != errno.ENOENT:
                    print e2
793

794
def do_test(name, way, func, args):
795 796 797
    full_name = name + '(' + way + ')'

    try:
798
        print '=====>', full_name, t.total_tests, 'of', len(allTestNames), \
799 800 801
                        str([t.n_unexpected_passes,   \
                             t.n_unexpected_failures, \
                             t.n_framework_failures])
802

803 804
        if config.use_threads:
            t.lock.release()
805 806 807 808

        try:
            preCmd = getTestOpts().pre_cmd
            if preCmd != None:
809
                result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
810 811 812 813 814
                if result != 0:
                    framework_fail(name, way, 'pre-command failed: ' + str(result))
        except e:
            framework_fail(name, way, 'pre-command exception')

ei@vuokko.info's avatar
ei@vuokko.info committed
815 816 817
        try:
            result = apply(func, [name,way] + args)
        finally:
818 819
            if config.use_threads:
                t.lock.acquire()
820

821 822 823
        if getTestOpts().expect != 'pass' and \
                getTestOpts().expect != 'fail' and \
                getTestOpts().expect != 'missing-lib':
824
            framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
825

826 827 828 829 830 831
        try:
            passFail = result['passFail']
        except:
            passFail = 'No passFail found'

        if passFail == 'pass':
ei@vuokko.info's avatar
ei@vuokko.info committed
832 833
            if getTestOpts().expect == 'pass' \
               and way not in getTestOpts().expect_fail_for:
834
                t.n_expected_passes = t.n_expected_passes + 1
835 836 837 838
                if name in t.expected_passes:
                    t.expected_passes[name].append(way)
                else:
                    t.expected_passes[name] = [way]
839 840 841
            else:
                print '*** unexpected pass for', full_name
                t.n_unexpected_passes = t.n_unexpected_passes + 1
842
                addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
843
        elif passFail == 'fail':
ei@vuokko.info's avatar
ei@vuokko.info committed
844 845
            if getTestOpts().expect == 'pass' \
               and way not in getTestOpts().expect_fail_for:
846 847
                print '*** unexpected failure for', full_name
                t.n_unexpected_failures = t.n_unexpected_failures + 1
848 849
                reason = result['reason']
                addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
850
            else:
851 852 853 854 855 856
                if getTestOpts().expect == 'missing-lib':
                    t.n_missing_libs = t.n_missing_libs + 1
                    if name in t.missing_libs:
                        t.missing_libs[name].append(way)
                    else:
                        t.missing_libs[name] = [way]
857
                else:
858 859 860 861 862
                    t.n_expected_failures = t.n_expected_failures + 1
                    if name in t.expected_failures:
                        t.expected_failures[name].append(way)
                    else:
                        t.expected_failures[name] = [way]
863 864
        else:
            framework_fail(name, way, 'bad result ' + passFail)
865
    except KeyboardInterrupt:
866
        stopNow()
867
    except:
868
        framework_fail(name, way, 'do_test exception')
869 870
        traceback.print_exc()

871
def addPassingTestInfo (testInfos, directory, name, way):
872 873 874 875 876 877 878 879 880 881
    directory = re.sub('^\\.[/\\\\]', '', directory)

    if not directory in testInfos:
        testInfos[directory] = {}

    if not name in testInfos[directory]:
        testInfos[directory][name] = []

    testInfos[directory][name].append(way)

882 883 884 885 886 887 888 889 890 891 892 893 894 895
def addFailingTestInfo (testInfos, directory, name, reason, way):
    directory = re.sub('^\\.[/\\\\]', '', directory)

    if not directory in testInfos:
        testInfos[directory] = {}

    if not name in testInfos[directory]:
        testInfos[directory][name] = {}

    if not reason in testInfos[directory][name]:
        testInfos[directory][name][reason] = []

    testInfos[directory][name][reason].append(way)

896
def skiptest (name, way):
897 898
    # print 'Skipping test \"', name, '\"'
    t.n_tests_skipped = t.n_tests_skipped + 1
899 900 901 902
    if name in t.tests_skipped:
        t.tests_skipped[name].append(way)
    else:
        t.tests_skipped[name] = [way]
903

904 905
def framework_fail( name, way, reason ):
    full_name = name + '(' + way + ')'
906
    print '*** framework failure for', full_name, reason
simonmar's avatar