testlib.py 61.4 KB
Newer Older
1
#
2 3 4 5 6
# (c) Simon Marlow 2002
#

import sys
import os
7
import errno
8 9 10 11
import string
import re
import traceback
import copy
12
import glob
13
import types
14

15 16 17 18 19 20 21
have_subprocess = False
try:
    import subprocess
    have_subprocess = True
except:
    print "Warning: subprocess not found, will fall back to spawnv"

22
from string import join
23
from testglobals import *
24 25
from testutil import *

26 27 28
if config.use_threads:
    import threading
    import thread
29 30 31 32 33 34 35 36 37 38 39 40 41

# Options valid for all the tests in the current "directory".  After
# each test, we reset the options to these.  To change the options for
# multiple tests, the function setTestOpts() below can be used to alter
# these options.
global thisdir_testopts
thisdir_testopts = TestOptions()

def getThisDirTestOpts():
    return thisdir_testopts

# Options valid for the current test only (these get reset to
# testdir_testopts after each test).
42

ei@vuokko.info's avatar
ei@vuokko.info committed
43
global testopts_local
44 45 46 47 48 49
if config.use_threads:
    testopts_local = threading.local()
else:
    class TestOpts_Local:
        pass
    testopts_local = TestOpts_Local()
50 51

def getTestOpts():
ei@vuokko.info's avatar
ei@vuokko.info committed
52
    return testopts_local.x
53

ei@vuokko.info's avatar
ei@vuokko.info committed
54 55 56
def setLocalTestOpts(opts):
    global testopts_local
    testopts_local.x=opts
57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82

# This can be called at the top of a file of tests, to set default test options
# for the following tests.
def setTestOpts( f ):
    f( thisdir_testopts );

# -----------------------------------------------------------------------------
# Canned setup functions for common cases.  eg. for a test you might say
#
#      test('test001', normal, compile, [''])
#
# to run it without any options, but change it to
#
#      test('test001', expect_fail, compile, [''])
#
# to expect failure for this test.

def normal( opts ):
    return;

def skip( opts ):
    opts.skip = 1

def expect_fail( opts ):
    opts.expect = 'fail';

83 84 85
def reqlib( lib ):
    return lambda opts, l=lib: _reqlib (opts, l )

86 87 88 89
# Cache the results of looking to see if we have a library or not.
# This makes quite a difference, especially on Windows.
have_lib = {}

90
def _reqlib( opts, lib ):
91 92
    if have_lib.has_key(lib):
        got_it = have_lib[lib]
93
    else:
94 95 96
        if have_subprocess:
            # By preference we use subprocess, as the alternative uses
            # /dev/null which mingw doesn't have.
97
            p = subprocess.Popen([config.ghc_pkg, '--no-user-package-conf', 'describe', lib],
98 99 100 101 102 103 104 105 106 107 108 109 110
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
            # read from stdout and stderr to avoid blocking due to
            # buffers filling
            p.communicate()
            r = p.wait()
        else:
            r = os.system(config.ghc_pkg + ' describe ' + lib
                                         + ' > /dev/null 2> /dev/null')
        got_it = r == 0
        have_lib[lib] = got_it

    if not got_it:
111 112
        opts.expect = 'fail'

113 114 115 116
def req_profiling( opts ):
    if not config.have_profiling:
        opts.expect = 'fail'

Ian Lynagh's avatar
Ian Lynagh committed
117 118 119 120
def req_interp( opts ):
    if not config.have_interp:
        opts.expect = 'fail'

Ian Lynagh's avatar
Ian Lynagh committed
121 122 123 124 125 126
def expect_broken( bug ):
    return lambda opts, b=bug: _expect_broken (opts, b )

def _expect_broken( opts, bug ):
    opts.expect = 'fail';

127 128 129
def ignore_output( opts ):
    opts.ignore_output = 1

130 131 132
def no_stdin( opts ):
    opts.no_stdin = 1

133 134 135 136 137 138 139 140
# -----

def expect_fail_for( ways ):
    return lambda opts, w=ways: _expect_fail_for( opts, w )

def _expect_fail_for( opts, ways ):
    opts.expect_fail_for = ways

Ian Lynagh's avatar
Ian Lynagh committed
141 142 143 144
def expect_broken_for( bug, ways ):
    return lambda opts, b=bug, w=ways: _expect_broken_for( opts, b, w )

def _expect_broken_for( opts, bug, ways ):
Ian Lynagh's avatar
Ian Lynagh committed
145
    opts.expect_fail_for = ways
Ian Lynagh's avatar
Ian Lynagh committed
146

147 148 149 150 151 152 153 154 155 156
# -----

def omit_ways( ways ):
    return lambda opts, w=ways: _omit_ways( opts, w )

def _omit_ways( opts, ways ):
    opts.omit_ways = ways

# -----

157 158 159 160 161 162 163 164
def only_ways( ways ):
    return lambda opts, w=ways: _only_ways( opts, w )

def _only_ways( opts, ways ):
    opts.only_ways = ways

# -----

165 166 167 168 169 170 171 172
def extra_ways( ways ):
    return lambda opts, w=ways: _extra_ways( opts, w )

def _extra_ways( opts, ways ):
    opts.extra_ways = ways

# -----

ross's avatar
ross committed
173 174 175 176 177
def omit_compiler_types( compiler_types ):
   return lambda opts, c=compiler_types: _omit_compiler_types(opts, c)

def _omit_compiler_types( opts, compiler_types ):
    if config.compiler_type in compiler_types:
dterei's avatar
dterei committed
178
        opts.skip = 1
ross's avatar
ross committed
179 180 181 182 183 184 185 186

# -----

def only_compiler_types( compiler_types ):
   return lambda opts, c=compiler_types: _only_compiler_types(opts, c)

def _only_compiler_types( opts, compiler_types ):
    if config.compiler_type not in compiler_types:
dterei's avatar
dterei committed
187
        opts.skip = 1
ross's avatar
ross committed
188 189 190

# -----

191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212
def set_stdin( file ):
   return lambda opts, f=file: _set_stdin(opts, f);

def _set_stdin( opts, f ):
   opts.stdin = f

# -----

def exit_code( val ):
    return lambda opts, v=val: _exit_code(opts, v);

def _exit_code( opts, v ):
    opts.exit_code = v

# -----

def extra_run_opts( val ):
    return lambda opts, v=val: _extra_run_opts(opts, v);

def _extra_run_opts( opts, v ):
    opts.extra_run_opts = v

213 214 215 216 217 218 219 220
# -----

def extra_clean( files ):
    return lambda opts, v=files: _extra_clean(opts, v);

def _extra_clean( opts, v ):
    opts.clean_files = v

221 222
# -----

223 224
def stats_num_field( field, min, max ):
    return lambda opts, f=field, x=min, y=max: _stats_num_field(opts, f, x, y);
225

226
def _stats_num_field( opts, f, x, y ):
227 228 229
    # copy the dictionary, as the config gets shared between all tests
    opts.stats_num_fields = opts.stats_num_fields.copy()
    opts.stats_num_fields[f] = (x, y)
230

Ian Lynagh's avatar
Ian Lynagh committed
231 232 233 234
def compiler_stats_num_field( field, min, max ):
    return lambda opts, f=field, x=min, y=max: _compiler_stats_num_field(opts, f, x, y);

def _compiler_stats_num_field( opts, f, x, y ):
235 236 237
    # copy the dictionary, as the config gets shared between all tests
    opts.compiler_stats_num_fields = opts.compiler_stats_num_fields.copy()
    opts.compiler_stats_num_fields[f] = (x, y)
Ian Lynagh's avatar
Ian Lynagh committed
238

239 240
# -----

241
def skip_if_no_ghci(opts):
dterei's avatar
dterei committed
242 243
    if not ('ghci' in config.run_ways):
        opts.skip = 1
244

245 246 247
# ----

def skip_if_fast(opts):
dterei's avatar
dterei committed
248 249
    if config.fast:
        opts.skip = 1
250

Simon Marlow's avatar
Simon Marlow committed
251 252
# -----

253
def if_platform( plat, f ):
Simon Marlow's avatar
Simon Marlow committed
254
    if config.platform == plat:
255
        return f
256 257 258
    else:
        return normal

Ian Lynagh's avatar
Ian Lynagh committed
259 260 261 262 263 264
def if_not_platform( plat, f ):
    if config.platform != plat:
        return f
    else:
        return normal

265 266 267
def if_os( os, f ):
    if config.os == os:
        return f
268 269 270
    else:
        return normal

Ian Lynagh's avatar
Ian Lynagh committed
271 272 273 274 275 276
def unless_os( os, f ):
    if config.os == os:
        return normal
    else:
        return f

277 278 279 280 281 282
def if_arch( arch, f ):
    if config.arch == arch:
        return f
    else:
        return normal

Ian Lynagh's avatar
Ian Lynagh committed
283
def if_wordsize( ws, f ):
Ian Lynagh's avatar
Ian Lynagh committed
284
    if config.wordsize == str(ws):
Ian Lynagh's avatar
Ian Lynagh committed
285 286 287 288
        return f
    else:
        return normal

Ian Lynagh's avatar
Ian Lynagh committed
289 290 291 292 293 294 295 296 297 298 299 300
def if_msys( f ):
    if config.msys:
        return f
    else:
        return normal

def if_cygwin( f ):
    if config.cygwin:
        return f
    else:
        return normal

301 302
# ---

303 304 305 306 307 308 309 310 311 312 313 314
def if_in_tree_compiler( f ):
    if config.in_tree_compiler:
        return f
    else:
        return normal

def unless_in_tree_compiler( f ):
    if config.in_tree_compiler:
        return normal
    else:
        return f

315 316 317 318 319
def if_compiler_type( compiler, f ):
    if config.compiler_type == compiler:
        return f
    else:
        return normal
320 321 322 323 324 325

def if_compiler_profiled( f ):
    if config.compiler_profiled:
        return f
    else:
        return normal
326

327 328 329 330 331 332
def unless_compiler_profiled( f ):
    if config.compiler_profiled:
        return normal
    else:
        return f

Ian Lynagh's avatar
Ian Lynagh committed
333
def if_compiler_lt( compiler, version, f ):
334
    if config.compiler_type == compiler and \
Ian Lynagh's avatar
Ian Lynagh committed
335 336 337 338
       version_lt(config.compiler_version, version):
        return f
    else:
        return normal
339

Ian Lynagh's avatar
Ian Lynagh committed
340
def if_compiler_le( compiler, version, f ):
341 342
    if config.compiler_type == compiler and \
       version_le(config.compiler_version, version):
Ian Lynagh's avatar
Ian Lynagh committed
343 344 345
        return f
    else:
        return normal
346

Ian Lynagh's avatar
Ian Lynagh committed
347
def if_compiler_gt( compiler, version, f ):
348
    if config.compiler_type == compiler and \
Ian Lynagh's avatar
Ian Lynagh committed
349 350 351 352
       version_gt(config.compiler_version, version):
        return f
    else:
        return normal
353

Ian Lynagh's avatar
Ian Lynagh committed
354
def if_compiler_ge( compiler, version, f ):
355
    if config.compiler_type == compiler and \
Ian Lynagh's avatar
Ian Lynagh committed
356 357 358 359
       version_ge(config.compiler_version, version):
        return f
    else:
        return normal
360

Ian Lynagh's avatar
Ian Lynagh committed
361 362
def namebase( nb ):
   return lambda opts, nb=nb: _namebase(opts, nb)
363

Ian Lynagh's avatar
Ian Lynagh committed
364 365
def _namebase( opts, nb ):
    opts.with_namebase = nb
366

367 368
# ---

369
def if_tag( tag, f ):
370
    if tag in config.compiler_tags:
371 372 373
        return f
    else:
        return normal
374

375
def unless_tag( tag, f ):
376
    if not (tag in config.compiler_tags):
377 378 379
        return f
    else:
        return normal
380

ei@vuokko.info's avatar
ei@vuokko.info committed
381
# ---
382
def alone(opts):
383
    opts.alone = True
ei@vuokko.info's avatar
ei@vuokko.info committed
384

Ian Lynagh's avatar
Ian Lynagh committed
385 386 387 388
# ---
def literate( opts ):
    opts.literate = 1;

389 390 391
def c_src( opts ):
    opts.c_src = 1;

Austin Seipp's avatar
Austin Seipp committed
392 393 394
def objc_src( opts ):
    opts.objc_src = 1;

395 396 397
def objcpp_src( opts ):
    opts.objcpp_src = 1;

398 399
# ----

400 401 402 403 404 405 406 407
def pre_cmd( cmd ):
    return lambda opts, c=cmd: _pre_cmd(opts, cmd)

def _pre_cmd( opts, cmd ):
    opts.pre_cmd = cmd

# ----

408 409 410 411 412 413 414 415
def clean_cmd( cmd ):
    return lambda opts, c=cmd: _clean_cmd(opts, cmd)

def _clean_cmd( opts, cmd ):
    opts.clean_cmd = cmd

# ----

416 417 418 419 420 421
def cmd_prefix( prefix ):
    return lambda opts, p=prefix: _cmd_prefix(opts, prefix)

def _cmd_prefix( opts, prefix ):
    opts.cmd_prefix = prefix

422 423
# ----

Ian Lynagh's avatar
Ian Lynagh committed
424 425 426 427 428 429 430 431
def compile_cmd_prefix( prefix ):
    return lambda opts, p=prefix: _compile_cmd_prefix(opts, prefix)

def _compile_cmd_prefix( opts, prefix ):
    opts.compile_cmd_prefix = prefix

# ----

432 433 434
def normalise_slashes( opts ):
    opts.extra_normaliser = normalise_slashes_

435 436 437 438 439 440
def normalise_fun( fun ):
    return lambda opts, f=fun: _normalise_fun(opts, f)

def _normalise_fun( opts, f ):
    opts.extra_normaliser = f

441 442 443
# ----
# Function for composing two opt-fns together

444 445 446
def composes( fs ):
    return reduce(lambda f, g: compose(f, g), fs)

447 448 449
def compose( f, g ):
    return lambda opts, f=f, g=g: _compose(opts,f,g)

450
def _compose( opts, f, g ):
451 452 453 454 455 456 457
    f(opts)
    g(opts)

# -----------------------------------------------------------------------------
# The current directory of tests

def newTestDir( dir ):
458
    global thisdir_testopts
459 460
    # reset the options for this test directory
    thisdir_testopts = copy.copy(default_testopts)
461
    thisdir_testopts.testdir = dir
462 463 464 465

# -----------------------------------------------------------------------------
# Actually doing tests

466
allTests = []
467
allTestNames = set([])
468 469

def runTest (opts, name, setup, func, args):
ei@vuokko.info's avatar
ei@vuokko.info committed
470
    n = 1
471 472 473 474

    if type(setup) is types.ListType:
       setup = composes(setup)

ei@vuokko.info's avatar
ei@vuokko.info committed
475
    setup(opts)
476

477
    if opts.alone:
ei@vuokko.info's avatar
ei@vuokko.info committed
478
        n = config.threads
479

ei@vuokko.info's avatar
ei@vuokko.info committed
480 481
    ok = 0

482
    if config.use_threads:
ei@vuokko.info's avatar
ei@vuokko.info committed
483
        t.thread_pool.acquire()
484 485 486 487 488 489 490 491 492 493 494 495
        try:
            while config.threads<(t.running_threads+n):
                t.thread_pool.wait()
            t.running_threads = t.running_threads+n
            ok=1
            t.thread_pool.release()
            thread.start_new_thread(test_common_thread, (n, name, opts, func, args))
        except:
            if not ok:
                t.thread_pool.release()
    else:
        test_common_work (name, opts, func, args)
496

497
# name  :: String
498
# setup :: TestOpts -> IO ()
499 500
def test (name, setup, func, args):
    global allTests
501 502 503
    global allTestNames
    if name in allTestNames:
        framework_fail(name, 'duplicate', 'There are multiple tests with this name')
504 505
    myTestOpts = copy.copy(thisdir_testopts)
    allTests += [lambda : runTest(myTestOpts, name, setup, func, args)]
506
    allTestNames.add(name)
507

508 509 510 511 512 513 514 515 516 517 518
if config.use_threads:
    def test_common_thread(n, name, opts, func, args):
        t.lock.acquire()
        try:
            test_common_work(name,opts,func,args)
        finally:
            t.lock.release()
            t.thread_pool.acquire()
            t.running_threads = t.running_threads - n
            t.thread_pool.notify()
            t.thread_pool.release()
519

520 521 522 523 524 525 526 527 528 529
def get_package_cache_timestamp():
    if config.package_conf_cache_file == '':
        return 0.0
    else:
        try:
            return os.stat(config.package_conf_cache_file).st_mtime
        except:
            return 0.0


ei@vuokko.info's avatar
ei@vuokko.info committed
530 531 532
def test_common_work (name, opts, func, args):
    t.total_tests = t.total_tests+1
    setLocalTestOpts(opts)
533

534
    package_conf_cache_file_start_timestamp = get_package_cache_timestamp()
535

536
    # All the ways we might run this test
537
    if func == compile or func == multimod_compile:
538
        all_ways = config.compile_ways
dterei's avatar
dterei committed
539
    elif func == compile_and_run or func == multimod_compile_and_run or func == multisrc_compile_and_run:
540
        all_ways = config.run_ways
541 542
    elif func == ghci_script:
        if 'ghci' in config.run_ways:
543
            all_ways = ['ghci']
544
        else:
545
            all_ways = []
546
    else:
547 548
        all_ways = ['normal']

549 550 551
    # A test itself can request extra ways by setting opts.extra_ways
    all_ways = all_ways + filter(lambda way: way not in all_ways,
                                 opts.extra_ways)
552

553 554 555
    t.total_test_cases = t.total_test_cases + len(all_ways)

    ok_way = lambda way: \
ei@vuokko.info's avatar
ei@vuokko.info committed
556
        not getTestOpts().skip \
557
        and (config.only == [] or name in config.only) \
ei@vuokko.info's avatar
ei@vuokko.info committed
558
        and (getTestOpts().only_ways == [] or way in getTestOpts().only_ways) \
559
        and (config.cmdline_ways == [] or way in config.cmdline_ways) \
ei@vuokko.info's avatar
ei@vuokko.info committed
560
        and way not in getTestOpts().omit_ways
561 562 563 564 565 566 567 568

    # Which ways we are asked to skip
    do_ways = filter (ok_way,all_ways)

    # In fast mode, we skip all but one way
    if config.fast and len(do_ways) > 0:
        do_ways = [do_ways[0]]

569 570 571 572
    if not config.clean_only:
        # Run the required tests...
        for way in do_ways:
            do_test (name, way, func, args)
573

574 575 576
        for way in all_ways:
            if way not in do_ways:
                skiptest (name,way)
577

578
    if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
Ian Lynagh's avatar
Ian Lynagh committed
579 580 581 582 583 584 585 586 587
        clean(map (lambda suff: name + suff,
                  ['', '.exe', '.exe.manifest', '.genscript',
                   '.stderr.normalised',        '.stdout.normalised',
                   '.run.stderr',               '.run.stdout',
                   '.run.stderr.normalised',    '.run.stdout.normalised',
                   '.comp.stderr',              '.comp.stdout',
                   '.comp.stderr.normalised',   '.comp.stdout.normalised',
                   '.interp.stderr',            '.interp.stdout',
                   '.interp.stderr.normalised', '.interp.stdout.normalised',
Ian Lynagh's avatar
Ian Lynagh committed
588
                   '.stats', '.comp.stats',
Ian Lynagh's avatar
Ian Lynagh committed
589 590
                   '.hi', '.o', '.prof', '.exe.prof', '.hc',
                   '_stub.h', '_stub.c', '_stub.o',
591
                   '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))
Ian Lynagh's avatar
Ian Lynagh committed
592 593 594

        clean(getTestOpts().clean_files)

595 596 597
        try:
            cleanCmd = getTestOpts().clean_cmd
            if cleanCmd != None:
598
                result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
599 600 601
                if result != 0:
                    framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
        except e:
602 603
            framework_fail(name, 'cleaning', 'clean-command exception')

604
    package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
605 606 607

    if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
        framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
608

Ian Lynagh's avatar
Ian Lynagh committed
609 610 611 612 613 614 615 616 617 618 619
    try:
        for f in files_written[name]:
            if os.path.exists(f):
                try:
                    if not f in files_written_not_removed[name]:
                        files_written_not_removed[name].append(f)
                except:
                    files_written_not_removed[name] = [f]
    except:
        pass

620 621 622 623
def clean(strs):
    for str in strs:
        for name in glob.glob(in_testdir(str)):
            clean_full_path(name)
624

625
def clean_full_path(name):
Ian Lynagh's avatar
Ian Lynagh committed
626 627 628 629
        try:
            # Remove files...
            os.remove(name)
        except OSError, e1:
630
            try:
Ian Lynagh's avatar
Ian Lynagh committed
631 632 633 634 635 636 637 638 639 640 641
                # ... and empty directories
                os.rmdir(name)
            except OSError, e2:
                # We don't want to fail here, but we do want to know
                # what went wrong, so print out the exceptions.
                # ENOENT isn't a problem, though, as we clean files
                # that don't necessarily exist.
                if e1.errno != errno.ENOENT:
                    print e1
                if e2.errno != errno.ENOENT:
                    print e2
642

643
def do_test(name, way, func, args):
644 645 646
    full_name = name + '(' + way + ')'

    try:
647 648 649 650
        print '=====>', full_name, t.total_tests, 'of', len(allTests), \
                        str([t.n_unexpected_passes,   \
                             t.n_unexpected_failures, \
                             t.n_framework_failures])
651

652 653
        if config.use_threads:
            t.lock.release()
654 655 656 657

        try:
            preCmd = getTestOpts().pre_cmd
            if preCmd != None:
658
                result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
659 660 661 662 663
                if result != 0:
                    framework_fail(name, way, 'pre-command failed: ' + str(result))
        except e:
            framework_fail(name, way, 'pre-command exception')

ei@vuokko.info's avatar
ei@vuokko.info committed
664 665 666
        try:
            result = apply(func, [name,way] + args)
        finally:
667 668
            if config.use_threads:
                t.lock.acquire()
669

670 671
        if getTestOpts().expect != 'pass' and getTestOpts().expect != 'fail':
            framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
672

673 674 675 676 677 678
        try:
            passFail = result['passFail']
        except:
            passFail = 'No passFail found'

        if passFail == 'pass':
ei@vuokko.info's avatar
ei@vuokko.info committed
679 680
            if getTestOpts().expect == 'pass' \
               and way not in getTestOpts().expect_fail_for:
681
                t.n_expected_passes = t.n_expected_passes + 1
682 683 684 685
                if name in t.expected_passes:
                    t.expected_passes[name].append(way)
                else:
                    t.expected_passes[name] = [way]
686 687 688
            else:
                print '*** unexpected pass for', full_name
                t.n_unexpected_passes = t.n_unexpected_passes + 1
689
                addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
690
        elif passFail == 'fail':
ei@vuokko.info's avatar
ei@vuokko.info committed
691 692
            if getTestOpts().expect == 'pass' \
               and way not in getTestOpts().expect_fail_for:
693 694
                print '*** unexpected failure for', full_name
                t.n_unexpected_failures = t.n_unexpected_failures + 1
695 696
                reason = result['reason']
                addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
697 698
            else:
                t.n_expected_failures = t.n_expected_failures + 1
699 700 701 702
                if name in t.expected_failures:
                    t.expected_failures[name].append(way)
                else:
                    t.expected_failures[name] = [way]
703 704
        else:
            framework_fail(name, way, 'bad result ' + passFail)
705
    except:
706
        framework_fail(name, way, 'do_test exception')
707 708
        traceback.print_exc()

709
def addPassingTestInfo (testInfos, directory, name, way):
710 711 712 713 714 715 716 717 718 719
    directory = re.sub('^\\.[/\\\\]', '', directory)

    if not directory in testInfos:
        testInfos[directory] = {}

    if not name in testInfos[directory]:
        testInfos[directory][name] = []

    testInfos[directory][name].append(way)

720 721 722 723 724 725 726 727 728 729 730 731 732 733
def addFailingTestInfo (testInfos, directory, name, reason, way):
    directory = re.sub('^\\.[/\\\\]', '', directory)

    if not directory in testInfos:
        testInfos[directory] = {}

    if not name in testInfos[directory]:
        testInfos[directory][name] = {}

    if not reason in testInfos[directory][name]:
        testInfos[directory][name][reason] = []

    testInfos[directory][name][reason].append(way)

734
def skiptest (name, way):
735 736
    # print 'Skipping test \"', name, '\"'
    t.n_tests_skipped = t.n_tests_skipped + 1
737 738 739 740
    if name in t.tests_skipped:
        t.tests_skipped[name].append(way)
    else:
        t.tests_skipped[name] = [way]
741

742 743 744
def framework_fail( name, way, reason ):
    full_name = name + '(' + way + ')'
    print '*** framework failure for', full_name, reason, ':'
745
    t.n_framework_failures = t.n_framework_failures + 1
746 747 748 749
    if name in t.framework_failures:
        t.framework_failures[name].append(way)
    else:
        t.framework_failures[name] = [way]
750

751 752 753 754 755 756 757 758 759 760 761 762 763 764
def badResult(result):
    try:
        if result['passFail'] == 'pass':
            return False
        return True
    except:
        return True

def passed():
    return {'passFail': 'pass'}

def failBecause(reason):
    return {'passFail': 'fail', 'reason': reason}

765 766 767 768
# -----------------------------------------------------------------------------
# Generic command tests

# A generic command test is expected to run and exit successfully.
769 770 771 772 773 774
#
# The expected exit code can be changed via exit_code() as normal, and
# the expected stdout/stderr are stored in <testname>.stdout and
# <testname>.stderr.  The output of the command can be ignored
# altogether by using run_command_ignore_output instead of
# run_command.
775

776
def run_command( name, way, cmd ):
777
    return simple_run( name, '', cmd, '' )
778

779 780 781 782
# -----------------------------------------------------------------------------
# GHCi tests

def ghci_script( name, way, script ):
783
    # filter out -fforce-recomp from compiler_always_flags, because we're
784
    # actually testing the recompilation behaviour in the GHCi tests.
785
    flags = filter(lambda f: f != '-fforce-recomp', config.compiler_always_flags)
786
    flags.append(getTestOpts().extra_hc_opts)
787 788 789 790 791 792

    # We pass HC and HC_OPTS as environment variables, so that the
    # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
    cmd = "HC='" + config.compiler + "' " + \
          "HC_OPTS='" + join(flags,' ') + "' " + \
          "'" + config.compiler + "'" + \
793
          ' --interactive -v0 -ignore-dot-ghci ' + \
794 795
          join(flags,' ')

ei@vuokko.info's avatar
ei@vuokko.info committed
796
    getTestOpts().stdin = script
797
    return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
798

799 800 801 802
# -----------------------------------------------------------------------------
# Compile-only tests

def compile( name, way, extra_hc_opts ):
803
    return do_compile( name, way, 0, '', [], extra_hc_opts )
804 805

def compile_fail( name, way, extra_hc_opts ):
806
    return do_compile( name, way, 1, '', [], extra_hc_opts )
807 808

def multimod_compile( name, way, top_mod, extra_hc_opts ):
809
    return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
810

811
def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
812 813 814 815 816 817 818 819 820
    return do_compile( name, way, 1, top_mod, [], extra_hc_opts )

def multisrc_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
    extra_mods = map ((lambda y : (y, '')), extra_mods)
    return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)

def multisrc_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
    extra_mods = map ((lambda y : (y, '')), extra_mods)
    return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)
821

822 823 824 825 826 827 828
def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
    return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)

def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
    return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)

def do_compile( name, way, should_fail, top_mod, extra_mods, extra_hc_opts ):
829 830
    # print 'Compile only, extra args = ', extra_hc_opts
    pretest_cleanup(name)
831 832 833 834 835 836

    result = extras_build( way, extra_mods, extra_hc_opts )
    if badResult(result):
       return result
    extra_hc_opts = result['hc_opts']

837 838 839
    force = 0
    if extra_mods:
       force = 1
840 841
    result = simple_build( name, way, extra_hc_opts, should_fail, top_mod, 0, 1, force)

842
    if badResult(result):
Ian Lynagh's avatar
Ian Lynagh committed
843
        return result
844 845 846 847 848

    # the actual stderr should always match the expected, regardless
    # of whether we expected the compilation to fail or not (successful
    # compilations may generate warnings).

Ian Lynagh's avatar
Ian Lynagh committed
849
    if getTestOpts().with_namebase == None:
850 851
        namebase = name
    else:
Ian Lynagh's avatar
Ian Lynagh committed
852
        namebase = getTestOpts().with_namebase
853 854

    (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
855 856
    actual_stderr_file = qualify(name, 'comp.stderr')

857 858
    if not compare_outputs('stderr', normalise_errmsg, normalise_whitespace, \
                           expected_stderr_file, actual_stderr_file):
859
        return failBecause('stderr mismatch')
860 861

    # no problems found, this test passed
862
    return passed()
863 864 865 866

# -----------------------------------------------------------------------------
# Compile-and-run tests

867
def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts ):
868 869 870
    # print 'Compile and run, extra args = ', extra_hc_opts
    pretest_cleanup(name)

871 872 873 874
    result = extras_build( way, extra_mods, extra_hc_opts )
    if badResult(result):
       return result
    extra_hc_opts = result['hc_opts']
dterei's avatar
dterei committed
875

876
    if way == 'ghci': # interpreted...
877
        return interpreter_run( name, way, extra_hc_opts, 0, top_mod )
krc's avatar
krc committed
878
    elif way == 'extcore' or way == 'optextcore' :
879
        return extcore_run( name, way, extra_hc_opts, 0, top_mod )
880
    else: # compiled...
dterei's avatar
dterei committed
881 882 883 884
        force = 0
        if extra_mods:
           force = 1

885
        result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1, 1, force)
886
        if badResult(result):
Ian Lynagh's avatar
Ian Lynagh committed
887
            return result
888

889
        cmd = './' + name;
890 891
        if getTestOpts().cmd_prefix != '':
            cmd = getTestOpts().cmd_prefix + ' ' + cmd;
892

893
        # we don't check the compiler's stderr for a compile-and-run test
894 895
        return simple_run( name, way, cmd, getTestOpts().extra_run_opts )

896
def compile_and_run( name, way, extra_hc_opts ):
897
    return compile_and_run__( name, way, '', [], extra_hc_opts)
898 899

def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
900
    return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
dterei's avatar
dterei committed
901 902

def multisrc_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
903 904 905 906 907
    extra_mods = map ((lambda y : (y, '')), extra_mods)
    return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)

def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
    return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
908

Ian Lynagh's avatar
Ian Lynagh committed
909 910 911 912
# -----------------------------------------------------------------------------
# Check -t stats info

def checkStats(stats_file, num_fields):
913
    result = passed()
914
    if len(num_fields) > 0:
Ian Lynagh's avatar
Ian Lynagh committed
915 916 917 918
        f = open(in_testdir(stats_file))
        contents = f.read()
        f.close()

919
        for (field, (min, max)) in num_fields.items():
Ian Lynagh's avatar
Ian Lynagh committed
920 921 922
            m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
            if m == None:
                print 'Failed to find field: ', field
923
                result = failBecause('no such stats field')
Ian Lynagh's avatar
Ian Lynagh committed
924 925 926 927 928 929
            val = int(m.group(1))

            if val < min:
                print field, val, 'is less than minimum allowed', min
                print 'If this is because you have improved GHC, please'
                print 'update the test so that GHC doesn\'t regress again'
930
                result = failBecause('stat too good')
Ian Lynagh's avatar
Ian Lynagh committed
931 932
            if val > max:
                print field, val, 'is more than maximum allowed', max
933
                result = failBecause('stat not good enough')
Ian Lynagh's avatar
Ian Lynagh committed
934

935
    return result
Ian Lynagh's avatar
Ian Lynagh committed
936

937 938 939
# -----------------------------------------------------------------------------
# Build a single-module program

940 941 942 943 944 945 946 947 948 949 950 951 952
def extras_build( way, extra_mods, extra_hc_opts ):
    for modopts in extra_mods:
        mod, opts = modopts
        result = simple_build( mod, way, opts + extra_hc_opts, 0, '', 0, 0, 0)
        if not (mod.endswith(".hs") or mod.endswith(".lhs")):
            extra_hc_opts += " " + replace_suffix(mod, 'o')
        if badResult(result):
            return result

    return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}


def simple_build( name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, noforce ):
Ian Lynagh's avatar
Ian Lynagh committed
953
    opts = getTestOpts()
954
    errname = add_suffix(name, 'comp.stderr')
955 956
    rm_no_fail( qualify(errname, '') )

957 958
    if top_mod != '':
        srcname = top_mod
959 960 961 962
        rm_no_fail( qualify(name, '') )
        base, suf = os.path.splitext(top_mod)
        rm_no_fail( qualify(base, '') )
        rm_no_fail( qualify(base, 'exe') )
dterei's avatar
dterei committed
963
    elif addsuf:
Ian Lynagh's avatar
Ian Lynagh committed
964
        srcname = add_hs_lhs_suffix(name)
965
        rm_no_fail( qualify(name, '') )
dterei's avatar
dterei committed
966 967
    else:
        srcname = name
968 969 970
        rm_no_fail( qualify(name, 'o') )

    rm_no_fail( qualify(replace_suffix(srcname, "o"), '') )
971 972 973

    to_do = ''
    if top_mod != '':
974 975 976
        to_do = '--make '
        if link:
            to_do = to_do + '-o ' + name
977 978
    elif link:
        to_do = '-o ' + name
Ian Lynagh's avatar
Ian Lynagh committed
979
    elif opts.compile_to_hc:
980 981 982 983
        to_do = '-C'
    else:
        to_do = '-c' # just compile

Ian Lynagh's avatar
Ian Lynagh committed
984
    stats_file = name + '.comp.stats'
985
    if len(opts.compiler_stats_num_fields) > 0:
986
        extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
987

Ian Lynagh's avatar
Ian Lynagh committed
988 989 990 991 992
    if getTestOpts().compile_cmd_prefix == '':
        cmd_prefix = ''
    else:
        cmd_prefix = getTestOpts().compile_cmd_prefix + ' '

993 994 995 996
    comp_flags = config.compiler_always_flags
    if noforce:
        comp_flags = filter(lambda f: f != '-fforce-recomp', comp_flags)

Ian Lynagh's avatar
Ian Lynagh committed
997
    cmd = 'cd ' + getTestOpts().testdir + " && " + cmd_prefix + "'" \
998
          + config.compiler + "' " \
999
          + join(comp_flags,' ') + ' ' \