testlib.py 63.9 KB
Newer Older
1
#
2
3
4
# (c) Simon Marlow 2002
#

5
6
7
# This allows us to use the "with X:" syntax with python 2.5:
from __future__ import with_statement

8
9
import sys
import os
10
import errno
11
12
13
14
import string
import re
import traceback
import copy
15
import glob
16
import types
17

18
19
20
21
22
23
24
have_subprocess = False
try:
    import subprocess
    have_subprocess = True
except:
    print "Warning: subprocess not found, will fall back to spawnv"

25
from string import join
26
from testglobals import *
27
28
from testutil import *

29
30
31
if config.use_threads:
    import threading
    import thread
32
33
34
35
36
37
38
39
40
41
42
43
44

# Options valid for all the tests in the current "directory".  After
# each test, we reset the options to these.  To change the options for
# multiple tests, the function setTestOpts() below can be used to alter
# these options.
global thisdir_testopts
thisdir_testopts = TestOptions()

def getThisDirTestOpts():
    return thisdir_testopts

# Options valid for the current test only (these get reset to
# testdir_testopts after each test).
45

ei@vuokko.info's avatar
ei@vuokko.info committed
46
global testopts_local
47
48
49
50
51
52
if config.use_threads:
    testopts_local = threading.local()
else:
    class TestOpts_Local:
        pass
    testopts_local = TestOpts_Local()
53
54

def getTestOpts():
ei@vuokko.info's avatar
ei@vuokko.info committed
55
    return testopts_local.x
56

ei@vuokko.info's avatar
ei@vuokko.info committed
57
58
59
def setLocalTestOpts(opts):
    global testopts_local
    testopts_local.x=opts
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85

# This can be called at the top of a file of tests, to set default test options
# for the following tests.
def setTestOpts( f ):
    f( thisdir_testopts );

# -----------------------------------------------------------------------------
# Canned setup functions for common cases.  eg. for a test you might say
#
#      test('test001', normal, compile, [''])
#
# to run it without any options, but change it to
#
#      test('test001', expect_fail, compile, [''])
#
# to expect failure for this test.

def normal( opts ):
    return;

def skip( opts ):
    opts.skip = 1

def expect_fail( opts ):
    opts.expect = 'fail';

86
87
88
def reqlib( lib ):
    return lambda opts, l=lib: _reqlib (opts, l )

89
90
91
92
# Cache the results of looking to see if we have a library or not.
# This makes quite a difference, especially on Windows.
have_lib = {}

93
def _reqlib( opts, lib ):
94
95
    if have_lib.has_key(lib):
        got_it = have_lib[lib]
96
    else:
97
98
99
        if have_subprocess:
            # By preference we use subprocess, as the alternative uses
            # /dev/null which mingw doesn't have.
100
            p = subprocess.Popen([config.ghc_pkg, '--no-user-package-conf', 'describe', lib],
101
102
103
104
105
106
107
108
109
110
111
112
113
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
            # read from stdout and stderr to avoid blocking due to
            # buffers filling
            p.communicate()
            r = p.wait()
        else:
            r = os.system(config.ghc_pkg + ' describe ' + lib
                                         + ' > /dev/null 2> /dev/null')
        got_it = r == 0
        have_lib[lib] = got_it

    if not got_it:
114
115
        opts.expect = 'fail'

116
117
118
119
def req_profiling( opts ):
    if not config.have_profiling:
        opts.expect = 'fail'

Ian Lynagh's avatar
Ian Lynagh committed
120
121
122
123
def req_interp( opts ):
    if not config.have_interp:
        opts.expect = 'fail'

Ian Lynagh's avatar
Ian Lynagh committed
124
125
126
127
128
129
def expect_broken( bug ):
    return lambda opts, b=bug: _expect_broken (opts, b )

def _expect_broken( opts, bug ):
    opts.expect = 'fail';

130
131
132
def ignore_output( opts ):
    opts.ignore_output = 1

133
134
135
def no_stdin( opts ):
    opts.no_stdin = 1

136
137
138
139
140
141
142
143
# -----

def expect_fail_for( ways ):
    return lambda opts, w=ways: _expect_fail_for( opts, w )

def _expect_fail_for( opts, ways ):
    opts.expect_fail_for = ways

Ian Lynagh's avatar
Ian Lynagh committed
144
145
146
147
def expect_broken_for( bug, ways ):
    return lambda opts, b=bug, w=ways: _expect_broken_for( opts, b, w )

def _expect_broken_for( opts, bug, ways ):
Ian Lynagh's avatar
Ian Lynagh committed
148
    opts.expect_fail_for = ways
Ian Lynagh's avatar
Ian Lynagh committed
149

150
151
152
153
154
155
156
157
158
159
# -----

def omit_ways( ways ):
    return lambda opts, w=ways: _omit_ways( opts, w )

def _omit_ways( opts, ways ):
    opts.omit_ways = ways

# -----

160
161
162
163
164
165
166
167
def only_ways( ways ):
    return lambda opts, w=ways: _only_ways( opts, w )

def _only_ways( opts, ways ):
    opts.only_ways = ways

# -----

168
169
170
171
172
173
174
175
def extra_ways( ways ):
    return lambda opts, w=ways: _extra_ways( opts, w )

def _extra_ways( opts, ways ):
    opts.extra_ways = ways

# -----

ross's avatar
ross committed
176
177
178
179
180
def omit_compiler_types( compiler_types ):
   return lambda opts, c=compiler_types: _omit_compiler_types(opts, c)

def _omit_compiler_types( opts, compiler_types ):
    if config.compiler_type in compiler_types:
dterei's avatar
dterei committed
181
        opts.skip = 1
ross's avatar
ross committed
182
183
184
185
186
187
188
189

# -----

def only_compiler_types( compiler_types ):
   return lambda opts, c=compiler_types: _only_compiler_types(opts, c)

def _only_compiler_types( opts, compiler_types ):
    if config.compiler_type not in compiler_types:
dterei's avatar
dterei committed
190
        opts.skip = 1
ross's avatar
ross committed
191
192
193

# -----

194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
def set_stdin( file ):
   return lambda opts, f=file: _set_stdin(opts, f);

def _set_stdin( opts, f ):
   opts.stdin = f

# -----

def exit_code( val ):
    return lambda opts, v=val: _exit_code(opts, v);

def _exit_code( opts, v ):
    opts.exit_code = v

# -----

def extra_run_opts( val ):
    return lambda opts, v=val: _extra_run_opts(opts, v);

def _extra_run_opts( opts, v ):
    opts.extra_run_opts = v

216
217
218
219
220
221
222
223
# -----

def extra_clean( files ):
    return lambda opts, v=files: _extra_clean(opts, v);

def _extra_clean( opts, v ):
    opts.clean_files = v

224
225
# -----

226
227
def stats_num_field( field, min, max ):
    return lambda opts, f=field, x=min, y=max: _stats_num_field(opts, f, x, y);
228

229
def _stats_num_field( opts, f, x, y ):
230
231
232
    # copy the dictionary, as the config gets shared between all tests
    opts.stats_num_fields = opts.stats_num_fields.copy()
    opts.stats_num_fields[f] = (x, y)
233

Ian Lynagh's avatar
Ian Lynagh committed
234
235
236
237
def compiler_stats_num_field( field, min, max ):
    return lambda opts, f=field, x=min, y=max: _compiler_stats_num_field(opts, f, x, y);

def _compiler_stats_num_field( opts, f, x, y ):
238
239
240
    # copy the dictionary, as the config gets shared between all tests
    opts.compiler_stats_num_fields = opts.compiler_stats_num_fields.copy()
    opts.compiler_stats_num_fields[f] = (x, y)
Ian Lynagh's avatar
Ian Lynagh committed
241

242
243
# -----

244
def skip_if_no_ghci(opts):
dterei's avatar
dterei committed
245
246
    if not ('ghci' in config.run_ways):
        opts.skip = 1
247

248
249
250
# ----

def skip_if_fast(opts):
dterei's avatar
dterei committed
251
252
    if config.fast:
        opts.skip = 1
253

Simon Marlow's avatar
Simon Marlow committed
254
255
# -----

256
def if_platform( plat, f ):
Simon Marlow's avatar
Simon Marlow committed
257
    if config.platform == plat:
258
        return f
259
260
261
    else:
        return normal

Ian Lynagh's avatar
Ian Lynagh committed
262
263
264
265
266
267
def if_not_platform( plat, f ):
    if config.platform != plat:
        return f
    else:
        return normal

268
269
270
def if_os( os, f ):
    if config.os == os:
        return f
271
272
273
    else:
        return normal

Ian Lynagh's avatar
Ian Lynagh committed
274
275
276
277
278
279
def unless_os( os, f ):
    if config.os == os:
        return normal
    else:
        return f

280
281
282
283
284
285
def if_arch( arch, f ):
    if config.arch == arch:
        return f
    else:
        return normal

Ian Lynagh's avatar
Ian Lynagh committed
286
def if_wordsize( ws, f ):
Ian Lynagh's avatar
Ian Lynagh committed
287
    if config.wordsize == str(ws):
Ian Lynagh's avatar
Ian Lynagh committed
288
289
290
291
        return f
    else:
        return normal

Ian Lynagh's avatar
Ian Lynagh committed
292
293
294
295
296
297
298
299
300
301
302
303
def if_msys( f ):
    if config.msys:
        return f
    else:
        return normal

def if_cygwin( f ):
    if config.cygwin:
        return f
    else:
        return normal

304
305
# ---

306
307
308
309
310
311
312
313
314
315
316
317
def if_in_tree_compiler( f ):
    if config.in_tree_compiler:
        return f
    else:
        return normal

def unless_in_tree_compiler( f ):
    if config.in_tree_compiler:
        return normal
    else:
        return f

318
319
320
321
322
def if_compiler_type( compiler, f ):
    if config.compiler_type == compiler:
        return f
    else:
        return normal
323
324
325
326
327
328

def if_compiler_profiled( f ):
    if config.compiler_profiled:
        return f
    else:
        return normal
329

330
331
332
333
334
335
def unless_compiler_profiled( f ):
    if config.compiler_profiled:
        return normal
    else:
        return f

Ian Lynagh's avatar
Ian Lynagh committed
336
def if_compiler_lt( compiler, version, f ):
337
    if config.compiler_type == compiler and \
Ian Lynagh's avatar
Ian Lynagh committed
338
339
340
341
       version_lt(config.compiler_version, version):
        return f
    else:
        return normal
342

Ian Lynagh's avatar
Ian Lynagh committed
343
def if_compiler_le( compiler, version, f ):
344
345
    if config.compiler_type == compiler and \
       version_le(config.compiler_version, version):
Ian Lynagh's avatar
Ian Lynagh committed
346
347
348
        return f
    else:
        return normal
349

Ian Lynagh's avatar
Ian Lynagh committed
350
def if_compiler_gt( compiler, version, f ):
351
    if config.compiler_type == compiler and \
Ian Lynagh's avatar
Ian Lynagh committed
352
353
354
355
       version_gt(config.compiler_version, version):
        return f
    else:
        return normal
356

Ian Lynagh's avatar
Ian Lynagh committed
357
def if_compiler_ge( compiler, version, f ):
358
    if config.compiler_type == compiler and \
Ian Lynagh's avatar
Ian Lynagh committed
359
360
361
362
       version_ge(config.compiler_version, version):
        return f
    else:
        return normal
363

Ian Lynagh's avatar
Ian Lynagh committed
364
365
def namebase( nb ):
   return lambda opts, nb=nb: _namebase(opts, nb)
366

Ian Lynagh's avatar
Ian Lynagh committed
367
368
def _namebase( opts, nb ):
    opts.with_namebase = nb
369

370
371
# ---

372
def if_tag( tag, f ):
373
    if tag in config.compiler_tags:
374
375
376
        return f
    else:
        return normal
377

378
def unless_tag( tag, f ):
379
    if not (tag in config.compiler_tags):
380
381
382
        return f
    else:
        return normal
383

ei@vuokko.info's avatar
ei@vuokko.info committed
384
# ---
385
def alone(opts):
386
    opts.alone = True
ei@vuokko.info's avatar
ei@vuokko.info committed
387

Ian Lynagh's avatar
Ian Lynagh committed
388
389
390
391
# ---
def literate( opts ):
    opts.literate = 1;

392
393
394
def c_src( opts ):
    opts.c_src = 1;

Austin Seipp's avatar
Austin Seipp committed
395
396
397
def objc_src( opts ):
    opts.objc_src = 1;

398
399
400
def objcpp_src( opts ):
    opts.objcpp_src = 1;

401
402
# ----

403
404
405
406
407
408
409
410
def pre_cmd( cmd ):
    return lambda opts, c=cmd: _pre_cmd(opts, cmd)

def _pre_cmd( opts, cmd ):
    opts.pre_cmd = cmd

# ----

411
412
413
414
415
416
417
418
def clean_cmd( cmd ):
    return lambda opts, c=cmd: _clean_cmd(opts, cmd)

def _clean_cmd( opts, cmd ):
    opts.clean_cmd = cmd

# ----

419
420
421
422
def cmd_prefix( prefix ):
    return lambda opts, p=prefix: _cmd_prefix(opts, prefix)

def _cmd_prefix( opts, prefix ):
423
424
425
426
427
428
429
430
431
    opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;

# ----

def cmd_wrapper( fun ):
    return lambda opts, f=fun: _cmd_wrapper(opts, fun)

def _cmd_wrapper( opts, fun ):
    opts.cmd_wrapper = fun
432

433
434
# ----

Ian Lynagh's avatar
Ian Lynagh committed
435
436
437
438
439
440
441
442
def compile_cmd_prefix( prefix ):
    return lambda opts, p=prefix: _compile_cmd_prefix(opts, prefix)

def _compile_cmd_prefix( opts, prefix ):
    opts.compile_cmd_prefix = prefix

# ----

443
444
445
def normalise_slashes( opts ):
    opts.extra_normaliser = normalise_slashes_

446
447
448
449
450
451
def normalise_fun( fun ):
    return lambda opts, f=fun: _normalise_fun(opts, f)

def _normalise_fun( opts, f ):
    opts.extra_normaliser = f

452
453
454
455
456
457
458
459
460
def normalise_errmsg_fun( fun ):
    return lambda opts, f=fun: _normalise_errmsg_fun(opts, f)

def _normalise_errmsg_fun( opts, f ):
    opts.extra_errmsg_normaliser = f

def two_normalisers(f, g):
    return lambda x, f=f, g=g: f(g(x))

461
462
463
# ----
# Function for composing two opt-fns together

464
465
466
def composes( fs ):
    return reduce(lambda f, g: compose(f, g), fs)

467
468
469
def compose( f, g ):
    return lambda opts, f=f, g=g: _compose(opts,f,g)

470
def _compose( opts, f, g ):
471
472
473
474
475
476
477
    f(opts)
    g(opts)

# -----------------------------------------------------------------------------
# The current directory of tests

def newTestDir( dir ):
478
    global thisdir_testopts
479
480
    # reset the options for this test directory
    thisdir_testopts = copy.copy(default_testopts)
481
    thisdir_testopts.testdir = dir
482
483
484
485

# -----------------------------------------------------------------------------
# Actually doing tests

486
allTests = []
487
allTestNames = set([])
488
489

def runTest (opts, name, setup, func, args):
ei@vuokko.info's avatar
ei@vuokko.info committed
490
    n = 1
491
492
493
494

    if type(setup) is types.ListType:
       setup = composes(setup)

ei@vuokko.info's avatar
ei@vuokko.info committed
495
    setup(opts)
496

497
    if opts.alone:
ei@vuokko.info's avatar
ei@vuokko.info committed
498
        n = config.threads
499

ei@vuokko.info's avatar
ei@vuokko.info committed
500
501
    ok = 0

502
    if config.use_threads:
ei@vuokko.info's avatar
ei@vuokko.info committed
503
        t.thread_pool.acquire()
504
505
506
507
508
509
510
511
512
513
514
515
        try:
            while config.threads<(t.running_threads+n):
                t.thread_pool.wait()
            t.running_threads = t.running_threads+n
            ok=1
            t.thread_pool.release()
            thread.start_new_thread(test_common_thread, (n, name, opts, func, args))
        except:
            if not ok:
                t.thread_pool.release()
    else:
        test_common_work (name, opts, func, args)
516

517
# name  :: String
518
# setup :: TestOpts -> IO ()
519
520
def test (name, setup, func, args):
    global allTests
521
522
523
    global allTestNames
    if name in allTestNames:
        framework_fail(name, 'duplicate', 'There are multiple tests with this name')
524
525
    myTestOpts = copy.copy(thisdir_testopts)
    allTests += [lambda : runTest(myTestOpts, name, setup, func, args)]
526
    allTestNames.add(name)
527

528
529
530
531
532
533
534
535
536
537
538
if config.use_threads:
    def test_common_thread(n, name, opts, func, args):
        t.lock.acquire()
        try:
            test_common_work(name,opts,func,args)
        finally:
            t.lock.release()
            t.thread_pool.acquire()
            t.running_threads = t.running_threads - n
            t.thread_pool.notify()
            t.thread_pool.release()
539

540
541
542
543
544
545
546
547
548
549
def get_package_cache_timestamp():
    if config.package_conf_cache_file == '':
        return 0.0
    else:
        try:
            return os.stat(config.package_conf_cache_file).st_mtime
        except:
            return 0.0


ei@vuokko.info's avatar
ei@vuokko.info committed
550
def test_common_work (name, opts, func, args):
551
552
553
554
555
556
557
558
559
    try:
        t.total_tests = t.total_tests+1
        setLocalTestOpts(opts)

        package_conf_cache_file_start_timestamp = get_package_cache_timestamp()

        # All the ways we might run this test
        if func == compile or func == multimod_compile:
            all_ways = config.compile_ways
560
        elif func == compile_and_run or func == multimod_compile_and_run:
561
562
563
564
565
566
            all_ways = config.run_ways
        elif func == ghci_script:
            if 'ghci' in config.run_ways:
                all_ways = ['ghci']
            else:
                all_ways = []
567
        else:
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
            all_ways = ['normal']

        # A test itself can request extra ways by setting opts.extra_ways
        all_ways = all_ways + filter(lambda way: way not in all_ways,
                                     opts.extra_ways)

        t.total_test_cases = t.total_test_cases + len(all_ways)

        ok_way = lambda way: \
            not getTestOpts().skip \
            and (config.only == [] or name in config.only) \
            and (getTestOpts().only_ways == [] or way in getTestOpts().only_ways) \
            and (config.cmdline_ways == [] or way in config.cmdline_ways) \
            and way not in getTestOpts().omit_ways

        # Which ways we are asked to skip
        do_ways = filter (ok_way,all_ways)

        # In fast mode, we skip all but one way
        if config.fast and len(do_ways) > 0:
            do_ways = [do_ways[0]]

        if not config.clean_only:
            # Run the required tests...
            for way in do_ways:
                do_test (name, way, func, args)

            for way in all_ways:
                if way not in do_ways:
                    skiptest (name,way)

        if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
            clean(map (lambda suff: name + suff,
                      ['', '.exe', '.exe.manifest', '.genscript',
                       '.stderr.normalised',        '.stdout.normalised',
                       '.run.stderr',               '.run.stdout',
                       '.run.stderr.normalised',    '.run.stdout.normalised',
                       '.comp.stderr',              '.comp.stdout',
                       '.comp.stderr.normalised',   '.comp.stdout.normalised',
                       '.interp.stderr',            '.interp.stdout',
                       '.interp.stderr.normalised', '.interp.stdout.normalised',
                       '.stats', '.comp.stats',
                       '.hi', '.o', '.prof', '.exe.prof', '.hc',
                       '_stub.h', '_stub.c', '_stub.o',
                       '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))

614
            if func == multi_compile or func == multi_compile_fail:
615
616
617
618
619
                    extra_mods = args[1]
                    clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
                    clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))

            clean(getTestOpts().clean_files)
Ian Lynagh's avatar
Ian Lynagh committed
620

621
622
623
624
625
626
627
628
            try:
                cleanCmd = getTestOpts().clean_cmd
                if cleanCmd != None:
                    result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
                    if result != 0:
                        framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
            except e:
                framework_fail(name, 'cleaning', 'clean-command exception')
629

630
        package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
631

632
633
        if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
            framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
634

635
636
637
638
639
640
641
642
643
644
645
646
        try:
            for f in files_written[name]:
                if os.path.exists(f):
                    try:
                        if not f in files_written_not_removed[name]:
                            files_written_not_removed[name].append(f)
                    except:
                        files_written_not_removed[name] = [f]
        except:
            pass
    except Exception, e:
        framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
Ian Lynagh's avatar
Ian Lynagh committed
647

648
649
650
651
def clean(strs):
    for str in strs:
        for name in glob.glob(in_testdir(str)):
            clean_full_path(name)
652

653
def clean_full_path(name):
Ian Lynagh's avatar
Ian Lynagh committed
654
655
656
657
        try:
            # Remove files...
            os.remove(name)
        except OSError, e1:
658
            try:
Ian Lynagh's avatar
Ian Lynagh committed
659
660
661
662
663
664
665
666
667
668
669
                # ... and empty directories
                os.rmdir(name)
            except OSError, e2:
                # We don't want to fail here, but we do want to know
                # what went wrong, so print out the exceptions.
                # ENOENT isn't a problem, though, as we clean files
                # that don't necessarily exist.
                if e1.errno != errno.ENOENT:
                    print e1
                if e2.errno != errno.ENOENT:
                    print e2
670

671
def do_test(name, way, func, args):
672
673
674
    full_name = name + '(' + way + ')'

    try:
675
676
677
678
        print '=====>', full_name, t.total_tests, 'of', len(allTests), \
                        str([t.n_unexpected_passes,   \
                             t.n_unexpected_failures, \
                             t.n_framework_failures])
679

680
681
        if config.use_threads:
            t.lock.release()
682
683
684
685

        try:
            preCmd = getTestOpts().pre_cmd
            if preCmd != None:
686
                result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
687
688
689
690
691
                if result != 0:
                    framework_fail(name, way, 'pre-command failed: ' + str(result))
        except e:
            framework_fail(name, way, 'pre-command exception')

ei@vuokko.info's avatar
ei@vuokko.info committed
692
693
694
        try:
            result = apply(func, [name,way] + args)
        finally:
695
696
            if config.use_threads:
                t.lock.acquire()
697

698
699
        if getTestOpts().expect != 'pass' and getTestOpts().expect != 'fail':
            framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
700

701
702
703
704
705
706
        try:
            passFail = result['passFail']
        except:
            passFail = 'No passFail found'

        if passFail == 'pass':
ei@vuokko.info's avatar
ei@vuokko.info committed
707
708
            if getTestOpts().expect == 'pass' \
               and way not in getTestOpts().expect_fail_for:
709
                t.n_expected_passes = t.n_expected_passes + 1
710
711
712
713
                if name in t.expected_passes:
                    t.expected_passes[name].append(way)
                else:
                    t.expected_passes[name] = [way]
714
715
716
            else:
                print '*** unexpected pass for', full_name
                t.n_unexpected_passes = t.n_unexpected_passes + 1
717
                addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
718
        elif passFail == 'fail':
ei@vuokko.info's avatar
ei@vuokko.info committed
719
720
            if getTestOpts().expect == 'pass' \
               and way not in getTestOpts().expect_fail_for:
721
722
                print '*** unexpected failure for', full_name
                t.n_unexpected_failures = t.n_unexpected_failures + 1
723
724
                reason = result['reason']
                addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
725
726
            else:
                t.n_expected_failures = t.n_expected_failures + 1
727
728
729
730
                if name in t.expected_failures:
                    t.expected_failures[name].append(way)
                else:
                    t.expected_failures[name] = [way]
731
732
        else:
            framework_fail(name, way, 'bad result ' + passFail)
733
    except:
734
        framework_fail(name, way, 'do_test exception')
735
736
        traceback.print_exc()

737
def addPassingTestInfo (testInfos, directory, name, way):
738
739
740
741
742
743
744
745
746
747
    directory = re.sub('^\\.[/\\\\]', '', directory)

    if not directory in testInfos:
        testInfos[directory] = {}

    if not name in testInfos[directory]:
        testInfos[directory][name] = []

    testInfos[directory][name].append(way)

748
749
750
751
752
753
754
755
756
757
758
759
760
761
def addFailingTestInfo (testInfos, directory, name, reason, way):
    directory = re.sub('^\\.[/\\\\]', '', directory)

    if not directory in testInfos:
        testInfos[directory] = {}

    if not name in testInfos[directory]:
        testInfos[directory][name] = {}

    if not reason in testInfos[directory][name]:
        testInfos[directory][name][reason] = []

    testInfos[directory][name][reason].append(way)

762
def skiptest (name, way):
763
764
    # print 'Skipping test \"', name, '\"'
    t.n_tests_skipped = t.n_tests_skipped + 1
765
766
767
768
    if name in t.tests_skipped:
        t.tests_skipped[name].append(way)
    else:
        t.tests_skipped[name] = [way]
769

770
771
772
def framework_fail( name, way, reason ):
    full_name = name + '(' + way + ')'
    print '*** framework failure for', full_name, reason, ':'
773
    t.n_framework_failures = t.n_framework_failures + 1
774
775
776
777
    if name in t.framework_failures:
        t.framework_failures[name].append(way)
    else:
        t.framework_failures[name] = [way]
778

779
780
781
782
783
784
785
786
787
788
789
790
791
792
def badResult(result):
    try:
        if result['passFail'] == 'pass':
            return False
        return True
    except:
        return True

def passed():
    return {'passFail': 'pass'}

def failBecause(reason):
    return {'passFail': 'fail', 'reason': reason}

793
794
795
796
# -----------------------------------------------------------------------------
# Generic command tests

# A generic command test is expected to run and exit successfully.
797
798
799
800
801
802
#
# The expected exit code can be changed via exit_code() as normal, and
# the expected stdout/stderr are stored in <testname>.stdout and
# <testname>.stderr.  The output of the command can be ignored
# altogether by using run_command_ignore_output instead of
# run_command.
803

804
def run_command( name, way, cmd ):
805
    return simple_run( name, '', cmd, '' )
806

807
808
809
810
# -----------------------------------------------------------------------------
# GHCi tests

def ghci_script( name, way, script ):
811
    # filter out -fforce-recomp from compiler_always_flags, because we're
812
    # actually testing the recompilation behaviour in the GHCi tests.
813
    flags = filter(lambda f: f != '-fforce-recomp', config.compiler_always_flags)
814
    flags.append(getTestOpts().extra_hc_opts)
815
816
817
818
819
820

    # We pass HC and HC_OPTS as environment variables, so that the
    # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
    cmd = "HC='" + config.compiler + "' " + \
          "HC_OPTS='" + join(flags,' ') + "' " + \
          "'" + config.compiler + "'" + \
821
          ' --interactive -v0 -ignore-dot-ghci ' + \
822
823
          join(flags,' ')

ei@vuokko.info's avatar
ei@vuokko.info committed
824
    getTestOpts().stdin = script
825
    return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
826

827
828
829
830
# -----------------------------------------------------------------------------
# Compile-only tests

def compile( name, way, extra_hc_opts ):
831
    return do_compile( name, way, 0, '', [], extra_hc_opts )
832
833

def compile_fail( name, way, extra_hc_opts ):
834
    return do_compile( name, way, 1, '', [], extra_hc_opts )
835
836

def multimod_compile( name, way, top_mod, extra_hc_opts ):
837
    return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
838

839
def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
840
841
842
843
844
845
846
847
848
    return do_compile( name, way, 1, top_mod, [], extra_hc_opts )

def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
    return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)

def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
    return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)

def do_compile( name, way, should_fail, top_mod, extra_mods, extra_hc_opts ):
849
850
    # print 'Compile only, extra args = ', extra_hc_opts
    pretest_cleanup(name)
851
852
853
854
855
856

    result = extras_build( way, extra_mods, extra_hc_opts )
    if badResult(result):
       return result
    extra_hc_opts = result['hc_opts']

857
858
859
    force = 0
    if extra_mods:
       force = 1
860
861
    result = simple_build( name, way, extra_hc_opts, should_fail, top_mod, 0, 1, force)

862
    if badResult(result):
Ian Lynagh's avatar
Ian Lynagh committed
863
        return result
864
865
866
867
868

    # the actual stderr should always match the expected, regardless
    # of whether we expected the compilation to fail or not (successful
    # compilations may generate warnings).

Ian Lynagh's avatar
Ian Lynagh committed
869
    if getTestOpts().with_namebase == None:
870
871
        namebase = name
    else:
Ian Lynagh's avatar
Ian Lynagh committed
872
        namebase = getTestOpts().with_namebase
873
874

    (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
875
876
    actual_stderr_file = qualify(name, 'comp.stderr')

877
878
    if not compare_outputs('stderr', \
                           two_normalisers(two_normalisers(getTestOpts().extra_errmsg_normaliser, normalise_errmsg), normalise_whitespace), \
879
                           expected_stderr_file, actual_stderr_file):
880
        return failBecause('stderr mismatch')
881
882

    # no problems found, this test passed
883
    return passed()
884
885
886
887

# -----------------------------------------------------------------------------
# Compile-and-run tests

888
def compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts ):
889
890
891
    # print 'Compile and run, extra args = ', extra_hc_opts
    pretest_cleanup(name)

892
893
894
895
    result = extras_build( way, extra_mods, extra_hc_opts )
    if badResult(result):
       return result
    extra_hc_opts = result['hc_opts']
dterei's avatar
dterei committed
896

897
    if way == 'ghci': # interpreted...
898
        return interpreter_run( name, way, extra_hc_opts, 0, top_mod )
krc's avatar
krc committed
899
    elif way == 'extcore' or way == 'optextcore' :
900
        return extcore_run( name, way, extra_hc_opts, 0, top_mod )
901
    else: # compiled...
dterei's avatar
dterei committed
902
903
904
905
        force = 0
        if extra_mods:
           force = 1

906
        result = simple_build( name, way, extra_hc_opts, 0, top_mod, 1, 1, force)
907
        if badResult(result):
Ian Lynagh's avatar
Ian Lynagh committed
908
            return result
909

910
911
        cmd = './' + name;

912
        # we don't check the compiler's stderr for a compile-and-run test
913
914
        return simple_run( name, way, cmd, getTestOpts().extra_run_opts )

915
def compile_and_run( name, way, extra_hc_opts ):
916
    return compile_and_run__( name, way, '', [], extra_hc_opts)
917
918

def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
919
    return compile_and_run__( name, way, top_mod, [], extra_hc_opts)
dterei's avatar
dterei committed
920

921
922
def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
    return compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
923

Ian Lynagh's avatar
Ian Lynagh committed
924
925
926
927
# -----------------------------------------------------------------------------
# Check -t stats info

def checkStats(stats_file, num_fields):
928
    result = passed()
929
    if len(num_fields) > 0:
Ian Lynagh's avatar
Ian Lynagh committed
930
931
932
933
        f = open(in_testdir(stats_file))
        contents = f.read()
        f.close()

934
        for (field, (min, max)) in num_fields.items():
Ian Lynagh's avatar
Ian Lynagh committed
935
936
937
            m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
            if m == None:
                print 'Failed to find field: ', field
938
                result = failBecause('no such stats field')
Ian Lynagh's avatar
Ian Lynagh committed
939
940
941
942
943
944
            val = int(m.group(1))

            if val < min:
                print field, val, 'is less than minimum allowed', min
                print 'If this is because you have improved GHC, please'
                print 'update the test so that GHC doesn\'t regress again'
945
                result = failBecause('stat too good')
Ian Lynagh's avatar
Ian Lynagh committed
946
947
            if val > max:
                print field, val, 'is more than maximum allowed', max
948
                result = failBecause('stat not good enough')
Ian Lynagh's avatar
Ian Lynagh committed
949

950
    return result
Ian Lynagh's avatar
Ian Lynagh committed
951

952
953
954
# -----------------------------------------------------------------------------
# Build a single-module program

955
956
957
def extras_build( way, extra_mods, extra_hc_opts ):
    for modopts in extra_mods:
        mod, opts = modopts
958
959
960
        result = simple_build( mod, way, opts + ' ' + extra_hc_opts, 0, '', 0, 0, 0)
        if not (mod.endswith('.hs') or mod.endswith('.lhs')):
            extra_hc_opts += ' ' + replace_suffix(mod, 'o')
961
962
963
964
965
966
967
        if badResult(result):
            return result

    return {'passFail' : 'pass', 'hc_opts' : extra_hc_opts}


def simple_build( name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, noforce ):
Ian Lynagh's avatar
Ian Lynagh committed
968
    opts = getTestOpts()
969
    errname = add_suffix(name, 'comp.stderr')
970
971
    rm_no_fail( qualify(errname, '') )

972
973
    if top_mod != '':
        srcname = top_mod
974
975
976
977
        rm_no_fail( qualify(name, '') )
        base, suf = os.path.splitext(top_mod)
        rm_no_fail( qualify(base, '') )
        rm_no_fail( qualify(base, 'exe') )
dterei's avatar
dterei committed
978
    elif addsuf:
Ian Lynagh's avatar
Ian Lynagh committed
979
        srcname = add_hs_lhs_suffix(name)
980
        rm_no_fail( qualify(name, '') )
dterei's avatar
dterei committed
981
982
    else:
        srcname = name
983
984
985
        rm_no_fail( qualify(name, 'o') )

    rm_no_fail( qualify(replace_suffix(srcname, "o"), '') )
986
987
988

    to_do = ''
    if top_mod != '':
989
990
991
        to_do = '--make '
        if link:
            to_do = to_do + '-o ' + name
992
993
    elif link:
        to_do = '-o ' + name
Ian Lynagh's avatar
Ian Lynagh committed
994
    elif opts.compile_to_hc:
995
996
997
998
        to_do = '-C'
    else:
        to_do = '-c' # just compile

Ian Lynagh's avatar
Ian Lynagh committed
999
    stats_file = name + '.comp.stats'
1000
    if len(opts.compiler_stats_num_fields) > 0:
1001
        extra_hc_opts += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1002

Ian Lynagh's avatar
Ian Lynagh committed
1003
1004
1005
1006
1007
    if getTestOpts().compile_cmd_prefix == '':
        cmd_prefix = ''
    else:
        cmd_prefix = getTestOpts().compile_cmd_prefix + ' '

1008
1009
1010
1011
    comp_flags = config.compiler_always_flags
    if noforce:
        comp_flags = filter(lambda f: f != '-fforce-recomp', comp_flags)

Ian Lynagh's avatar
Ian Lynagh committed
1012
    cmd = 'cd ' + getTestOpts().testdir + " && " + cmd_prefix + "'" \
1013
          + config.compiler + "' " \
1014
          + join(comp_flags,' ') + ' ' \
1015
1016
          + to_do + ' ' + srcname + ' ' \
          + join(config.way_flags[way],' ') + ' ' \
1017
          + extra_hc_opts + ' ' \
Ian Lynagh's avatar
Ian Lynagh committed
1018
          + opts.extra_hc_opts + ' ' \
1019
          + '>' + errname + ' 2>&1'
1020

1021
    result = runCmdFor(name, cmd)
1022
1023
1024
1025

    if result != 0 and not should_fail:
        actual_stderr = qualify(name, 'comp.stderr')
        if_verbose(1,'Compile failed (status ' + `result` + ') errors were:')
Ian Lynagh's avatar
Ian Lynagh committed
1026
        if_verbose_dump(1,actual_stderr)
1027
1028
1029

    # ToDo: if the sub-shell was killed by ^C, then exit

1030
    statsResult = checkStats(stats_file, opts.compiler_stats_num_fields)
Ian Lynagh's avatar
Ian Lynagh committed
1031

1032
1033
    if badResult(statsResult):
        return statsResult
Ian Lynagh's avatar
Ian Lynagh committed
1034
1035
1036

    if should_fail:
        if result == 0:
1037
            return failBecause('exit code 0')
Ian Lynagh's avatar
Ian Lynagh committed
1038
1039
    else:
        if result != 0:
1040
            return failBecause('exit code non-0')
Ian Lynagh's avatar
Ian Lynagh committed
1041

1042
    return passed()
1043
1044
1045
1046
1047

# -----------------------------------------------------------------------------
# Run a program and check its output
#
# If testname.stdin exists, route input from that, else
1048
# from /dev/null.  Route output to testname.run.stdout and
1049
1050
# testname.run.stderr.  Returns the exit code of the run.

1051
def simple_run( name, way, prog, args ):
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
    opts = getTestOpts()

    # figure out what to use for stdin
    if opts.stdin != '':
        use_stdin = opts.stdin
    else:
        stdin_file = add_suffix(name, 'stdin')
        if os.path.exists(in_testdir(stdin_file)):
            use_stdin = stdin_file
        else:
            use_stdin = '/dev/null'

    run_stdout = add_suffix(name,'run.stdout')
    run_stderr = add_suffix(name,'run.stderr')

    rm_no_fail(qualify(name,'run.stdout'))
    rm_no_fail(qualify(name,'run.stderr'))
    rm_no_fail(qualify(name, 'hp'))
    rm_no_fail(qualify(name,'ps'))
    rm_no_fail(qualify(name, 'prof'))
1072

1073
1074
    my_rts_flags = rts_flags(way)

Ian Lynagh's avatar
Ian Lynagh committed
1075
    stats_file = name + '.stats'
1076
    if len(opts.stats_num_fields) > 0:
1077
        args += ' +RTS -V0 -t' + stats_file + ' --machine-readable -RTS'
1078

1079
1080
1081
1082
    if opts.no_stdin:
        stdin_comes_from = ''
    else:
        stdin_comes_from = ' <' + use_stdin
1083
    cmd = prog + ' ' + args + ' '  \
1084
1085
1086
1087
1088
        + my_rts_flags + ' '       \
        + stdin_comes_from         \
        + ' >' + run_stdout        \
        + ' 2>' + run_stderr

1089
1090
1091
1092
1093
    if getTestOpts().cmd_wrapper != None:
        cmd = getTestOpts().cmd_wrapper(cmd);

    cmd = 'cd ' + getTestOpts().testdir + ' && ' + cmd

1094
    # run the command
1095
    result = runCmdFor(name, cmd)
1096
1097
1098
1099
1100
1101
1102
1103
1104

    exit_code = result >> 8
    signal    = result & 0xff

    # check the exit code
    if exit_code != opts.exit_code:
        print 'Wrong exit code (expected', opts.exit_code, ', actual', exit_code, ')'
        dump_stdout(name)
        dump_stderr(name)
1105
        return failBecause('bad exit code')
1106
1107
1108
1109

    check_hp = my_rts_flags.find("-h") != -1
    check_prof = my_rts_flags.find("-p") != -1

1110
1111
    if not opts.ignore_output:
        if not check_stderr_ok(name):
1112
            return failBecause('bad stderr')
1113
        if not check_stdout_ok(name):
1114
            return failBecause('bad stdout')
1115
        # exit_code > 127 probably indicates a crash, so don't try to run hp2ps.
1116
        if check_hp and (exit_code <= 127 or exit_code == 251) and not check_hp_ok(name):
1117
            return failBecause('bad heap profile')
1118
        if check_prof and not check_prof_ok(name):
1119
            return failBecause('bad profile')
1120

1121
    return checkStats(stats_file, opts.stats_num_fields)
1122

1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
def rts_flags(way):
    if (way == ''):
        return ''
    else:
        args = config.way_rts_flags[way]

    if args == []:
        return ''
    else:
        return '+RTS ' + join(args,' ') + ' -RTS'

1134
1135
1136
# -----------------------------------------------------------------------------
# Run a program in the interpreter and check its output

1137
def interpreter_run( name, way, extra_hc_opts, compile_only, top_mod ):
1138
1139
1140
1141
1142
    outname = add_suffix(name, 'interp.stdout')
    errname = add_suffix(name, 'interp.stderr')
    rm_no_fail(outname)
    rm_no_fail(errname)
    rm_no_fail(name)
1143

1144
    if (top_mod == ''):
Ian Lynagh's avatar
Ian Lynagh committed
1145
        srcname = add_hs_lhs_suffix(name)
1146
1147
    else:
        srcname = top_mod
1148

1149
    scriptname = add_suffix(name, 'genscript')
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
    qscriptname = in_testdir(scriptname)
    rm_no_fail(qscriptname)

    delimiter = '===== program output begins here\n'

    script = open(qscriptname, 'w')
    if not compile_only:
        # set the prog name and command-line args to match the compiled
        # environment.
        script.write(':set prog ' + name + '\n')
ei@vuokko.info's avatar
ei@vuokko.info committed
1160
        script.write(':set args ' + getTestOpts().extra_run_opts + '\n')
1161
1162
1163
1164
        # Add marker lines to the stdout and stderr output files, so we
        # can separate GHCi's output from the program's.
        script.write(':! echo ' + delimiter)
        script.write(':! echo 1>&2 ' + delimiter)
1165
1166
        # Set stdout to be line-buffered to match the compiled environment.
        script.write('System.IO.hSetBuffering System.IO.stdout System.IO.LineBuffering\n')
1167
1168
        # wrapping in GHC.TopHandler.runIO ensures we get the same output
        # in the event of an exception as for the compiled program.
1169
        script.write('GHC.TopHandler.runIOFastExit Main.main Prelude.>> Prelude.return ()\n')
simonmar's avatar