testlib.py 73.2 KB
Newer Older
1
#
2
3
4
# (c) Simon Marlow 2002
#

5
6
7
# This allows us to use the "with X:" syntax with python 2.5:
from __future__ import with_statement

8
import shutil
9
10
import sys
import os
11
import errno
12
13
14
15
import string
import re
import traceback
import copy
16
import glob
17
import types
ian@well-typed.com's avatar
ian@well-typed.com committed
18
from math import ceil, trunc
19

20
21
22
23
24
25
26
have_subprocess = False
try:
    import subprocess
    have_subprocess = True
except:
    print "Warning: subprocess not found, will fall back to spawnv"

27
from string import join
28
from testglobals import *
29
30
from testutil import *

31
32
33
if config.use_threads:
    import threading
    import thread
34

35
36
37
38
39
40
41
42
global wantToStop
wantToStop = False
def stopNow():
    global wantToStop
    wantToStop = True
def stopping():
    return wantToStop

43
44
# Options valid for the current test only (these get reset to
# testdir_testopts after each test).
45

ei@vuokko.info's avatar
ei@vuokko.info committed
46
global testopts_local
47
48
49
50
51
52
if config.use_threads:
    testopts_local = threading.local()
else:
    class TestOpts_Local:
        pass
    testopts_local = TestOpts_Local()
53
54

def getTestOpts():
ei@vuokko.info's avatar
ei@vuokko.info committed
55
    return testopts_local.x
56

ei@vuokko.info's avatar
ei@vuokko.info committed
57
58
59
def setLocalTestOpts(opts):
    global testopts_local
    testopts_local.x=opts
60
61
62
63

# This can be called at the top of a file of tests, to set default test options
# for the following tests.
def setTestOpts( f ):
64
65
    global thisdir_settings
    thisdir_settings = compose(thisdir_settings, f)
66
67
68
69
70
71
72
73
74
75
76
77

# -----------------------------------------------------------------------------
# Canned setup functions for common cases.  eg. for a test you might say
#
#      test('test001', normal, compile, [''])
#
# to run it without any options, but change it to
#
#      test('test001', expect_fail, compile, [''])
#
# to expect failure for this test.

78
def normal( name, opts ):
79
80
    return;

81
def skip( name, opts ):
82
83
    opts.skip = 1

84
def expect_fail( name, opts ):
85
86
    opts.expect = 'fail';

87
def reqlib( lib ):
88
    return lambda name, opts, l=lib: _reqlib (name, opts, l )
89

90
91
92
93
# Cache the results of looking to see if we have a library or not.
# This makes quite a difference, especially on Windows.
have_lib = {}

94
def _reqlib( name, opts, lib ):
95
96
    if have_lib.has_key(lib):
        got_it = have_lib[lib]
97
    else:
98
99
100
        if have_subprocess:
            # By preference we use subprocess, as the alternative uses
            # /dev/null which mingw doesn't have.
101
            p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
102
103
104
105
106
107
108
109
110
111
112
113
114
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
            # read from stdout and stderr to avoid blocking due to
            # buffers filling
            p.communicate()
            r = p.wait()
        else:
            r = os.system(config.ghc_pkg + ' describe ' + lib
                                         + ' > /dev/null 2> /dev/null')
        got_it = r == 0
        have_lib[lib] = got_it

    if not got_it:
115
        opts.expect = 'missing-lib'
116

117
def req_profiling( name, opts ):
118
119
120
    if not config.have_profiling:
        opts.expect = 'fail'

121
def req_shared_libs( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
122
123
124
    if not config.have_shared_libs:
        opts.expect = 'fail'

125
def req_interp( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
126
127
128
    if not config.have_interp:
        opts.expect = 'fail'

129
def req_smp( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
130
131
132
    if not config.have_smp:
        opts.expect = 'fail'

133
def ignore_output( name, opts ):
134
135
    opts.ignore_output = 1

136
def no_stdin( name, opts ):
137
138
    opts.no_stdin = 1

139
def combined_output( name, opts ):
pcapriotti's avatar
pcapriotti committed
140
141
    opts.combined_output = True

142
143
144
# -----

def expect_fail_for( ways ):
145
    return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
146

147
def _expect_fail_for( name, opts, ways ):
148
149
    opts.expect_fail_for = ways

150
151
152
153
def expect_broken( bug ):
    return lambda name, opts, b=bug: _expect_broken (name, opts, b )

def _expect_broken( name, opts, bug ):
154
    record_broken(name, bug)
155
156
    opts.expect = 'fail';

Ian Lynagh's avatar
Ian Lynagh committed
157
def expect_broken_for( bug, ways ):
158
    return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
Ian Lynagh's avatar
Ian Lynagh committed
159

160
def _expect_broken_for( name, opts, bug, ways ):
161
    record_broken(name, bug)
Ian Lynagh's avatar
Ian Lynagh committed
162
    opts.expect_fail_for = ways
Ian Lynagh's avatar
Ian Lynagh committed
163

164
165
166
167
168
169
def record_broken(name, bug):
    global brokens
    me = (bug, name)
    if not me in brokens:
        brokens.append(me)

170
171
172
# -----

def omit_ways( ways ):
173
    return lambda name, opts, w=ways: _omit_ways( name, opts, w )
174

175
def _omit_ways( name, opts, ways ):
176
177
178
179
    opts.omit_ways = ways

# -----

180
def only_ways( ways ):
181
    return lambda name, opts, w=ways: _only_ways( name, opts, w )
182

183
def _only_ways( name, opts, ways ):
184
185
186
187
    opts.only_ways = ways

# -----

188
def extra_ways( ways ):
189
    return lambda name, opts, w=ways: _extra_ways( name, opts, w )
190

191
def _extra_ways( name, opts, ways ):
192
193
194
195
    opts.extra_ways = ways

# -----

ross's avatar
ross committed
196
def omit_compiler_types( compiler_types ):
197
   return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
ross's avatar
ross committed
198

199
def _omit_compiler_types( name, opts, compiler_types ):
ross's avatar
ross committed
200
    if config.compiler_type in compiler_types:
dterei's avatar
dterei committed
201
        opts.skip = 1
ross's avatar
ross committed
202
203
204
205

# -----

def only_compiler_types( compiler_types ):
206
   return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
ross's avatar
ross committed
207

208
def _only_compiler_types( name, opts, compiler_types ):
ross's avatar
ross committed
209
    if config.compiler_type not in compiler_types:
dterei's avatar
dterei committed
210
        opts.skip = 1
ross's avatar
ross committed
211
212
213

# -----

214
def set_stdin( file ):
215
   return lambda name, opts, f=file: _set_stdin(name, opts, f);
216

217
def _set_stdin( name, opts, f ):
218
219
220
221
222
   opts.stdin = f

# -----

def exit_code( val ):
223
    return lambda name, opts, v=val: _exit_code(name, opts, v);
224

225
def _exit_code( name, opts, v ):
226
227
228
229
    opts.exit_code = v

# -----

230
def timeout_multiplier( val ):
231
    return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
232

233
def _timeout_multiplier( name, opts, v ):
234
235
236
237
    opts.timeout_multiplier = v

# -----

238
def extra_run_opts( val ):
239
    return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
240

241
def _extra_run_opts( name, opts, v ):
242
243
    opts.extra_run_opts = v

244
245
# -----

Simon Marlow's avatar
Simon Marlow committed
246
def extra_hc_opts( val ):
247
    return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
Simon Marlow's avatar
Simon Marlow committed
248

249
def _extra_hc_opts( name, opts, v ):
Simon Marlow's avatar
Simon Marlow committed
250
251
252
253
    opts.extra_hc_opts = v

# -----

254
def extra_clean( files ):
255
    return lambda name, opts, v=files: _extra_clean(name, opts, v);
256

257
def _extra_clean( name, opts, v ):
258
259
    opts.clean_files = v

260
261
# -----

262
263
264
265
266
267
268
def stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);

def _stats_num_field( name, opts, field, expecteds ):
    if field in opts.stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

269
270
271
272
273
274
    if type(expecteds) is types.ListType:
        for (b, expected, dev) in expecteds:
            if b:
                opts.stats_range_fields[field] = (expected, dev)
                return
        framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
275

276
277
278
    else:
        (expected, dev) = expecteds
        opts.stats_range_fields[field] = (expected, dev)
279
280
281

def compiler_stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
282

283
284
285
286
287
288
289
290
def _compiler_stats_num_field( name, opts, field, expecteds ):
    if field in opts.compiler_stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

    for (b, expected, dev) in expecteds:
        if b:
            opts.compiler_stats_range_fields[field] = (expected, dev)
            return
291

292
293
    framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')

294
295
# -----

296
def skip_if_no_ghci(name, opts):
dterei's avatar
dterei committed
297
298
    if not ('ghci' in config.run_ways):
        opts.skip = 1
299

300
301
# ----

302
def skip_if_fast(name, opts):
dterei's avatar
dterei committed
303
304
    if config.fast:
        opts.skip = 1
305

Simon Marlow's avatar
Simon Marlow committed
306
307
# -----

308
def when(b, f):
ian@well-typed.com's avatar
ian@well-typed.com committed
309
310
311
    # When list_brokens is on, we want to see all expect_broken calls,
    # so we always do f
    if b or config.list_broken:
312
313
314
315
316
317
318
        return f
    else:
        return normal

def unless(b, f):
    return when(not b, f)

319
320
def platform( plat ):
    return config.platform == plat
Ian Lynagh's avatar
Ian Lynagh committed
321

322
323
def opsys( os ):
    return config.os == os
Ian Lynagh's avatar
Ian Lynagh committed
324

325
326
def arch( arch ):
    return config.arch == arch
327

328
329
def wordsize( ws ):
    return config.wordsize == str(ws)
tibbe's avatar
tibbe committed
330

ian@well-typed.com's avatar
ian@well-typed.com committed
331
332
333
334
335
336
337
338
339
340
341
342
def if_unregisterised( f ):
    if config.unregisterised:
        return f
    else:
        return normal

def unless_unregisterised( f ):
    if config.unregisterised:
        return normal
    else:
        return f

Ian Lynagh's avatar
Ian Lynagh committed
343
344
345
346
347
348
349
350
351
352
353
354
def if_msys( f ):
    if config.msys:
        return f
    else:
        return normal

def if_cygwin( f ):
    if config.cygwin:
        return f
    else:
        return normal

355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
def when_have_vanilla( f ):
    if config.have_vanilla:
        return f
    else:
        return normal

def unless_have_vanilla( f ):
    if config.have_vanilla:
        return normal
    else:
        return f

def when_have_dynamic( f ):
    if config.have_dynamic:
        return f
    else:
        return normal

def unless_have_dynamic( f ):
    if config.have_dynamic:
        return normal
    else:
        return f

def when_have_profiling( f ):
    if config.have_profiling:
        return f
    else:
        return normal

def unless_have_profiling( f ):
    if config.have_profiling:
        return normal
    else:
        return f

391
392
# ---

393
394
395
396
397
398
def if_ghci_dynamic( f ):
    if config.ghc_dynamic_by_default:
        return f
    else:
        return normal

399
400
401
402
403
404
405
406
407
408
409
410
def if_in_tree_compiler( f ):
    if config.in_tree_compiler:
        return f
    else:
        return normal

def unless_in_tree_compiler( f ):
    if config.in_tree_compiler:
        return normal
    else:
        return f

411
412
413
414
415
def if_compiler_type( compiler, f ):
    if config.compiler_type == compiler:
        return f
    else:
        return normal
416
417
418
419
420
421

def if_compiler_profiled( f ):
    if config.compiler_profiled:
        return f
    else:
        return normal
422

423
424
425
426
427
428
def unless_compiler_profiled( f ):
    if config.compiler_profiled:
        return normal
    else:
        return f

Ian Lynagh's avatar
Ian Lynagh committed
429
def if_compiler_lt( compiler, version, f ):
430
    if config.compiler_type == compiler and \
Ian Lynagh's avatar
Ian Lynagh committed
431
432
433
434
       version_lt(config.compiler_version, version):
        return f
    else:
        return normal
435

Ian Lynagh's avatar
Ian Lynagh committed
436
def if_compiler_le( compiler, version, f ):
437
438
    if config.compiler_type == compiler and \
       version_le(config.compiler_version, version):
Ian Lynagh's avatar
Ian Lynagh committed
439
440
441
        return f
    else:
        return normal
442

Ian Lynagh's avatar
Ian Lynagh committed
443
def if_compiler_gt( compiler, version, f ):
444
    if config.compiler_type == compiler and \
Ian Lynagh's avatar
Ian Lynagh committed
445
446
447
448
       version_gt(config.compiler_version, version):
        return f
    else:
        return normal
449

Ian Lynagh's avatar
Ian Lynagh committed
450
def if_compiler_ge( compiler, version, f ):
451
    if config.compiler_type == compiler and \
Ian Lynagh's avatar
Ian Lynagh committed
452
453
454
455
       version_ge(config.compiler_version, version):
        return f
    else:
        return normal
456

457
458
459
460
461
462
def if_compiler_debugged( f ):
    if config.compiler_debugged:
        return f
    else:
        return normal

Ian Lynagh's avatar
Ian Lynagh committed
463
464
def namebase( nb ):
   return lambda opts, nb=nb: _namebase(opts, nb)
465

Ian Lynagh's avatar
Ian Lynagh committed
466
467
def _namebase( opts, nb ):
    opts.with_namebase = nb
468

469
470
# ---

471
def if_tag( tag, f ):
472
    if tag in config.compiler_tags:
473
474
475
        return f
    else:
        return normal
476

477
def unless_tag( tag, f ):
478
    if not (tag in config.compiler_tags):
479
480
481
        return f
    else:
        return normal
482

ei@vuokko.info's avatar
ei@vuokko.info committed
483
# ---
484
def high_memory_usage(name, opts):
485
486
    opts.alone = True

Ian Lynagh's avatar
Ian Lynagh committed
487
# ---
488
def literate( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
489
490
    opts.literate = 1;

491
def c_src( name, opts ):
492
493
    opts.c_src = 1;

494
def objc_src( name, opts ):
Austin Seipp's avatar
Austin Seipp committed
495
496
    opts.objc_src = 1;

497
def objcpp_src( name, opts ):
498
499
    opts.objcpp_src = 1;

500
def cmm_src( name, opts ):
501
502
    opts.cmm_src = 1;

503
def outputdir( odir ):
504
    return lambda name, opts, d=odir: _outputdir(name, opts, d)
505

506
def _outputdir( name, opts, odir ):
507
508
    opts.outputdir = odir;

509
510
# ----

511
def pre_cmd( cmd ):
512
    return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
513

514
def _pre_cmd( name, opts, cmd ):
515
516
517
518
    opts.pre_cmd = cmd

# ----

519
def clean_cmd( cmd ):
520
    return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
521

522
def _clean_cmd( name, opts, cmd ):
523
524
525
526
    opts.clean_cmd = cmd

# ----

527
def cmd_prefix( prefix ):
528
    return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
529

530
def _cmd_prefix( name, opts, prefix ):
531
532
533
534
535
    opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;

# ----

def cmd_wrapper( fun ):
536
    return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
537

538
def _cmd_wrapper( name, opts, fun ):
539
    opts.cmd_wrapper = fun
540

541
542
# ----

Ian Lynagh's avatar
Ian Lynagh committed
543
def compile_cmd_prefix( prefix ):
544
    return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
Ian Lynagh's avatar
Ian Lynagh committed
545

546
def _compile_cmd_prefix( name, opts, prefix ):
Ian Lynagh's avatar
Ian Lynagh committed
547
548
549
550
    opts.compile_cmd_prefix = prefix

# ----

551
def normalise_slashes( name, opts ):
552
553
    opts.extra_normaliser = normalise_slashes_

554
def normalise_exe( name, opts ):
555
556
    opts.extra_normaliser = normalise_exe_

557
def normalise_fun( fun ):
558
    return lambda name, opts, f=fun: _normalise_fun(name, opts, f)
559

560
def _normalise_fun( name, opts, f ):
561
562
    opts.extra_normaliser = f

563
def normalise_errmsg_fun( fun ):
564
    return lambda name, opts, f=fun: _normalise_errmsg_fun(name, opts, f)
565

566
def _normalise_errmsg_fun( name, opts, f ):
567
568
569
570
571
    opts.extra_errmsg_normaliser = f

def two_normalisers(f, g):
    return lambda x, f=f, g=g: f(g(x))

572
573
574
# ----
# Function for composing two opt-fns together

575
576
577
def composes( fs ):
    return reduce(lambda f, g: compose(f, g), fs)

578
def compose( f, g ):
579
    return lambda name, opts, f=f, g=g: _compose(name, opts, f, g)
580

581
582
583
def _compose( name, opts, f, g ):
    f(name, opts)
    g(name, opts)
584
585
586
587
588

# -----------------------------------------------------------------------------
# The current directory of tests

def newTestDir( dir ):
589
    global thisdir_settings
590
    # reset the options for this test directory
591
592
593
594
595
    thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )

def _newTestDir( name, opts, dir ):
    opts.testdir = dir
    opts.compiler_always_flags = config.compiler_always_flags
596
597
598
599

# -----------------------------------------------------------------------------
# Actually doing tests

600
601
parallelTests = []
aloneTests = []
602
allTestNames = set([])
603

604
def runTest (opts, name, func, args):
ei@vuokko.info's avatar
ei@vuokko.info committed
605
606
    ok = 0

607
    if config.use_threads:
ei@vuokko.info's avatar
ei@vuokko.info committed
608
        t.thread_pool.acquire()
609
        try:
610
            while config.threads<(t.running_threads+1):
611
                t.thread_pool.wait()
612
            t.running_threads = t.running_threads+1
613
614
            ok=1
            t.thread_pool.release()
615
            thread.start_new_thread(test_common_thread, (name, opts, func, args))
616
617
618
619
620
        except:
            if not ok:
                t.thread_pool.release()
    else:
        test_common_work (name, opts, func, args)
621

622
# name  :: String
623
# setup :: TestOpts -> IO ()
624
def test (name, setup, func, args):
625
626
    global aloneTests
    global parallelTests
627
    global allTestNames
628
    global thisdir_settings
629
630
    if name in allTestNames:
        framework_fail(name, 'duplicate', 'There are multiple tests with this name')
631
    if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
632
        framework_fail(name, 'bad_name', 'This test has an invalid name')
633
634
635
636
637

    # Make a deep copy of the default_testopts, as we need our own copy
    # of any dictionaries etc inside it. Otherwise, if one test modifies
    # them, all tests will see the modified version!
    myTestOpts = copy.deepcopy(default_testopts)
638
639
640
641

    if type(setup) is types.ListType:
       setup = composes(setup)

642
643
    setup = compose(thisdir_settings, setup)
    setup(name, myTestOpts)
644
645
646
647
648
649

    thisTest = lambda : runTest(myTestOpts, name, func, args)
    if myTestOpts.alone:
        aloneTests.append(thisTest)
    else:
        parallelTests.append(thisTest)
650
    allTestNames.add(name)
651

652
if config.use_threads:
653
    def test_common_thread(name, opts, func, args):
654
655
656
657
658
659
        t.lock.acquire()
        try:
            test_common_work(name,opts,func,args)
        finally:
            t.lock.release()
            t.thread_pool.acquire()
660
            t.running_threads = t.running_threads - 1
661
662
            t.thread_pool.notify()
            t.thread_pool.release()
663

664
665
666
667
668
669
670
671
672
673
def get_package_cache_timestamp():
    if config.package_conf_cache_file == '':
        return 0.0
    else:
        try:
            return os.stat(config.package_conf_cache_file).st_mtime
        except:
            return 0.0


ei@vuokko.info's avatar
ei@vuokko.info committed
674
def test_common_work (name, opts, func, args):
675
676
677
678
679
680
681
682
683
    try:
        t.total_tests = t.total_tests+1
        setLocalTestOpts(opts)

        package_conf_cache_file_start_timestamp = get_package_cache_timestamp()

        # All the ways we might run this test
        if func == compile or func == multimod_compile:
            all_ways = config.compile_ways
684
        elif func == compile_and_run or func == multimod_compile_and_run:
685
686
687
688
689
690
            all_ways = config.run_ways
        elif func == ghci_script:
            if 'ghci' in config.run_ways:
                all_ways = ['ghci']
            else:
                all_ways = []
691
        else:
692
693
694
695
696
697
698
699
700
701
702
            all_ways = ['normal']

        # A test itself can request extra ways by setting opts.extra_ways
        all_ways = all_ways + filter(lambda way: way not in all_ways,
                                     opts.extra_ways)

        t.total_test_cases = t.total_test_cases + len(all_ways)

        ok_way = lambda way: \
            not getTestOpts().skip \
            and (config.only == [] or name in config.only) \
703
            and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
704
705
706
707
708
709
710
711
712
713
714
715
716
            and (config.cmdline_ways == [] or way in config.cmdline_ways) \
            and way not in getTestOpts().omit_ways

        # Which ways we are asked to skip
        do_ways = filter (ok_way,all_ways)

        # In fast mode, we skip all but one way
        if config.fast and len(do_ways) > 0:
            do_ways = [do_ways[0]]

        if not config.clean_only:
            # Run the required tests...
            for way in do_ways:
717
718
                if stopping():
                    break
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
                do_test (name, way, func, args)

            for way in all_ways:
                if way not in do_ways:
                    skiptest (name,way)

        if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
            clean(map (lambda suff: name + suff,
                      ['', '.exe', '.exe.manifest', '.genscript',
                       '.stderr.normalised',        '.stdout.normalised',
                       '.run.stderr',               '.run.stdout',
                       '.run.stderr.normalised',    '.run.stdout.normalised',
                       '.comp.stderr',              '.comp.stdout',
                       '.comp.stderr.normalised',   '.comp.stdout.normalised',
                       '.interp.stderr',            '.interp.stdout',
                       '.interp.stderr.normalised', '.interp.stdout.normalised',
                       '.stats', '.comp.stats',
                       '.hi', '.o', '.prof', '.exe.prof', '.hc',
                       '_stub.h', '_stub.c', '_stub.o',
                       '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))

740
            if func == multi_compile or func == multi_compile_fail:
741
742
743
744
745
                    extra_mods = args[1]
                    clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
                    clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))

            clean(getTestOpts().clean_files)
Ian Lynagh's avatar
Ian Lynagh committed
746

747
748
749
750
751
752
753
            if getTestOpts().outputdir != None:
                odir = in_testdir(getTestOpts().outputdir)
                try:
                    shutil.rmtree(odir)
                except:
                    pass

754
755
756
757
758
            try:
                shutil.rmtree(in_testdir('.hpc.' + name))
            except:
                pass

759
760
761
762
763
764
765
766
            try:
                cleanCmd = getTestOpts().clean_cmd
                if cleanCmd != None:
                    result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
                    if result != 0:
                        framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
            except e:
                framework_fail(name, 'cleaning', 'clean-command exception')
767

768
        package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
769

770
771
        if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
            framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
772

773
774
775
776
777
778
779
780
781
782
783
784
        try:
            for f in files_written[name]:
                if os.path.exists(f):
                    try:
                        if not f in files_written_not_removed[name]:
                            files_written_not_removed[name].append(f)
                    except:
                        files_written_not_removed[name] = [f]
        except:
            pass
    except Exception, e:
        framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
Ian Lynagh's avatar
Ian Lynagh committed
785

786
787
788
789
def clean(strs):
    for str in strs:
        for name in glob.glob(in_testdir(str)):
            clean_full_path(name)
790

791
def clean_full_path(name):
Ian Lynagh's avatar
Ian Lynagh committed
792
793
794
795
        try:
            # Remove files...
            os.remove(name)
        except OSError, e1:
796
            try:
Ian Lynagh's avatar
Ian Lynagh committed
797
798
799
800
801
802
803
804
805
806
807
                # ... and empty directories
                os.rmdir(name)
            except OSError, e2:
                # We don't want to fail here, but we do want to know
                # what went wrong, so print out the exceptions.
                # ENOENT isn't a problem, though, as we clean files
                # that don't necessarily exist.
                if e1.errno != errno.ENOENT:
                    print e1
                if e2.errno != errno.ENOENT:
                    print e2
808

809
def do_test(name, way, func, args):
810
811
812
    full_name = name + '(' + way + ')'

    try:
813
        print '=====>', full_name, t.total_tests, 'of', len(allTestNames), \
814
815
816
                        str([t.n_unexpected_passes,   \
                             t.n_unexpected_failures, \
                             t.n_framework_failures])
817

818
819
        if config.use_threads:
            t.lock.release()
820
821
822
823

        try:
            preCmd = getTestOpts().pre_cmd
            if preCmd != None:
824
                result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
825
826
827
828
829
                if result != 0:
                    framework_fail(name, way, 'pre-command failed: ' + str(result))
        except e:
            framework_fail(name, way, 'pre-command exception')

ei@vuokko.info's avatar
ei@vuokko.info committed
830
831
832
        try:
            result = apply(func, [name,way] + args)
        finally:
833
834
            if config.use_threads:
                t.lock.acquire()
835

836
837
838
        if getTestOpts().expect != 'pass' and \
                getTestOpts().expect != 'fail' and \
                getTestOpts().expect != 'missing-lib':
839
            framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
840

841
842
843
844
845
846
        try:
            passFail = result['passFail']
        except:
            passFail = 'No passFail found'

        if passFail == 'pass':
ei@vuokko.info's avatar
ei@vuokko.info committed
847
848
            if getTestOpts().expect == 'pass' \
               and way not in getTestOpts().expect_fail_for:
849
                t.n_expected_passes = t.n_expected_passes + 1
850
851
852
853
                if name in t.expected_passes:
                    t.expected_passes[name].append(way)
                else:
                    t.expected_passes[name] = [way]
854
855
856
            else:
                print '*** unexpected pass for', full_name
                t.n_unexpected_passes = t.n_unexpected_passes + 1
857
                addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
858
        elif passFail == 'fail':
ei@vuokko.info's avatar
ei@vuokko.info committed
859
860
            if getTestOpts().expect == 'pass' \
               and way not in getTestOpts().expect_fail_for:
861
862
                print '*** unexpected failure for', full_name
                t.n_unexpected_failures = t.n_unexpected_failures + 1
863
864
                reason = result['reason']
                addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
865
            else:
866
867
868
869
870
871
                if getTestOpts().expect == 'missing-lib':
                    t.n_missing_libs = t.n_missing_libs + 1
                    if name in t.missing_libs:
                        t.missing_libs[name].append(way)
                    else:
                        t.missing_libs[name] = [way]
872
                else:
873
874
875
876
877
                    t.n_expected_failures = t.n_expected_failures + 1
                    if name in t.expected_failures:
                        t.expected_failures[name].append(way)
                    else:
                        t.expected_failures[name] = [way]
878
879
        else:
            framework_fail(name, way, 'bad result ' + passFail)
880
    except KeyboardInterrupt:
881
        stopNow()
882
    except:
883
        framework_fail(name, way, 'do_test exception')
884
885
        traceback.print_exc()

886
def addPassingTestInfo (testInfos, directory, name, way):
887
888
889
890
891
892
893
894
895
896
    directory = re.sub('^\\.[/\\\\]', '', directory)

    if not directory in testInfos:
        testInfos[directory] = {}

    if not name in testInfos[directory]:
        testInfos[directory][name] = []

    testInfos[directory][name].append(way)

897
898
899
900
901
902
903
904
905
906
907
908
909
910
def addFailingTestInfo (testInfos, directory, name, reason, way):
    directory = re.sub('^\\.[/\\\\]', '', directory)

    if not directory in testInfos:
        testInfos[directory] = {}

    if not name in testInfos[directory]:
        testInfos[directory][name] = {}

    if not reason in testInfos[directory][name]:
        testInfos[directory][name][reason] = []

    testInfos[directory][name][reason].append(way)

911
def skiptest (name, way):
912
913
    # print 'Skipping test \"', name, '\"'
    t.n_tests_skipped = t.n_tests_skipped + 1
914
915
916
917
    if name in t.tests_skipped:
        t.tests_skipped[name].append(way)
    else:
        t.tests_skipped[name] = [way]
918

919
920
def framework_fail( name, way, reason ):
    full_name = name + '(' + way + ')'
921
    print '*** framework failure for', full_name, reason
922
    t.n_framework_failures = t.n_framework_failures + 1
923
924
925
926
    if name in t.framework_failures:
        t.framework_failures[name].append(way)
    else:
        t.framework_failures[name] = [way]
927

928
929
930
931
932
933
934
935
936
937
938
939
940
941
def badResult(result):
    try:
        if result['passFail'] == 'pass':
            return False
        return True
    except:
        return True

def passed():
    return {'passFail': 'pass'}

def failBecause(reason):
    return {'passFail': 'fail', 'reason': reason}

942
943
944
945
# -----------------------------------------------------------------------------
# Generic command tests

# A generic command test is expected to run and exit successfully.
946
947
948
949
950
951
#
# The expected exit code can be changed via exit_code() as normal, and
# the expected stdout/stderr are stored in <testname>.stdout and
# <testname>.stderr.  The output of the command can be ignored
# altogether by using run_command_ignore_output instead of
# run_command.
952

953
def run_command( name, way, cmd ):
954
    return simple_run( name, '', cmd, '' )
955

956
957
958
959
# -----------------------------------------------------------------------------
# GHCi tests

def ghci_script( name, way, script ):
960
    # filter out -fforce-recomp from compiler_always_flags, because we're
961
    # actually testing the recompilation behaviour in the GHCi tests.
962
    flags = filter(lambda f: f != '-fforce-recomp', getTestOpts().compiler_always_flags)
simonpj's avatar
simonpj committed
963
    flags.append(getTestOpts().extra_hc_opts)
964
965
    if getTestOpts().outputdir != None:
        flags.extend(["-outputdir", getTestOpts().outputdir])
966
967
968
969
970
971

    # We pass HC and HC_OPTS as environment variables, so that the
    # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
    cmd = "HC='" + config.compiler + "' " + \
          "HC_OPTS='" + join(flags,' ') + "' " + \
          "'" + config.compiler + "'" + \
972
          ' --interactive -v0 -ignore-dot-ghci ' + \
973
974
          join(flags,' ')

ei@vuokko.info's avatar
ei@vuokko.info committed
975
    getTestOpts().stdin = script
976
    return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
977

978
979
980
981
# -----------------------------------------------------------------------------
# Compile-only tests

def compile( name, way, extra_hc_opts ):
982
    return do_compile( name, way, 0, '', [], extra_hc_opts )
983
984

def compile_fail( name, way, extra_hc_opts ):
985
    return do_compile( name, way, 1, '', [], extra_hc_opts )
986
987

def multimod_compile( name, way, top_mod, extra_hc_opts ):
988
    return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
989

990
def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
991
992
993
994
995
996
997
998
999
    return do_compile( name, way, 1, top_mod, [], extra_hc_opts )

def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
    return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)

def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
    return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)

def do_compile( name, way, should_fail, top_mod, extra_mods, extra_hc_opts ):
1000
    # print 'Compile only, extra args = ', extra_hc_opts
For faster browsing, not all history is shown. View entire blame