testlib.py 69.7 KB
Newer Older
1
#
2
3
4
# (c) Simon Marlow 2002
#

5
6
7
# This allows us to use the "with X:" syntax with python 2.5:
from __future__ import with_statement

8
import shutil
9
10
import sys
import os
11
import errno
12
13
14
import string
import re
import traceback
15
16
import time
import datetime
17
import copy
18
import glob
19
import types
ian@well-typed.com's avatar
ian@well-typed.com committed
20
from math import ceil, trunc
21

22
23
24
25
26
27
28
have_subprocess = False
try:
    import subprocess
    have_subprocess = True
except:
    print "Warning: subprocess not found, will fall back to spawnv"

29
from string import join
30
from testglobals import *
31
32
from testutil import *

33
34
35
if config.use_threads:
    import threading
    import thread
36

37
38
39
40
41
42
43
44
global wantToStop
wantToStop = False
def stopNow():
    global wantToStop
    wantToStop = True
def stopping():
    return wantToStop

45
46
# Options valid for the current test only (these get reset to
# testdir_testopts after each test).
47

ei@vuokko.info's avatar
ei@vuokko.info committed
48
global testopts_local
49
50
51
52
53
54
if config.use_threads:
    testopts_local = threading.local()
else:
    class TestOpts_Local:
        pass
    testopts_local = TestOpts_Local()
55
56

def getTestOpts():
ei@vuokko.info's avatar
ei@vuokko.info committed
57
    return testopts_local.x
58

ei@vuokko.info's avatar
ei@vuokko.info committed
59
60
61
def setLocalTestOpts(opts):
    global testopts_local
    testopts_local.x=opts
62

63
64
65
66
67
def isStatsTest():
    opts = getTestOpts()
    return len(opts.compiler_stats_range_fields) > 0 or len(opts.stats_range_fields) > 0


68
69
70
# This can be called at the top of a file of tests, to set default test options
# for the following tests.
def setTestOpts( f ):
71
    global thisdir_settings
72
    thisdir_settings = [thisdir_settings, f]
73
74
75
76
77
78
79
80
81
82
83
84

# -----------------------------------------------------------------------------
# Canned setup functions for common cases.  eg. for a test you might say
#
#      test('test001', normal, compile, [''])
#
# to run it without any options, but change it to
#
#      test('test001', expect_fail, compile, [''])
#
# to expect failure for this test.

85
def normal( name, opts ):
86
87
    return;

88
def skip( name, opts ):
89
90
    opts.skip = 1

91
def expect_fail( name, opts ):
92
93
    opts.expect = 'fail';

94
def reqlib( lib ):
95
    return lambda name, opts, l=lib: _reqlib (name, opts, l )
96

97
98
99
100
# Cache the results of looking to see if we have a library or not.
# This makes quite a difference, especially on Windows.
have_lib = {}

101
def _reqlib( name, opts, lib ):
102
103
    if have_lib.has_key(lib):
        got_it = have_lib[lib]
104
    else:
105
106
107
        if have_subprocess:
            # By preference we use subprocess, as the alternative uses
            # /dev/null which mingw doesn't have.
108
            p = subprocess.Popen([config.ghc_pkg, '--no-user-package-db', 'describe', lib],
109
110
111
112
113
114
115
116
117
118
119
120
121
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
            # read from stdout and stderr to avoid blocking due to
            # buffers filling
            p.communicate()
            r = p.wait()
        else:
            r = os.system(config.ghc_pkg + ' describe ' + lib
                                         + ' > /dev/null 2> /dev/null')
        got_it = r == 0
        have_lib[lib] = got_it

    if not got_it:
122
        opts.expect = 'missing-lib'
123

124
def req_profiling( name, opts ):
125
126
127
    if not config.have_profiling:
        opts.expect = 'fail'

128
def req_shared_libs( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
129
130
131
    if not config.have_shared_libs:
        opts.expect = 'fail'

132
def req_interp( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
133
134
135
    if not config.have_interp:
        opts.expect = 'fail'

136
def req_smp( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
137
138
139
    if not config.have_smp:
        opts.expect = 'fail'

140
def ignore_output( name, opts ):
141
142
    opts.ignore_output = 1

143
def no_stdin( name, opts ):
144
145
    opts.no_stdin = 1

146
def combined_output( name, opts ):
pcapriotti's avatar
pcapriotti committed
147
148
    opts.combined_output = True

149
150
151
# -----

def expect_fail_for( ways ):
152
    return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
153

154
def _expect_fail_for( name, opts, ways ):
155
156
    opts.expect_fail_for = ways

157
158
159
160
def expect_broken( bug ):
    return lambda name, opts, b=bug: _expect_broken (name, opts, b )

def _expect_broken( name, opts, bug ):
161
    record_broken(name, opts, bug)
162
163
    opts.expect = 'fail';

Ian Lynagh's avatar
Ian Lynagh committed
164
def expect_broken_for( bug, ways ):
165
    return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
Ian Lynagh's avatar
Ian Lynagh committed
166

167
def _expect_broken_for( name, opts, bug, ways ):
168
    record_broken(name, opts, bug)
Ian Lynagh's avatar
Ian Lynagh committed
169
    opts.expect_fail_for = ways
Ian Lynagh's avatar
Ian Lynagh committed
170

171
def record_broken(name, opts, bug):
172
    global brokens
173
    me = (bug, opts.testdir, name)
174
175
176
    if not me in brokens:
        brokens.append(me)

177
178
179
# -----

def omit_ways( ways ):
180
    return lambda name, opts, w=ways: _omit_ways( name, opts, w )
181

182
def _omit_ways( name, opts, ways ):
183
184
185
186
    opts.omit_ways = ways

# -----

187
def only_ways( ways ):
188
    return lambda name, opts, w=ways: _only_ways( name, opts, w )
189

190
def _only_ways( name, opts, ways ):
191
192
193
194
    opts.only_ways = ways

# -----

195
def extra_ways( ways ):
196
    return lambda name, opts, w=ways: _extra_ways( name, opts, w )
197

198
def _extra_ways( name, opts, ways ):
199
200
201
202
    opts.extra_ways = ways

# -----

ross's avatar
ross committed
203
def omit_compiler_types( compiler_types ):
204
   return lambda name, opts, c=compiler_types: _omit_compiler_types(name, opts, c)
ross's avatar
ross committed
205

206
def _omit_compiler_types( name, opts, compiler_types ):
ross's avatar
ross committed
207
    if config.compiler_type in compiler_types:
dterei's avatar
dterei committed
208
        opts.skip = 1
ross's avatar
ross committed
209
210
211
212

# -----

def only_compiler_types( compiler_types ):
213
   return lambda name, opts, c=compiler_types: _only_compiler_types(name, opts, c)
ross's avatar
ross committed
214

215
def _only_compiler_types( name, opts, compiler_types ):
ross's avatar
ross committed
216
    if config.compiler_type not in compiler_types:
dterei's avatar
dterei committed
217
        opts.skip = 1
ross's avatar
ross committed
218
219
220

# -----

221
def set_stdin( file ):
222
   return lambda name, opts, f=file: _set_stdin(name, opts, f);
223

224
def _set_stdin( name, opts, f ):
225
226
227
228
229
   opts.stdin = f

# -----

def exit_code( val ):
230
    return lambda name, opts, v=val: _exit_code(name, opts, v);
231

232
def _exit_code( name, opts, v ):
233
234
235
236
    opts.exit_code = v

# -----

237
def timeout_multiplier( val ):
238
    return lambda name, opts, v=val: _timeout_multiplier(name, opts, v)
239

240
def _timeout_multiplier( name, opts, v ):
241
242
243
244
    opts.timeout_multiplier = v

# -----

245
def extra_run_opts( val ):
246
    return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
247

248
def _extra_run_opts( name, opts, v ):
249
250
    opts.extra_run_opts = v

251
252
# -----

Simon Marlow's avatar
Simon Marlow committed
253
def extra_hc_opts( val ):
254
    return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
Simon Marlow's avatar
Simon Marlow committed
255

256
def _extra_hc_opts( name, opts, v ):
Simon Marlow's avatar
Simon Marlow committed
257
258
259
260
    opts.extra_hc_opts = v

# -----

261
def extra_clean( files ):
262
    return lambda name, opts, v=files: _extra_clean(name, opts, v);
263

264
def _extra_clean( name, opts, v ):
265
266
    opts.clean_files = v

267
268
# -----

269
270
271
272
273
274
275
def stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);

def _stats_num_field( name, opts, field, expecteds ):
    if field in opts.stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

276
277
278
279
280
281
    if type(expecteds) is types.ListType:
        for (b, expected, dev) in expecteds:
            if b:
                opts.stats_range_fields[field] = (expected, dev)
                return
        framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
282

283
284
285
    else:
        (expected, dev) = expecteds
        opts.stats_range_fields[field] = (expected, dev)
286
287
288

def compiler_stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
289

290
291
292
293
def _compiler_stats_num_field( name, opts, field, expecteds ):
    if field in opts.compiler_stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

294
295
    # Compiler performance numbers change when debugging is on, making the results
    # useless and confusing. Therefore, skip if debugging is on.
296
297
298
    if compiler_debugged():
        skip(name, opts)

299
300
301
302
    for (b, expected, dev) in expecteds:
        if b:
            opts.compiler_stats_range_fields[field] = (expected, dev)
            return
303

304
305
    framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')

306
307
# -----

308
def when(b, f):
ian@well-typed.com's avatar
ian@well-typed.com committed
309
310
311
    # When list_brokens is on, we want to see all expect_broken calls,
    # so we always do f
    if b or config.list_broken:
312
313
314
315
316
317
318
        return f
    else:
        return normal

def unless(b, f):
    return when(not b, f)

ian@well-typed.com's avatar
ian@well-typed.com committed
319
320
321
def doing_ghci():
    return 'ghci' in config.run_ways

322
def ghci_dynamic( ):
ian@well-typed.com's avatar
ian@well-typed.com committed
323
    return config.ghc_dynamic
324
325
326
327

def fast():
    return config.fast

328
329
def platform( plat ):
    return config.platform == plat
Ian Lynagh's avatar
Ian Lynagh committed
330

331
332
def opsys( os ):
    return config.os == os
Ian Lynagh's avatar
Ian Lynagh committed
333

334
335
def arch( arch ):
    return config.arch == arch
336

337
338
def wordsize( ws ):
    return config.wordsize == str(ws)
tibbe's avatar
tibbe committed
339

340
341
def msys( ):
    return config.msys
ian@well-typed.com's avatar
ian@well-typed.com committed
342

343
344
def cygwin( ):
    return config.cygwin
Ian Lynagh's avatar
Ian Lynagh committed
345

346
347
def have_vanilla( ):
    return config.have_vanilla
348

349
350
def have_dynamic( ):
    return config.have_dynamic
351

352
353
def have_profiling( ):
    return config.have_profiling
354

355
356
def in_tree_compiler( ):
    return config.in_tree_compiler
357

358
359
def compiler_type( compiler ):
    return config.compiler_type == compiler
360

361
362
363
def compiler_lt( compiler, version ):
    return config.compiler_type == compiler and \
           version_lt(config.compiler_version, version)
364

365
366
367
def compiler_le( compiler, version ):
    return config.compiler_type == compiler and \
           version_le(config.compiler_version, version)
368

369
370
371
def compiler_gt( compiler, version ):
    return config.compiler_type == compiler and \
           version_gt(config.compiler_version, version)
372

373
374
375
def compiler_ge( compiler, version ):
    return config.compiler_type == compiler and \
           version_ge(config.compiler_version, version)
376

377
378
379
380
381
382
def unregisterised( ):
    return config.unregisterised

def compiler_profiled( ):
    return config.compiler_profiled

383
384
def compiler_debugged( ):
    return config.compiler_debugged
385

386
387
388
389
390
def tag( t ):
    return t in config.compiler_tags

# ---

Ian Lynagh's avatar
Ian Lynagh committed
391
392
def namebase( nb ):
   return lambda opts, nb=nb: _namebase(opts, nb)
393

Ian Lynagh's avatar
Ian Lynagh committed
394
395
def _namebase( opts, nb ):
    opts.with_namebase = nb
396

397
398
# ---

399
def high_memory_usage(name, opts):
400
401
    opts.alone = True

402
403
404
405
406
# If a test is for a multi-CPU race, then running the test alone
# increases the chance that we'll actually see it.
def multi_cpu_race(name, opts):
    opts.alone = True

Ian Lynagh's avatar
Ian Lynagh committed
407
# ---
408
def literate( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
409
410
    opts.literate = 1;

411
def c_src( name, opts ):
412
413
    opts.c_src = 1;

414
def objc_src( name, opts ):
Austin Seipp's avatar
Austin Seipp committed
415
416
    opts.objc_src = 1;

417
def objcpp_src( name, opts ):
418
419
    opts.objcpp_src = 1;

420
def cmm_src( name, opts ):
421
422
    opts.cmm_src = 1;

423
def outputdir( odir ):
424
    return lambda name, opts, d=odir: _outputdir(name, opts, d)
425

426
def _outputdir( name, opts, odir ):
427
428
    opts.outputdir = odir;

429
430
# ----

431
def pre_cmd( cmd ):
432
    return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
433

434
def _pre_cmd( name, opts, cmd ):
435
436
437
438
    opts.pre_cmd = cmd

# ----

439
def clean_cmd( cmd ):
440
    return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
441

442
def _clean_cmd( name, opts, cmd ):
443
444
445
446
    opts.clean_cmd = cmd

# ----

447
def cmd_prefix( prefix ):
448
    return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
449

450
def _cmd_prefix( name, opts, prefix ):
451
452
453
454
455
    opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;

# ----

def cmd_wrapper( fun ):
456
    return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
457

458
def _cmd_wrapper( name, opts, fun ):
459
    opts.cmd_wrapper = fun
460

461
462
# ----

Ian Lynagh's avatar
Ian Lynagh committed
463
def compile_cmd_prefix( prefix ):
464
    return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
Ian Lynagh's avatar
Ian Lynagh committed
465

466
def _compile_cmd_prefix( name, opts, prefix ):
Ian Lynagh's avatar
Ian Lynagh committed
467
468
469
470
    opts.compile_cmd_prefix = prefix

# ----

471
472
473
474
475
476
477
478
def check_stdout( f ):
    return lambda name, opts, f=f: _check_stdout(name, opts, f)

def _check_stdout( name, opts, f ):
    opts.check_stdout = f

# ----

479
def normalise_slashes( name, opts ):
480
481
    opts.extra_normaliser = normalise_slashes_

482
def normalise_exe( name, opts ):
483
484
    opts.extra_normaliser = normalise_exe_

485
def normalise_fun( fun ):
486
    return lambda name, opts, f=fun: _normalise_fun(name, opts, f)
487

488
def _normalise_fun( name, opts, f ):
489
490
    opts.extra_normaliser = f

491
def normalise_errmsg_fun( fun ):
492
    return lambda name, opts, f=fun: _normalise_errmsg_fun(name, opts, f)
493

494
def _normalise_errmsg_fun( name, opts, f ):
495
496
497
498
499
    opts.extra_errmsg_normaliser = f

def two_normalisers(f, g):
    return lambda x, f=f, g=g: f(g(x))

500
501
502
# ----
# Function for composing two opt-fns together

503
504
505
506
507
508
509
def executeSetups(fs, name, opts):
    if type(fs) is types.ListType:
        # If we have a list of setups, then execute each one
        map (lambda f : executeSetups(f, name, opts), fs)
    else:
        # fs is a single function, so just apply it
        fs(name, opts)
510

511
512
513
514
# -----------------------------------------------------------------------------
# The current directory of tests

def newTestDir( dir ):
515
    global thisdir_settings
516
    # reset the options for this test directory
517
518
519
520
521
    thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )

def _newTestDir( name, opts, dir ):
    opts.testdir = dir
    opts.compiler_always_flags = config.compiler_always_flags
522
523
524
525

# -----------------------------------------------------------------------------
# Actually doing tests

526
527
parallelTests = []
aloneTests = []
528
allTestNames = set([])
529

530
def runTest (opts, name, func, args):
ei@vuokko.info's avatar
ei@vuokko.info committed
531
532
    ok = 0

533
    if config.use_threads:
ei@vuokko.info's avatar
ei@vuokko.info committed
534
        t.thread_pool.acquire()
535
        try:
536
            while config.threads<(t.running_threads+1):
537
                t.thread_pool.wait()
538
            t.running_threads = t.running_threads+1
539
540
            ok=1
            t.thread_pool.release()
541
            thread.start_new_thread(test_common_thread, (name, opts, func, args))
542
543
544
545
546
        except:
            if not ok:
                t.thread_pool.release()
    else:
        test_common_work (name, opts, func, args)
547

548
# name  :: String
549
# setup :: TestOpts -> IO ()
550
def test (name, setup, func, args):
551
552
    global aloneTests
    global parallelTests
553
    global allTestNames
554
    global thisdir_settings
555
556
    if name in allTestNames:
        framework_fail(name, 'duplicate', 'There are multiple tests with this name')
557
    if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
558
        framework_fail(name, 'bad_name', 'This test has an invalid name')
559
560
561
562
563

    # Make a deep copy of the default_testopts, as we need our own copy
    # of any dictionaries etc inside it. Otherwise, if one test modifies
    # them, all tests will see the modified version!
    myTestOpts = copy.deepcopy(default_testopts)
564

565
    executeSetups([thisdir_settings, setup], name, myTestOpts)
566
567
568
569
570
571

    thisTest = lambda : runTest(myTestOpts, name, func, args)
    if myTestOpts.alone:
        aloneTests.append(thisTest)
    else:
        parallelTests.append(thisTest)
572
    allTestNames.add(name)
573

574
if config.use_threads:
575
    def test_common_thread(name, opts, func, args):
576
577
578
579
580
581
        t.lock.acquire()
        try:
            test_common_work(name,opts,func,args)
        finally:
            t.lock.release()
            t.thread_pool.acquire()
582
            t.running_threads = t.running_threads - 1
583
584
            t.thread_pool.notify()
            t.thread_pool.release()
585

586
587
588
589
590
591
592
593
594
595
def get_package_cache_timestamp():
    if config.package_conf_cache_file == '':
        return 0.0
    else:
        try:
            return os.stat(config.package_conf_cache_file).st_mtime
        except:
            return 0.0


ei@vuokko.info's avatar
ei@vuokko.info committed
596
def test_common_work (name, opts, func, args):
597
598
599
600
601
602
603
604
605
    try:
        t.total_tests = t.total_tests+1
        setLocalTestOpts(opts)

        package_conf_cache_file_start_timestamp = get_package_cache_timestamp()

        # All the ways we might run this test
        if func == compile or func == multimod_compile:
            all_ways = config.compile_ways
606
        elif func == compile_and_run or func == multimod_compile_and_run:
607
608
609
610
611
612
            all_ways = config.run_ways
        elif func == ghci_script:
            if 'ghci' in config.run_ways:
                all_ways = ['ghci']
            else:
                all_ways = []
613
        else:
614
615
616
617
618
619
620
621
622
623
624
            all_ways = ['normal']

        # A test itself can request extra ways by setting opts.extra_ways
        all_ways = all_ways + filter(lambda way: way not in all_ways,
                                     opts.extra_ways)

        t.total_test_cases = t.total_test_cases + len(all_ways)

        ok_way = lambda way: \
            not getTestOpts().skip \
            and (config.only == [] or name in config.only) \
625
            and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
626
            and (config.cmdline_ways == [] or way in config.cmdline_ways) \
627
            and (not (config.skip_perf_tests and isStatsTest())) \
628
629
630
631
632
633
634
635
636
637
638
639
            and way not in getTestOpts().omit_ways

        # Which ways we are asked to skip
        do_ways = filter (ok_way,all_ways)

        # In fast mode, we skip all but one way
        if config.fast and len(do_ways) > 0:
            do_ways = [do_ways[0]]

        if not config.clean_only:
            # Run the required tests...
            for way in do_ways:
640
641
                if stopping():
                    break
642
643
644
645
646
647
648
                do_test (name, way, func, args)

            for way in all_ways:
                if way not in do_ways:
                    skiptest (name,way)

        if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
649
            pretest_cleanup(name)
650
651
652
653
654
655
656
657
658
659
660
            clean(map (lambda suff: name + suff,
                      ['', '.exe', '.exe.manifest', '.genscript',
                       '.stderr.normalised',        '.stdout.normalised',
                       '.run.stderr.normalised',    '.run.stdout.normalised',
                       '.comp.stderr.normalised',   '.comp.stdout.normalised',
                       '.interp.stderr.normalised', '.interp.stdout.normalised',
                       '.stats', '.comp.stats',
                       '.hi', '.o', '.prof', '.exe.prof', '.hc',
                       '_stub.h', '_stub.c', '_stub.o',
                       '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']))

661
            if func == multi_compile or func == multi_compile_fail:
662
663
664
665
666
                    extra_mods = args[1]
                    clean(map (lambda (f,x): replace_suffix(f, 'o'), extra_mods))
                    clean(map (lambda (f,x): replace_suffix(f, 'hi'), extra_mods))

            clean(getTestOpts().clean_files)
Ian Lynagh's avatar
Ian Lynagh committed
667

668
669
670
671
672
673
674
            if getTestOpts().outputdir != None:
                odir = in_testdir(getTestOpts().outputdir)
                try:
                    shutil.rmtree(odir)
                except:
                    pass

675
676
677
678
679
            try:
                shutil.rmtree(in_testdir('.hpc.' + name))
            except:
                pass

680
681
682
683
684
685
            try:
                cleanCmd = getTestOpts().clean_cmd
                if cleanCmd != None:
                    result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
                    if result != 0:
                        framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
686
            except:
687
                framework_fail(name, 'cleaning', 'clean-command exception')
688

689
        package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
690

691
692
        if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
            framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
693

694
695
696
697
698
699
700
701
702
703
704
705
        try:
            for f in files_written[name]:
                if os.path.exists(f):
                    try:
                        if not f in files_written_not_removed[name]:
                            files_written_not_removed[name].append(f)
                    except:
                        files_written_not_removed[name] = [f]
        except:
            pass
    except Exception, e:
        framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
Ian Lynagh's avatar
Ian Lynagh committed
706

707
708
709
710
def clean(strs):
    for str in strs:
        for name in glob.glob(in_testdir(str)):
            clean_full_path(name)
711

712
def clean_full_path(name):
Ian Lynagh's avatar
Ian Lynagh committed
713
714
715
716
        try:
            # Remove files...
            os.remove(name)
        except OSError, e1:
717
            try:
Ian Lynagh's avatar
Ian Lynagh committed
718
719
720
721
722
723
724
725
726
727
728
                # ... and empty directories
                os.rmdir(name)
            except OSError, e2:
                # We don't want to fail here, but we do want to know
                # what went wrong, so print out the exceptions.
                # ENOENT isn't a problem, though, as we clean files
                # that don't necessarily exist.
                if e1.errno != errno.ENOENT:
                    print e1
                if e2.errno != errno.ENOENT:
                    print e2
729

730
def do_test(name, way, func, args):
731
732
733
    full_name = name + '(' + way + ')'

    try:
734
735
736
737
738
        if_verbose(2, "=====> %s %d of %d %s " % \
                    (full_name, t.total_tests, len(allTestNames), \
                    [t.n_unexpected_passes, \
                     t.n_unexpected_failures, \
                     t.n_framework_failures]))
739

740
741
        if config.use_threads:
            t.lock.release()
742
743
744
745

        try:
            preCmd = getTestOpts().pre_cmd
            if preCmd != None:
746
                result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
747
748
                if result != 0:
                    framework_fail(name, way, 'pre-command failed: ' + str(result))
749
        except:
750
751
            framework_fail(name, way, 'pre-command exception')

ei@vuokko.info's avatar
ei@vuokko.info committed
752
753
754
        try:
            result = apply(func, [name,way] + args)
        finally:
755
756
            if config.use_threads:
                t.lock.acquire()
757

758
759
760
        if getTestOpts().expect != 'pass' and \
                getTestOpts().expect != 'fail' and \
                getTestOpts().expect != 'missing-lib':
761
            framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
762

763
764
765
766
767
768
        try:
            passFail = result['passFail']
        except:
            passFail = 'No passFail found'

        if passFail == 'pass':
ei@vuokko.info's avatar
ei@vuokko.info committed
769
770
            if getTestOpts().expect == 'pass' \
               and way not in getTestOpts().expect_fail_for:
771
                t.n_expected_passes = t.n_expected_passes + 1
772
773
774
775
                if name in t.expected_passes:
                    t.expected_passes[name].append(way)
                else:
                    t.expected_passes[name] = [way]
776
            else:
777
                if_verbose(1, '*** unexpected pass for %s' % full_name)
778
                t.n_unexpected_passes = t.n_unexpected_passes + 1
779
                addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
780
        elif passFail == 'fail':
ei@vuokko.info's avatar
ei@vuokko.info committed
781
782
            if getTestOpts().expect == 'pass' \
               and way not in getTestOpts().expect_fail_for:
783
                if_verbose(1, '*** unexpected failure for %s' % full_name)
784
                t.n_unexpected_failures = t.n_unexpected_failures + 1
785
786
                reason = result['reason']
                addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
787
            else:
788
789
790
791
792
793
                if getTestOpts().expect == 'missing-lib':
                    t.n_missing_libs = t.n_missing_libs + 1
                    if name in t.missing_libs:
                        t.missing_libs[name].append(way)
                    else:
                        t.missing_libs[name] = [way]
794
                else:
795
796
797
798
799
                    t.n_expected_failures = t.n_expected_failures + 1
                    if name in t.expected_failures:
                        t.expected_failures[name].append(way)
                    else:
                        t.expected_failures[name] = [way]
800
801
        else:
            framework_fail(name, way, 'bad result ' + passFail)
802
    except KeyboardInterrupt:
803
        stopNow()
804
    except:
805
        framework_fail(name, way, 'do_test exception')
806
807
        traceback.print_exc()

808
def addPassingTestInfo (testInfos, directory, name, way):
809
810
811
812
813
814
815
816
817
818
    directory = re.sub('^\\.[/\\\\]', '', directory)

    if not directory in testInfos:
        testInfos[directory] = {}

    if not name in testInfos[directory]:
        testInfos[directory][name] = []

    testInfos[directory][name].append(way)

819
820
821
822
823
824
825
826
827
828
829
830
831
832
def addFailingTestInfo (testInfos, directory, name, reason, way):
    directory = re.sub('^\\.[/\\\\]', '', directory)

    if not directory in testInfos:
        testInfos[directory] = {}

    if not name in testInfos[directory]:
        testInfos[directory][name] = {}

    if not reason in testInfos[directory][name]:
        testInfos[directory][name][reason] = []

    testInfos[directory][name][reason].append(way)

833
def skiptest (name, way):
834
835
    # print 'Skipping test \"', name, '\"'
    t.n_tests_skipped = t.n_tests_skipped + 1
836
837
838
839
    if name in t.tests_skipped:
        t.tests_skipped[name].append(way)
    else:
        t.tests_skipped[name] = [way]
840

841
842
def framework_fail( name, way, reason ):
    full_name = name + '(' + way + ')'
Joachim Breitner's avatar
Joachim Breitner committed
843
    if_verbose(1, '*** framework failure for %s %s ' % (full_name, reason))
844
    t.n_framework_failures = t.n_framework_failures + 1
845
846
847
848
    if name in t.framework_failures:
        t.framework_failures[name].append(way)
    else:
        t.framework_failures[name] = [way]
849

850
851
852
853
854
855
856
857
858
859
860
861
862
863
def badResult(result):
    try:
        if result['passFail'] == 'pass':
            return False
        return True
    except:
        return True

def passed():
    return {'passFail': 'pass'}

def failBecause(reason):
    return {'passFail': 'fail', 'reason': reason}

864
865
866
867
# -----------------------------------------------------------------------------
# Generic command tests

# A generic command test is expected to run and exit successfully.
868
869
870
871
872
873
#
# The expected exit code can be changed via exit_code() as normal, and
# the expected stdout/stderr are stored in <testname>.stdout and
# <testname>.stderr.  The output of the command can be ignored
# altogether by using run_command_ignore_output instead of
# run_command.
874

875
def run_command( name, way, cmd ):
876
    return simple_run( name, '', cmd, '' )
877

878
879
880
881
# -----------------------------------------------------------------------------
# GHCi tests

def ghci_script( name, way, script ):
882
    # filter out -fforce-recomp from compiler_always_flags, because we're
883
    # actually testing the recompilation behaviour in the GHCi tests.
884
    flags = filter(lambda f: f != '-fforce-recomp', getTestOpts().compiler_always_flags)
simonpj's avatar
simonpj committed
885
    flags.append(getTestOpts().extra_hc_opts)
886
887
    if getTestOpts().outputdir != None:
        flags.extend(["-outputdir", getTestOpts().outputdir])
888
889
890
891
892
893

    # We pass HC and HC_OPTS as environment variables, so that the
    # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
    cmd = "HC='" + config.compiler + "' " + \
          "HC_OPTS='" + join(flags,' ') + "' " + \
          "'" + config.compiler + "'" + \
894
          ' --interactive -v0 -ignore-dot-ghci ' + \
895
896
          join(flags,' ')

ei@vuokko.info's avatar
ei@vuokko.info committed
897
    getTestOpts().stdin = script
898
    return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
899

900
901
902
903
# -----------------------------------------------------------------------------
# Compile-only tests

def compile( name, way, extra_hc_opts ):
904
    return do_compile( name, way, 0, '', [], extra_hc_opts )
905
906

def compile_fail( name, way, extra_hc_opts ):
907
    return do_compile( name, way, 1, '', [], extra_hc_opts )
908
909

def multimod_compile( name, way, top_mod, extra_hc_opts ):
910
    return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
911

912
def multimod_compile_fail( name, way, top_mod, extra_hc_opts ):
913
914
915
916
917
918
919
920
921
    return do_compile( name, way, 1, top_mod, [], extra_hc_opts )

def multi_compile( name, way, top_mod, extra_mods, extra_hc_opts ):
    return do_compile( name, way, 0, top_mod, extra_mods, extra_hc_opts)

def multi_compile_fail( name, way, top_mod, extra_mods, extra_hc_opts ):
    return do_compile( name, way, 1, top_mod, extra_mods, extra_hc_opts)

def do_compile( name, way, should_fail, top_mod, extra_mods, extra_hc_opts ):
922
923
    # print 'Compile only, extra args = ', extra_hc_opts
    pretest_cleanup(name)
924
925
926
927
928
929

    result = extras_build( way, extra_mods, extra_hc_opts )
    if badResult(result):
       return result
    extra_hc_opts = result['hc_opts']

930
931
932
    force = 0
    if extra_mods:
       force = 1
933
934
    result = simple_build( name, way, extra_hc_opts, should_fail, top_mod, 0, 1, force)

935
    if badResult(result):
Ian Lynagh's avatar
Ian Lynagh committed
936
        return result
937
938
939
940
941

    # the actual stderr should always match the expected, regardless
    # of whether we expected the compilation to fail or not (successful
    # compilations may generate warnings).

Ian Lynagh's avatar
Ian Lynagh committed
942
    if getTestOpts().with_namebase == None:
943
944
        namebase = name
    else:
Ian Lynagh's avatar
Ian Lynagh committed
945
        namebase = getTestOpts().with_namebase
946
947

    (platform_specific, expected_stderr_file) = platform_wordsize_qualify(namebase, 'stderr')
948
949
    actual_stderr_file = qualify(name, 'comp.stderr')

950
951
    if not compare_outputs('stderr', \
                           two_normalisers(two_normalisers(getTestOpts().extra_errmsg_normaliser, normalise_errmsg), normalise_whitespace), \
952
                           expected_stderr_file, actual_stderr_file):
953
        return failBecause('stderr mismatch')
954
955

    # no problems found, this test passed
956
    return passed()
957

958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
def compile_cmp_asm( name, way, extra_hc_opts ):
    print 'Compile only, extra args = ', extra_hc_opts
    pretest_cleanup(name)
    result = simple_build( name + '.cmm', way, '-keep-s-files -O ' + extra_hc_opts, 0, '', 0, 0, 0)

    if badResult(result):
        return result

    # the actual stderr should always match the expected, regardless
    # of whether we expected the compilation to fail or not (successful
    # compilations may generate warnings).

    if getTestOpts().with_namebase == None:
        namebase = name
    else:
        namebase = getTestOpts().with_namebase