testlib.py 76.9 KB
Newer Older
1
# coding=utf8
2
#
3
4
5
# (c) Simon Marlow 2002
#

6
from __future__ import print_function
7

8
import shutil
9
10
import sys
import os
11
import errno
12
13
14
import string
import re
import traceback
15
16
import time
import datetime
17
import copy
18
import glob
ian@well-typed.com's avatar
ian@well-typed.com committed
19
from math import ceil, trunc
20
import collections
21
import subprocess
22

23
from testglobals import *
24
25
from testutil import *

26
27
if config.use_threads:
    import threading
28
29
30
31
    try:
        import thread
    except ImportError: # Python 3
        import _thread as thread
32

33
34
35
36
37
38
39
40
global wantToStop
wantToStop = False
def stopNow():
    global wantToStop
    wantToStop = True
def stopping():
    return wantToStop

41
42
# Options valid for the current test only (these get reset to
# testdir_testopts after each test).
43

ei@vuokko.info's avatar
ei@vuokko.info committed
44
global testopts_local
45
46
47
48
49
50
if config.use_threads:
    testopts_local = threading.local()
else:
    class TestOpts_Local:
        pass
    testopts_local = TestOpts_Local()
51
52

def getTestOpts():
ei@vuokko.info's avatar
ei@vuokko.info committed
53
    return testopts_local.x
54

ei@vuokko.info's avatar
ei@vuokko.info committed
55
56
57
def setLocalTestOpts(opts):
    global testopts_local
    testopts_local.x=opts
58

59
60
61
62
63
def isStatsTest():
    opts = getTestOpts()
    return len(opts.compiler_stats_range_fields) > 0 or len(opts.stats_range_fields) > 0


64
65
66
# This can be called at the top of a file of tests, to set default test options
# for the following tests.
def setTestOpts( f ):
67
    global thisdir_settings
68
    thisdir_settings = [thisdir_settings, f]
69
70
71
72
73
74
75
76
77
78
79
80

# -----------------------------------------------------------------------------
# Canned setup functions for common cases.  eg. for a test you might say
#
#      test('test001', normal, compile, [''])
#
# to run it without any options, but change it to
#
#      test('test001', expect_fail, compile, [''])
#
# to expect failure for this test.

81
def normal( name, opts ):
82
83
    return;

84
def skip( name, opts ):
85
86
    opts.skip = 1

87
def expect_fail( name, opts ):
88
89
90
    # The compiler, testdriver, OS or platform is missing a certain
    # feature, and we don't plan to or can't fix it now or in the
    # future.
91
92
    opts.expect = 'fail';

93
def reqlib( lib ):
94
    return lambda name, opts, l=lib: _reqlib (name, opts, l )
95

96
97
98
99
# Cache the results of looking to see if we have a library or not.
# This makes quite a difference, especially on Windows.
have_lib = {}

100
def _reqlib( name, opts, lib ):
101
    if lib in have_lib:
102
        got_it = have_lib[lib]
103
    else:
104
105
106
107
108
109
110
111
        cmd = strip_quotes(config.ghc_pkg)
        p = subprocess.Popen([cmd, '--no-user-package-db', 'describe', lib],
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
        # read from stdout and stderr to avoid blocking due to
        # buffers filling
        p.communicate()
        r = p.wait()
112
113
114
115
        got_it = r == 0
        have_lib[lib] = got_it

    if not got_it:
116
        opts.expect = 'missing-lib'
117

118
119
120
121
def req_haddock( name, opts ):
    if not config.haddock:
        opts.expect = 'missing-lib'

122
def req_profiling( name, opts ):
123
124
125
    if not config.have_profiling:
        opts.expect = 'fail'

126
def req_shared_libs( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
127
128
129
    if not config.have_shared_libs:
        opts.expect = 'fail'

130
def req_interp( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
131
132
133
    if not config.have_interp:
        opts.expect = 'fail'

134
def req_smp( name, opts ):
Simon Marlow's avatar
Simon Marlow committed
135
136
137
    if not config.have_smp:
        opts.expect = 'fail'

138
def ignore_output( name, opts ):
139
140
    opts.ignore_output = 1

141
def no_stdin( name, opts ):
142
143
    opts.no_stdin = 1

144
def combined_output( name, opts ):
pcapriotti's avatar
pcapriotti committed
145
146
    opts.combined_output = True

147
148
149
# -----

def expect_fail_for( ways ):
150
    return lambda name, opts, w=ways: _expect_fail_for( name, opts, w )
151

152
def _expect_fail_for( name, opts, ways ):
153
154
    opts.expect_fail_for = ways

155
def expect_broken( bug ):
156
157
    # This test is a expected not to work due to the indicated trac bug
    # number.
158
159
160
    return lambda name, opts, b=bug: _expect_broken (name, opts, b )

def _expect_broken( name, opts, bug ):
161
    record_broken(name, opts, bug)
162
163
    opts.expect = 'fail';

Ian Lynagh's avatar
Ian Lynagh committed
164
def expect_broken_for( bug, ways ):
165
    return lambda name, opts, b=bug, w=ways: _expect_broken_for( name, opts, b, w )
Ian Lynagh's avatar
Ian Lynagh committed
166

167
def _expect_broken_for( name, opts, bug, ways ):
168
    record_broken(name, opts, bug)
Ian Lynagh's avatar
Ian Lynagh committed
169
    opts.expect_fail_for = ways
Ian Lynagh's avatar
Ian Lynagh committed
170

171
def record_broken(name, opts, bug):
172
    global brokens
173
    me = (bug, opts.testdir, name)
174
175
176
    if not me in brokens:
        brokens.append(me)

177
178
179
180
181
def _expect_pass(way):
    # Helper function. Not intended for use in .T files.
    opts = getTestOpts()
    return opts.expect == 'pass' and way not in opts.expect_fail_for

182
183
184
# -----

def omit_ways( ways ):
185
    return lambda name, opts, w=ways: _omit_ways( name, opts, w )
186

187
def _omit_ways( name, opts, ways ):
188
189
190
191
    opts.omit_ways = ways

# -----

192
def only_ways( ways ):
193
    return lambda name, opts, w=ways: _only_ways( name, opts, w )
194

195
def _only_ways( name, opts, ways ):
196
197
198
199
    opts.only_ways = ways

# -----

200
def extra_ways( ways ):
201
    return lambda name, opts, w=ways: _extra_ways( name, opts, w )
202

203
def _extra_ways( name, opts, ways ):
204
205
206
207
    opts.extra_ways = ways

# -----

208
209
def only_compiler_types( _compiler_types ):
   # Don't delete yet. The libraries unix, stm and hpc still call this function.
210
   return lambda _name, _opts: None
ross's avatar
ross committed
211
212
213

# -----

214
def set_stdin( file ):
215
   return lambda name, opts, f=file: _set_stdin(name, opts, f);
216

217
def _set_stdin( name, opts, f ):
218
219
220
221
222
   opts.stdin = f

# -----

def exit_code( val ):
223
    return lambda name, opts, v=val: _exit_code(name, opts, v);
224

225
def _exit_code( name, opts, v ):
226
227
    opts.exit_code = v

228
229
230
231
232
233
234
235
236
237
238
def signal_exit_code( val ):
    if opsys('solaris2'):
        return exit_code( val );
    else:
        # When application running on Linux receives fatal error
        # signal, then its exit code is encoded as 128 + signal
        # value. See http://www.tldp.org/LDP/abs/html/exitcodes.html
        # I assume that Mac OS X behaves in the same way at least Mac
        # OS X builder behavior suggests this.
        return exit_code( val+128 );

239
240
# -----

241
242
def compile_timeout_multiplier( val ):
    return lambda name, opts, v=val: _compile_timeout_multiplier(name, opts, v)
243

244
245
246
247
248
249
250
251
def _compile_timeout_multiplier( name, opts, v ):
    opts.compile_timeout_multiplier = v

def run_timeout_multiplier( val ):
    return lambda name, opts, v=val: _run_timeout_multiplier(name, opts, v)

def _run_timeout_multiplier( name, opts, v ):
    opts.run_timeout_multiplier = v
252
253
254

# -----

255
def extra_run_opts( val ):
256
    return lambda name, opts, v=val: _extra_run_opts(name, opts, v);
257

258
def _extra_run_opts( name, opts, v ):
259
260
    opts.extra_run_opts = v

261
262
# -----

Simon Marlow's avatar
Simon Marlow committed
263
def extra_hc_opts( val ):
264
    return lambda name, opts, v=val: _extra_hc_opts(name, opts, v);
Simon Marlow's avatar
Simon Marlow committed
265

266
def _extra_hc_opts( name, opts, v ):
Simon Marlow's avatar
Simon Marlow committed
267
268
269
270
    opts.extra_hc_opts = v

# -----

271
def extra_clean( files ):
272
    assert not isinstance(files, str), files
273
    return lambda name, opts, v=files: _extra_clean(name, opts, v);
274

275
def _extra_clean( name, opts, v ):
276
277
    opts.clean_files = v

278
279
# -----

280
281
282
283
284
285
286
def stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _stats_num_field(name, opts, f, e);

def _stats_num_field( name, opts, field, expecteds ):
    if field in opts.stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

287
    if type(expecteds) is list:
288
289
290
291
292
        for (b, expected, dev) in expecteds:
            if b:
                opts.stats_range_fields[field] = (expected, dev)
                return
        framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
293

294
295
296
    else:
        (expected, dev) = expecteds
        opts.stats_range_fields[field] = (expected, dev)
297
298
299

def compiler_stats_num_field( field, expecteds ):
    return lambda name, opts, f=field, e=expecteds: _compiler_stats_num_field(name, opts, f, e);
300

301
302
303
304
def _compiler_stats_num_field( name, opts, field, expecteds ):
    if field in opts.compiler_stats_range_fields:
        framework_fail(name, 'duplicate-numfield', 'Duplicate ' + field + ' num_field check')

305
306
    # Compiler performance numbers change when debugging is on, making the results
    # useless and confusing. Therefore, skip if debugging is on.
307
308
309
    if compiler_debugged():
        skip(name, opts)

310
311
312
313
    for (b, expected, dev) in expecteds:
        if b:
            opts.compiler_stats_range_fields[field] = (expected, dev)
            return
314

315
316
    framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')

317
318
# -----

319
def when(b, f):
ian@well-typed.com's avatar
ian@well-typed.com committed
320
321
322
    # When list_brokens is on, we want to see all expect_broken calls,
    # so we always do f
    if b or config.list_broken:
323
324
325
326
327
328
329
        return f
    else:
        return normal

def unless(b, f):
    return when(not b, f)

ian@well-typed.com's avatar
ian@well-typed.com committed
330
331
332
def doing_ghci():
    return 'ghci' in config.run_ways

333
def ghci_dynamic( ):
ian@well-typed.com's avatar
ian@well-typed.com committed
334
    return config.ghc_dynamic
335
336

def fast():
337
    return config.speed == 2
338

339
340
def platform( plat ):
    return config.platform == plat
Ian Lynagh's avatar
Ian Lynagh committed
341

342
343
def opsys( os ):
    return config.os == os
Ian Lynagh's avatar
Ian Lynagh committed
344

345
346
def arch( arch ):
    return config.arch == arch
347

348
349
def wordsize( ws ):
    return config.wordsize == str(ws)
tibbe's avatar
tibbe committed
350

351
352
def msys( ):
    return config.msys
ian@well-typed.com's avatar
ian@well-typed.com committed
353

354
355
def cygwin( ):
    return config.cygwin
Ian Lynagh's avatar
Ian Lynagh committed
356

357
358
def have_vanilla( ):
    return config.have_vanilla
359

360
361
def have_dynamic( ):
    return config.have_dynamic
362

363
364
def have_profiling( ):
    return config.have_profiling
365

366
367
def in_tree_compiler( ):
    return config.in_tree_compiler
368

369
def compiler_lt( compiler, version ):
370
371
    assert compiler == 'ghc'
    return version_lt(config.compiler_version, version)
372

373
def compiler_le( compiler, version ):
374
375
    assert compiler == 'ghc'
    return version_le(config.compiler_version, version)
376

377
def compiler_gt( compiler, version ):
378
379
    assert compiler == 'ghc'
    return version_gt(config.compiler_version, version)
380

381
def compiler_ge( compiler, version ):
382
383
    assert compiler == 'ghc'
    return version_ge(config.compiler_version, version)
384

385
386
387
388
389
390
def unregisterised( ):
    return config.unregisterised

def compiler_profiled( ):
    return config.compiler_profiled

391
392
def compiler_debugged( ):
    return config.compiler_debugged
393

394
395
396
397
398
def tag( t ):
    return t in config.compiler_tags

# ---

399
def high_memory_usage(name, opts):
400
401
    opts.alone = True

402
403
404
405
406
# If a test is for a multi-CPU race, then running the test alone
# increases the chance that we'll actually see it.
def multi_cpu_race(name, opts):
    opts.alone = True

Ian Lynagh's avatar
Ian Lynagh committed
407
# ---
408
def literate( name, opts ):
Ian Lynagh's avatar
Ian Lynagh committed
409
410
    opts.literate = 1;

411
def c_src( name, opts ):
412
413
    opts.c_src = 1;

414
def objc_src( name, opts ):
Austin Seipp's avatar
Austin Seipp committed
415
416
    opts.objc_src = 1;

417
def objcpp_src( name, opts ):
418
419
    opts.objcpp_src = 1;

420
def cmm_src( name, opts ):
421
422
    opts.cmm_src = 1;

423
def outputdir( odir ):
424
    return lambda name, opts, d=odir: _outputdir(name, opts, d)
425

426
def _outputdir( name, opts, odir ):
427
428
    opts.outputdir = odir;

429
430
# ----

431
def pre_cmd( cmd ):
432
    return lambda name, opts, c=cmd: _pre_cmd(name, opts, cmd)
433

434
def _pre_cmd( name, opts, cmd ):
435
436
437
438
    opts.pre_cmd = cmd

# ----

439
def clean_cmd( cmd ):
440
    return lambda name, opts, c=cmd: _clean_cmd(name, opts, cmd)
441

442
def _clean_cmd( name, opts, cmd ):
443
444
445
446
    opts.clean_cmd = cmd

# ----

447
def cmd_prefix( prefix ):
448
    return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
449

450
def _cmd_prefix( name, opts, prefix ):
451
452
453
454
455
    opts.cmd_wrapper = lambda cmd, p=prefix: p + ' ' + cmd;

# ----

def cmd_wrapper( fun ):
456
    return lambda name, opts, f=fun: _cmd_wrapper(name, opts, fun)
457

458
def _cmd_wrapper( name, opts, fun ):
459
    opts.cmd_wrapper = fun
460

461
462
# ----

Ian Lynagh's avatar
Ian Lynagh committed
463
def compile_cmd_prefix( prefix ):
464
    return lambda name, opts, p=prefix: _compile_cmd_prefix(name, opts, prefix)
Ian Lynagh's avatar
Ian Lynagh committed
465

466
def _compile_cmd_prefix( name, opts, prefix ):
Ian Lynagh's avatar
Ian Lynagh committed
467
468
469
470
    opts.compile_cmd_prefix = prefix

# ----

471
472
473
474
475
476
477
478
def check_stdout( f ):
    return lambda name, opts, f=f: _check_stdout(name, opts, f)

def _check_stdout( name, opts, f ):
    opts.check_stdout = f

# ----

479
def normalise_slashes( name, opts ):
480
    _normalise_fun(name, opts, normalise_slashes_)
481

482
def normalise_exe( name, opts ):
483
    _normalise_fun(name, opts, normalise_exe_)
484

485
486
def normalise_fun( *fs ):
    return lambda name, opts: _normalise_fun(name, opts, fs)
487

488
def _normalise_fun( name, opts, *fs ):
489
    opts.extra_normaliser = join_normalisers(opts.extra_normaliser, fs)
490

491
492
def normalise_errmsg_fun( *fs ):
    return lambda name, opts: _normalise_errmsg_fun(name, opts, fs)
493

494
def _normalise_errmsg_fun( name, opts, *fs ):
495
496
497
498
499
500
501
502
503
504
505
506
507
    opts.extra_errmsg_normaliser =  join_normalisers(opts.extra_errmsg_normaliser, fs)

def normalise_version_( *pkgs ):
    def normalise_version__( str ):
        return re.sub('(' + '|'.join(map(re.escape,pkgs)) + ')-[0-9.]+',
                      '\\1-<VERSION>', str)
    return normalise_version__

def normalise_version( *pkgs ):
    def normalise_version__( name, opts ):
        _normalise_fun(name, opts, normalise_version_(*pkgs))
        _normalise_errmsg_fun(name, opts, normalise_version_(*pkgs))
    return normalise_version__
508

thomie's avatar
thomie committed
509
510
511
512
def normalise_drive_letter(name, opts):
    # Windows only. Change D:\\ to C:\\.
    _normalise_fun(name, opts, lambda str: re.sub(r'[A-Z]:\\', r'C:\\', str))

513
514
def join_normalisers(*a):
    """
515
    Compose functions, flattening sequences.
516

517
       join_normalisers(f1,[f2,f3],f4)
518
519
520

    is the same as

521
       lambda x: f1(f2(f3(f4(x))))
522
523
    """

524
525
526
527
528
529
530
531
532
533
534
535
    def flatten(l):
        """
        Taken from http://stackoverflow.com/a/2158532/946226
        """
        for el in l:
            if isinstance(el, collections.Iterable) and not isinstance(el, basestring):
                for sub in flatten(el):
                    yield sub
            else:
                yield el

    a = flatten(a)
536
537
538

    fn = lambda x:x # identity function
    for f in a:
539
        assert callable(f)
540
541
542
        fn = lambda x,f=f,fn=fn: fn(f(x))
    return fn

543
544
545
# ----
# Function for composing two opt-fns together

546
def executeSetups(fs, name, opts):
547
    if type(fs) is list:
548
        # If we have a list of setups, then execute each one
549
550
        for f in fs:
            executeSetups(f, name, opts)
551
552
553
    else:
        # fs is a single function, so just apply it
        fs(name, opts)
554

555
556
557
558
# -----------------------------------------------------------------------------
# The current directory of tests

def newTestDir( dir ):
559
    global thisdir_settings
560
    # reset the options for this test directory
561
562
563
564
565
    thisdir_settings = lambda name, opts, dir=dir: _newTestDir( name, opts, dir )

def _newTestDir( name, opts, dir ):
    opts.testdir = dir
    opts.compiler_always_flags = config.compiler_always_flags
566
567
568
569

# -----------------------------------------------------------------------------
# Actually doing tests

570
571
parallelTests = []
aloneTests = []
572
allTestNames = set([])
573

574
def runTest (opts, name, func, args):
ei@vuokko.info's avatar
ei@vuokko.info committed
575
576
    ok = 0

577
    if config.use_threads:
ei@vuokko.info's avatar
ei@vuokko.info committed
578
        t.thread_pool.acquire()
579
        try:
580
            while config.threads<(t.running_threads+1):
581
                t.thread_pool.wait()
582
            t.running_threads = t.running_threads+1
583
584
            ok=1
            t.thread_pool.release()
585
            thread.start_new_thread(test_common_thread, (name, opts, func, args))
586
587
588
589
590
        except:
            if not ok:
                t.thread_pool.release()
    else:
        test_common_work (name, opts, func, args)
591

592
# name  :: String
593
# setup :: TestOpts -> IO ()
594
def test (name, setup, func, args):
595
596
597
598
599
600
601
602
603
604
    if config.run_only_some_tests:
        if name not in config.only:
            return
        else:
            # Note [Mutating config.only]
            # config.only is initiallly the set of tests requested by
            # the user (via 'make TEST='). We then remove all tests that
            # we've already seen (in .T files), so that we can later
            # report on any tests we couldn't find and error out.
            config.only.remove(name)
605

606
607
    global aloneTests
    global parallelTests
608
    global allTestNames
609
    global thisdir_settings
610
611
    if name in allTestNames:
        framework_fail(name, 'duplicate', 'There are multiple tests with this name')
612
    if not re.match('^[0-9]*[a-zA-Z][a-zA-Z0-9._-]*$', name):
613
        framework_fail(name, 'bad_name', 'This test has an invalid name')
614
615
616
617
618

    # Make a deep copy of the default_testopts, as we need our own copy
    # of any dictionaries etc inside it. Otherwise, if one test modifies
    # them, all tests will see the modified version!
    myTestOpts = copy.deepcopy(default_testopts)
619

620
    executeSetups([thisdir_settings, setup], name, myTestOpts)
621
622
623
624
625
626

    thisTest = lambda : runTest(myTestOpts, name, func, args)
    if myTestOpts.alone:
        aloneTests.append(thisTest)
    else:
        parallelTests.append(thisTest)
627
    allTestNames.add(name)
628

629
if config.use_threads:
630
    def test_common_thread(name, opts, func, args):
631
632
633
634
635
636
        t.lock.acquire()
        try:
            test_common_work(name,opts,func,args)
        finally:
            t.lock.release()
            t.thread_pool.acquire()
637
            t.running_threads = t.running_threads - 1
638
639
            t.thread_pool.notify()
            t.thread_pool.release()
640

641
642
643
644
645
646
647
648
649
650
def get_package_cache_timestamp():
    if config.package_conf_cache_file == '':
        return 0.0
    else:
        try:
            return os.stat(config.package_conf_cache_file).st_mtime
        except:
            return 0.0


ei@vuokko.info's avatar
ei@vuokko.info committed
651
def test_common_work (name, opts, func, args):
652
653
654
655
656
657
658
659
660
    try:
        t.total_tests = t.total_tests+1
        setLocalTestOpts(opts)

        package_conf_cache_file_start_timestamp = get_package_cache_timestamp()

        # All the ways we might run this test
        if func == compile or func == multimod_compile:
            all_ways = config.compile_ways
661
        elif func == compile_and_run or func == multimod_compile_and_run:
662
663
664
665
666
667
            all_ways = config.run_ways
        elif func == ghci_script:
            if 'ghci' in config.run_ways:
                all_ways = ['ghci']
            else:
                all_ways = []
668
        else:
669
670
671
            all_ways = ['normal']

        # A test itself can request extra ways by setting opts.extra_ways
672
        all_ways = all_ways + [way for way in opts.extra_ways if way not in all_ways]
673
674
675
676
677

        t.total_test_cases = t.total_test_cases + len(all_ways)

        ok_way = lambda way: \
            not getTestOpts().skip \
678
            and (getTestOpts().only_ways == None or way in getTestOpts().only_ways) \
679
            and (config.cmdline_ways == [] or way in config.cmdline_ways) \
680
            and (not (config.skip_perf_tests and isStatsTest())) \
681
682
683
            and way not in getTestOpts().omit_ways

        # Which ways we are asked to skip
684
        do_ways = list(filter (ok_way,all_ways))
685

686
687
        # Only run all ways in slow mode.
        # See Note [validate and testsuite speed] in toplevel Makefile.
688
689
        if config.accept:
            # Only ever run one way
690
            do_ways = do_ways[:1]
691
692
693
694
695
696
        elif config.speed > 0:
            # However, if we EXPLICITLY asked for a way (with extra_ways)
            # please test it!
            explicit_ways = filter(lambda way: way in opts.extra_ways, do_ways)
            other_ways = filter(lambda way: way not in opts.extra_ways, do_ways)
            do_ways = other_ways[:1] + explicit_ways
697
698
699
700

        if not config.clean_only:
            # Run the required tests...
            for way in do_ways:
701
702
                if stopping():
                    break
703
704
705
706
707
708
709
                do_test (name, way, func, args)

            for way in all_ways:
                if way not in do_ways:
                    skiptest (name,way)

        if getTestOpts().cleanup != '' and (config.clean_only or do_ways != []):
710
            pretest_cleanup(name)
711
712
            clean([name + suff for suff in [
                       '', '.exe', '.exe.manifest', '.genscript',
713
714
715
716
717
718
719
                       '.stderr.normalised',        '.stdout.normalised',
                       '.run.stderr.normalised',    '.run.stdout.normalised',
                       '.comp.stderr.normalised',   '.comp.stdout.normalised',
                       '.interp.stderr.normalised', '.interp.stdout.normalised',
                       '.stats', '.comp.stats',
                       '.hi', '.o', '.prof', '.exe.prof', '.hc',
                       '_stub.h', '_stub.c', '_stub.o',
720
                       '.hp', '.exe.hp', '.ps', '.aux', '.hcr', '.eventlog']])
721

722
            if func == multi_compile or func == multi_compile_fail:
723
                    extra_mods = args[1]
724
725
726
                    clean([replace_suffix(fx[0],'o') for fx in extra_mods])
                    clean([replace_suffix(fx[0], 'hi') for fx in extra_mods])

727
728

            clean(getTestOpts().clean_files)
Ian Lynagh's avatar
Ian Lynagh committed
729

730
731
732
733
734
735
736
            if getTestOpts().outputdir != None:
                odir = in_testdir(getTestOpts().outputdir)
                try:
                    shutil.rmtree(odir)
                except:
                    pass

737
738
739
740
741
            try:
                shutil.rmtree(in_testdir('.hpc.' + name))
            except:
                pass

742
743
744
745
746
747
            try:
                cleanCmd = getTestOpts().clean_cmd
                if cleanCmd != None:
                    result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + cleanCmd)
                    if result != 0:
                        framework_fail(name, 'cleaning', 'clean-command failed: ' + str(result))
748
            except:
749
                framework_fail(name, 'cleaning', 'clean-command exception')
750

751
        package_conf_cache_file_end_timestamp = get_package_cache_timestamp();
752

753
754
        if package_conf_cache_file_start_timestamp != package_conf_cache_file_end_timestamp:
            framework_fail(name, 'whole-test', 'Package cache timestamps do not match: ' + str(package_conf_cache_file_start_timestamp) + ' ' + str(package_conf_cache_file_end_timestamp))
755

756
757
758
759
760
761
762
763
764
765
        try:
            for f in files_written[name]:
                if os.path.exists(f):
                    try:
                        if not f in files_written_not_removed[name]:
                            files_written_not_removed[name].append(f)
                    except:
                        files_written_not_removed[name] = [f]
        except:
            pass
766
    except Exception as e:
767
        framework_fail(name, 'runTest', 'Unhandled exception: ' + str(e))
Ian Lynagh's avatar
Ian Lynagh committed
768

769
770
def clean(strs):
    for str in strs:
771
772
773
774
775
        if (str.endswith('.package.conf') or
            str.startswith('package.conf.') and not str.endswith('/*')):
            # Package confs are directories now.
            str += '/*'

776
777
        for name in glob.glob(in_testdir(str)):
            clean_full_path(name)
778

779
def clean_full_path(name):
Ian Lynagh's avatar
Ian Lynagh committed
780
781
782
        try:
            # Remove files...
            os.remove(name)
783
        except OSError as e1:
784
            try:
Ian Lynagh's avatar
Ian Lynagh committed
785
786
                # ... and empty directories
                os.rmdir(name)
787
            except OSError as e2:
Ian Lynagh's avatar
Ian Lynagh committed
788
789
790
791
792
                # We don't want to fail here, but we do want to know
                # what went wrong, so print out the exceptions.
                # ENOENT isn't a problem, though, as we clean files
                # that don't necessarily exist.
                if e1.errno != errno.ENOENT:
793
                    print(e1)
Ian Lynagh's avatar
Ian Lynagh committed
794
                if e2.errno != errno.ENOENT:
795
                    print(e2)
796

797
def do_test(name, way, func, args):
798
799
800
    full_name = name + '(' + way + ')'

    try:
801
802
803
804
805
        if_verbose(2, "=====> %s %d of %d %s " % \
                    (full_name, t.total_tests, len(allTestNames), \
                    [t.n_unexpected_passes, \
                     t.n_unexpected_failures, \
                     t.n_framework_failures]))
806

807
808
        if config.use_threads:
            t.lock.release()
809
810
811
812

        try:
            preCmd = getTestOpts().pre_cmd
            if preCmd != None:
813
                result = runCmdFor(name, 'cd ' + getTestOpts().testdir + ' && ' + preCmd)
814
815
                if result != 0:
                    framework_fail(name, way, 'pre-command failed: ' + str(result))
816
        except:
817
818
            framework_fail(name, way, 'pre-command exception')

ei@vuokko.info's avatar
ei@vuokko.info committed
819
        try:
820
            result = func(*[name,way] + args)
ei@vuokko.info's avatar
ei@vuokko.info committed
821
        finally:
822
823
            if config.use_threads:
                t.lock.acquire()
824

825
826
827
        if getTestOpts().expect != 'pass' and \
                getTestOpts().expect != 'fail' and \
                getTestOpts().expect != 'missing-lib':
828
            framework_fail(name, way, 'bad expected ' + getTestOpts().expect)
829

830
831
832
833
834
835
        try:
            passFail = result['passFail']
        except:
            passFail = 'No passFail found'

        if passFail == 'pass':
836
            if _expect_pass(way):
837
                t.n_expected_passes = t.n_expected_passes + 1
838
839
840
841
                if name in t.expected_passes:
                    t.expected_passes[name].append(way)
                else:
                    t.expected_passes[name] = [way]
842
            else:
843
                if_verbose(1, '*** unexpected pass for %s' % full_name)
844
                t.n_unexpected_passes = t.n_unexpected_passes + 1
845
                addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
846
        elif passFail == 'fail':
847
            if _expect_pass(way):
848
                reason = result['reason']
849
850
851
852
853
854
855
856
857
                tag = result.get('tag')
                if tag == 'stat':
                    if_verbose(1, '*** unexpected stat test failure for %s' % full_name)
                    t.n_unexpected_stat_failures = t.n_unexpected_stat_failures + 1
                    addFailingTestInfo(t.unexpected_stat_failures, getTestOpts().testdir, name, reason, way)
                else:
                    if_verbose(1, '*** unexpected failure for %s' % full_name)
                    t.n_unexpected_failures = t.n_unexpected_failures + 1
                    addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
858
            else:
859
860
861
862
863
864
                if getTestOpts().expect == 'missing-lib':
                    t.n_missing_libs = t.n_missing_libs + 1
                    if name in t.missing_libs:
                        t.missing_libs[name].append(way)
                    else:
                        t.missing_libs[name] = [way]
865
                else:
866
867
868
869
870
                    t.n_expected_failures = t.n_expected_failures + 1
                    if name in t.expected_failures:
                        t.expected_failures[name].append(way)
                    else:
                        t.expected_failures[name] = [way]
871
872
        else:
            framework_fail(name, way, 'bad result ' + passFail)
873
    except KeyboardInterrupt:
874
        stopNow()
875
    except:
876
        framework_fail(name, way, 'do_test exception')
877
878
        traceback.print_exc()

879
def addPassingTestInfo (testInfos, directory, name, way):
880
881
882
883
884
885
886
887
888
889
    directory = re.sub('^\\.[/\\\\]', '', directory)

    if not directory in testInfos:
        testInfos[directory] = {}

    if not name in testInfos[directory]:
        testInfos[directory][name] = []

    testInfos[directory][name].append(way)

890
891
892
893
894
895
896
897
898
899
900
901
902
903
def addFailingTestInfo (testInfos, directory, name, reason, way):
    directory = re.sub('^\\.[/\\\\]', '', directory)

    if not directory in testInfos:
        testInfos[directory] = {}

    if not name in testInfos[directory]:
        testInfos[directory][name] = {}

    if not reason in testInfos[directory][name]:
        testInfos[directory][name][reason] = []

    testInfos[directory][name][reason].append(way)

904
def skiptest (name, way):
905
906
    # print 'Skipping test \"', name, '\"'
    t.n_tests_skipped = t.n_tests_skipped + 1
907
908
909
910
    if name in t.tests_skipped:
        t.tests_skipped[name].append(way)
    else:
        t.tests_skipped[name] = [way]
911

912
913
def framework_fail( name, way, reason ):
    full_name = name + '(' + way + ')'
Joachim Breitner's avatar
Joachim Breitner committed
914
    if_verbose(1, '*** framework failure for %s %s ' % (full_name, reason))
915
    t.n_framework_failures = t.n_framework_failures + 1
916
917
918
919
    if name in t.framework_failures:
        t.framework_failures[name].append(way)
    else:
        t.framework_failures[name] = [way]
920

921
922
923
924
925
926
927
928
929
930
931
def badResult(result):
    try:
        if result['passFail'] == 'pass':
            return False
        return True
    except:
        return True

def passed():
    return {'passFail': 'pass'}

932
933
def failBecause(reason, tag=None):
    return {'passFail': 'fail', 'reason': reason, 'tag': tag}
934

935
936
937
938
# -----------------------------------------------------------------------------
# Generic command tests

# A generic command test is expected to run and exit successfully.
939
940
941
942
943
944
#
# The expected exit code can be changed via exit_code() as normal, and
# the expected stdout/stderr are stored in <testname>.stdout and
# <testname>.stderr.  The output of the command can be ignored
# altogether by using run_command_ignore_output instead of
# run_command.
945

946
def run_command( name, way, cmd ):
947
    return simple_run( name, '', cmd, '' )
948

949
950
951
# -----------------------------------------------------------------------------
# GHCi tests

952
def ghci_script( name, way, script, override_flags = None ):
953
    # filter out -fforce-recomp from compiler_always_flags, because we're
954
    # actually testing the recompilation behaviour in the GHCi tests.