runtests.py 11.7 KB
Newer Older
Ben Gamari's avatar
Ben Gamari committed
1 2 3
#!/usr/bin/env python3

#
4 5 6
# (c) Simon Marlow 2002
#

7 8
from __future__ import print_function

9
import argparse
10
import signal
11 12 13
import sys
import os
import string
14 15
import shutil
import tempfile
16
import time
17
import re
18

19 20 21 22 23 24
# We don't actually need subprocess in runtests.py, but:
# * We do need it in testlibs.py
# * We can't import testlibs.py until after we have imported ctypes
# * If we import ctypes before subprocess on cygwin, then sys.exit(0)
#   says "Aborted" and we fail with exit code 134.
# So we import it here first, so that the testsuite doesn't appear to fail.
25
import subprocess
26

27 28
from testutil import *
from testglobals import *
29

30 31 32 33 34
# Readline sometimes spews out ANSI escapes for some values of TERM,
# which result in test failures. Thus set TERM to a nice, simple, safe
# value.
os.environ['TERM'] = 'vt100'

35
global config
36
config = getConfig() # get it from testglobals
37

38 39 40
def signal_handler(signal, frame):
        stopNow()

41 42 43
# -----------------------------------------------------------------------------
# cmd-line options

44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98
parser = argparse.ArgumentParser(description="GHC's testsuite driver",
                                 allow_abbrev=False)

parser.add_argument("-e", action='append', help="A string to execute from the command line.")
parser.add_argument("--config-file", action="append", help="config file")
parser.add_argument("--config", action='append', help="config field")
parser.add_argument("--rootdir", action='append', help="root of tree containing tests (default: .)")
parser.add_argument("--summary-file", help="file in which to save the (human-readable) summary")
parser.add_argument("--no-print-summary", action="store_true", help="should we print the summary?")
parser.add_argument("--only", action="append", help="just this test (can be give multiple --only= flags)")
parser.add_argument("--way", choices=config.run_ways+config.compile_ways+config.other_ways, help="just this way")
parser.add_argument("--skipway", action="append", choices=config.run_ways+config.compile_ways+config.other_ways, help="skip this way")
parser.add_argument("--threads", type=int, help="threads to run simultaneously")
parser.add_argument("--check-files-written", help="check files aren't written by multiple tests") # NOTE: This doesn't seem to exist?
parser.add_argument("--verbose", type=int, choices=[0,1,2,3,4,5], help="verbose (Values 0 through 5 accepted)")
parser.add_argument("--skip-perf-tests", action="store_true", help="skip performance tests")

args = parser.parse_args()

for e in args.e:
    exec(e)

for arg in args.config_file:
    exec(open(arg).read())

for arg in args.config:
    field, value = arg.split('=', 1)
    setattr(config, field, value)

config.rootdirs = args.rootdir
config.summary_file = args.summary_file
config.no_print_summary = args.no_print_summary

if args.only:
    config.only = args.only
    config.run_only_some_tests = True

if args.way:
    config.cmdline_ways = [args.way] + config.cmdline_ways
    if (args.way in config.other_ways):
        config.run_ways = [args.way] + config.run_ways
        config.compile_ways = [args.way] + config.compile_ways

if args.skipway:
    config.other_ways = [w for w in config.other_ways if w != args.skipway]
    config.run_ways = [w for w in config.run_ways if w != args.skipway]
    config.compile_ways = [w for w in config.compile_ways if w != args.skipway]

if args.threads:
    config.threads = args.threads
    config.use_threads = True

if args.verbose:
    config.verbose = args.verbose
config.skip_perf_tests = args.skip_perf_tests
99

Ian Lynagh's avatar
Ian Lynagh committed
100 101
config.cygwin = False
config.msys = False
102

Ian Lynagh's avatar
Ian Lynagh committed
103
if windows:
104 105 106 107
    h = os.popen('uname -s', 'r')
    v = h.read()
    h.close()
    if v.startswith("CYGWIN"):
Ian Lynagh's avatar
Ian Lynagh committed
108
        config.cygwin = True
109
    elif v.startswith("MINGW") or v.startswith("MSYS"):
110
# msys gives "MINGW32"
111
# msys2 gives "MINGW_NT-6.2" or "MSYS_NT-6.3"
Ian Lynagh's avatar
Ian Lynagh committed
112
        config.msys = True
113
    else:
114
        raise Exception("Can't detect Windows terminal type")
Ian Lynagh's avatar
Ian Lynagh committed
115

116 117
# Try to use UTF8
if windows:
118
    import ctypes
119
    # Windows and mingw* Python provide windll, msys2 python provides cdll.
120 121
    if hasattr(ctypes, 'WinDLL'):
        mydll = ctypes.WinDLL
122
    else:
123
        mydll = ctypes.CDLL
Ian Lynagh's avatar
Ian Lynagh committed
124

125 126 127
    # This actually leaves the terminal in codepage 65001 (UTF8) even
    # after python terminates. We ought really remember the old codepage
    # and set it back.
128 129
    kernel32 = mydll('kernel32.dll')
    if kernel32.SetConsoleCP(65001) == 0:
130
        raise Exception("Failure calling SetConsoleCP(65001)")
131
    if kernel32.SetConsoleOutputCP(65001) == 0:
132
        raise Exception("Failure calling SetConsoleOutputCP(65001)")
133 134 135

    # register the interrupt handler
    signal.signal(signal.SIGINT, signal_handler)
136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153
else:
    # Try and find a utf8 locale to use
    # First see if we already have a UTF8 locale
    h = os.popen('locale | grep LC_CTYPE | grep -i utf', 'r')
    v = h.read()
    h.close()
    if v == '':
        # We don't, so now see if 'locale -a' works
        h = os.popen('locale -a', 'r')
        v = h.read()
        h.close()
        if v != '':
            # If it does then use the first utf8 locale that is available
            h = os.popen('locale -a | grep -i "utf8\|utf-8" 2>/dev/null', 'r')
            v = h.readline().strip()
            h.close()
            if v != '':
                os.environ['LC_ALL'] = v
154
                print("setting LC_ALL to", v)
155
            else:
156 157
                print('WARNING: No UTF8 locale found.')
                print('You may get some spurious test failures.')
158

159 160 161
# This has to come after arg parsing as the args can change the compiler
get_compiler_info()

162 163 164 165
# Can't import this earlier as we need to know if threading will be
# enabled or not
from testlib import *

166 167
# On Windows we need to set $PATH to include the paths to all the DLLs
# in order for the dynamic library tests to work.
168
if windows or darwin:
169
    pkginfo = str(getStdout([config.ghc_pkg, 'dump']))
170
    topdir = config.libdir
171 172 173
    if windows:
        mingw = os.path.join(topdir, '../mingw/bin')
        os.environ['PATH'] = os.pathsep.join([os.environ.get("PATH", ""), mingw])
174 175 176 177
    for line in pkginfo.split('\n'):
        if line.startswith('library-dirs:'):
            path = line.rstrip()
            path = re.sub('^library-dirs: ', '', path)
178 179 180 181
            # Use string.replace instead of re.sub, because re.sub
            # interprets backslashes in the replacement string as
            # escape sequences.
            path = path.replace('$topdir', topdir)
182 183 184 185 186 187 188 189 190 191 192 193 194 195
            if path.startswith('"'):
                path = re.sub('^"(.*)"$', '\\1', path)
                path = re.sub('\\\\(.)', '\\1', path)
            if windows:
                if config.cygwin:
                    # On cygwin we can't put "c:\foo" in $PATH, as : is a
                    # field separator. So convert to /cygdrive/c/foo instead.
                    # Other pythons use ; as the separator, so no problem.
                    path = re.sub('([a-zA-Z]):', '/cygdrive/\\1', path)
                    path = re.sub('\\\\', '/', path)
                os.environ['PATH'] = os.pathsep.join([path, os.environ.get("PATH", "")])
            else:
                # darwin
                os.environ['DYLD_LIBRARY_PATH'] = os.pathsep.join([path, os.environ.get("DYLD_LIBRARY_PATH", "")])
196

197 198 199
global testopts_local
testopts_local.x = TestOptions()

200 201 202 203
# if timeout == -1 then we try to calculate a sensible value
if config.timeout == -1:
    config.timeout = int(read_no_crs(config.top + '/timeout/calibrate.out'))

204
print('Timeout is ' + str(config.timeout))
205

206 207 208
# -----------------------------------------------------------------------------
# The main dude

209 210 211
if config.rootdirs == []:
    config.rootdirs = ['.']

212
t_files = list(findTFiles(config.rootdirs))
213

214
print('Found', len(t_files), '.T files...')
215 216 217

t = getTestRun()

218
# Avoid cmd.exe built-in 'date' command on Windows
219
t.start_time = time.localtime()
220

221
print('Beginning test run at', time.strftime("%c %Z",t.start_time))
222 223

sys.stdout.flush()
224 225
# we output text, which cannot be unbuffered
sys.stdout = os.fdopen(sys.__stdout__.fileno(), "w")
226

227 228 229 230 231 232 233 234 235 236 237 238 239 240 241
if config.local:
    tempdir = ''
else:
    # See note [Running tests in /tmp]
    tempdir = tempfile.mkdtemp('', 'ghctest-')

    # opts.testdir should be quoted when used, to make sure the testsuite
    # keeps working when it contains backward slashes, for example from
    # using os.path.join. Windows native and mingw* python
    # (/mingw64/bin/python) set `os.path.sep = '\\'`, while msys2 python
    # (/bin/python, /usr/bin/python or /usr/local/bin/python) sets
    # `os.path.sep = '/'`.
    # To catch usage of unquoted opts.testdir early, insert some spaces into
    # tempdir.
    tempdir = os.path.join(tempdir, 'test   spaces')
242 243

def cleanup_and_exit(exitcode):
244
    if config.cleanup and tempdir:
245 246 247
        shutil.rmtree(tempdir, ignore_errors=True)
    exit(exitcode)

248
# First collect all the tests to be run
249
t_files_ok = True
250
for file in t_files:
251
    if_verbose(2, '====> Scanning %s' % file)
252
    newTestDir(tempdir, os.path.dirname(file))
253
    try:
254 255
        with io.open(file, encoding='utf8') as f:
            src = f.read()
256 257

        exec(src)
258
    except Exception as e:
259
        traceback.print_exc()
260 261
        framework_fail(file, '', str(e))
        t_files_ok = False
262

263 264 265 266 267 268 269 270 271
for name in config.only:
    if t_files_ok:
        # See Note [Mutating config.only]
        framework_fail(name, '', 'test not found')
    else:
        # Let user fix .T file errors before reporting on unfound tests.
        # The reson the test can not be found is likely because of those
        # .T file errors.
        pass
272

ian@well-typed.com's avatar
ian@well-typed.com committed
273 274
if config.list_broken:
    global brokens
275 276 277 278
    print('')
    print('Broken tests:')
    print(' '.join(map (lambda bdn: '#' + str(bdn[0]) + '(' + bdn[1] + '/' + bdn[2] + ')', brokens)))
    print('')
279

280 281
    if t.framework_failures:
        print('WARNING:', len(framework_failures), 'framework failures!')
282
        print('')
ian@well-typed.com's avatar
ian@well-typed.com committed
283
else:
284 285 286
    # completion watcher
    watcher = Watcher(len(parallelTests))

ian@well-typed.com's avatar
ian@well-typed.com committed
287 288 289 290
    # Now run all the tests
    for oneTest in parallelTests:
        if stopping():
            break
291 292 293 294 295 296 297
        oneTest(watcher)

    # wait for parallel tests to finish
    if not stopping():
        watcher.wait()

    # Run the following tests purely sequential
ian@well-typed.com's avatar
ian@well-typed.com committed
298 299 300 301
    config.use_threads = False
    for oneTest in aloneTests:
        if stopping():
            break
302 303 304 305 306
        oneTest(watcher)

    # flush everything before we continue
    sys.stdout.flush()

thomie's avatar
thomie committed
307
    summary(t, sys.stdout, config.no_print_summary)
308

309
    if config.summary_file:
310 311
        with open(config.summary_file, 'w') as file:
            summary(t, file)
312

313
cleanup_and_exit(0)
314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345

# Note [Running tests in /tmp]
#
# Use LOCAL=0 to run tests in /tmp, to catch tests that use files from
# the source directory without copying them to the test directory first.
#
# As an example, take a run_command test with a Makefile containing
# `$(TEST_HC) ../Foo.hs`. GHC will now create the output files Foo.o and
# Foo.hi in the source directory. There are 2 problems with this:
# * Output files in the source directory won't get cleaned up automatically.
# * Two tests might (over)write the same output file.
#
# Tests that only fail when run concurrently with other tests are the
# worst, so we try to catch them early by enabling LOCAL=0 in validate.
#
# Adding -outputdir='.' to TEST_HC_OPTS would help a bit, but it requires
# making changes to quite a few tests. The problem is that
# `$(TEST_HC) ../Foo.hs -outputdir=.` with Foo.hs containing
# `module Main where` does not produce Foo.o, as it would without
# -outputdir, but Main.o. See [1].
#
# Using -outputdir='.' is not foolproof anyway, since it does not change
# the destination of the final executable (Foo.exe).
#
# Another hardening method that could be tried is to `chmod -w` the
# source directory.
#
# By default we set LOCAL=1, because it makes it easier to inspect the
# test directory while working on a new test.
#
# [1]
# https://downloads.haskell.org/~ghc/8.0.1/docs/html/users_guide/separate_compilation.html#output-files