From 569b4c106a77f0e8e8550af458b4f796efa6a246 Mon Sep 17 00:00:00 2001
From: doyougnu <jeffrey.young@iohk.io>
Date: Wed, 6 Dec 2023 17:09:27 -0500
Subject: [PATCH] ts: add compile_artifact, ignore_extension flag
In b521354216f2821e00d75f088d74081d8b236810 the testsuite gained the
capability to collect generic metrics. But this assumed that the test
was not linking and producing artifacts and we only wanted to track
object files, interface files, or build artifacts from the compiler
build. However, some backends, such as the JS backend, produce artifacts when
compiling, such as the jsexe directory which we want to track.
This patch:
- tweaks the testsuite to collect generic metrics on any build artifact
in the test directory.
- expands the exe_extension function to consider windows and adds the
ignore_extension flag.
- Modifies certain tests to add the ignore_extension flag. Tests such as
heaprof002 expect a .ps file, but on windows without ignore_extensions
the testsuite will look for foo.exe.ps. Hence the flag.
- adds the size_hello_artifact test
---
testsuite/driver/testglobals.py | 3 ++
testsuite/driver/testlib.py | 48 ++++++++++++++-----
testsuite/tests/hpc/function/test.T | 1 +
testsuite/tests/hpc/function2/test.T | 1 +
testsuite/tests/hpc/simple/test.T | 3 +-
testsuite/tests/perf/size/all.T | 3 ++
.../tests/perf/size/size_hello_artifact.hs | 4 ++
testsuite/tests/profiling/should_run/all.T | 1 +
testsuite/tests/typecheck/testeq1/test.T | 1 +
9 files changed, 52 insertions(+), 13 deletions(-)
create mode 100644 testsuite/tests/perf/size/size_hello_artifact.hs
diff --git a/testsuite/driver/testglobals.py b/testsuite/driver/testglobals.py
index 6100c3065cc4..f22ebb994c6b 100644
--- a/testsuite/driver/testglobals.py
+++ b/testsuite/driver/testglobals.py
@@ -352,6 +352,9 @@ class TestOptions:
self.ignore_stdout = False
self.ignore_stderr = False
+ # don't use the executable extension
+ self.ignore_extension = False
+
# Backpack test
self.compile_backpack = False
diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py
index 62c2ae321eb6..f27b86890664 100644
--- a/testsuite/driver/testlib.py
+++ b/testsuite/driver/testlib.py
@@ -138,9 +138,6 @@ def no_deps( name, opts):
def skip( name, opts ):
opts.skip = True
-def js_arch() -> bool:
- return arch("javascript");
-
# disable test on JS arch
def js_skip( name, opts ):
if js_arch():
@@ -379,6 +376,18 @@ def ignore_stdout(name, opts):
def ignore_stderr(name, opts):
opts.ignore_stderr = True
+def ignore_extension(name, opts):
+ """
+ Some tests generate files that are not expected to be suffixed with an
+ extension type, such as .exe on windows. This option allows these tests to
+ have finer-grained control over the filename that the testsuite will look
+ for. Examples of such tests are hpc tests which expect a .tix extension and
+ hp2ps tests which expect .hp. For these tests, on windows and without
+ ignoring the extension, the testsuite will look for, e.g., 'foo.exe.tix'
+ instead of 'foo.tix'.
+ """
+ opts.ignore_extension = True
+
def combined_output( name, opts ):
opts.combined_output = True
@@ -792,6 +801,8 @@ KNOWN_OPERATING_SYSTEMS = set([
def exe_extension() -> str:
if config.arch == 'wasm32':
return '.wasm'
+ elif config.os == "mingw32":
+ return '.exe'
return ''
def opsys( os: str ) -> bool:
@@ -810,6 +821,9 @@ def msys( ) -> bool:
def cygwin( ) -> bool:
return config.cygwin
+def js_arch() -> bool:
+ return arch("javascript");
+
def have_vanilla( ) -> bool:
return config.have_vanilla
@@ -1577,6 +1591,10 @@ async def ghci_script( name, way, script):
async def compile( name, way, extra_hc_opts ):
return await do_compile( name, way, False, None, [], [], extra_hc_opts )
+async def compile_artifact( name, way, extra_hc_opts ):
+ # We suppress stderr so that the link output isn't compared
+ return await do_compile( name, way, False, None, [], [], extra_hc_opts, should_link=True, compare_stderr=False )
+
async def compile_fail( name, way, extra_hc_opts ):
return await do_compile( name, way, True, None, [], [], extra_hc_opts )
@@ -1592,9 +1610,6 @@ async def backpack_compile( name, way, extra_hc_opts ):
async def backpack_compile_fail( name, way, extra_hc_opts ):
return await do_compile( name, way, True, None, [], [], extra_hc_opts, backpack=True )
-async def backpack_run( name, way, extra_hc_opts ):
- return await compile_and_run__( name, way, None, [], extra_hc_opts, backpack=True )
-
async def multimod_compile( name, way, top_mod, extra_hc_opts ):
return await do_compile( name, way, False, top_mod, [], [], extra_hc_opts )
@@ -1623,6 +1638,8 @@ async def do_compile(name: TestName,
extra_mods: List[str],
units: List[str],
extra_hc_opts: str,
+ should_link=False,
+ compare_stderr=True,
**kwargs
) -> PassFail:
# print 'Compile only, extra args = ', extra_hc_opts
@@ -1632,7 +1649,7 @@ async def do_compile(name: TestName,
return result
extra_hc_opts = result.hc_opts
- result = await simple_build(name, way, extra_hc_opts, should_fail, top_mod, units, False, True, **kwargs)
+ result = await simple_build(name, way, extra_hc_opts, should_fail, top_mod, units, should_link, True, **kwargs)
if badResult(result):
return result
@@ -1645,7 +1662,7 @@ async def do_compile(name: TestName,
actual_stderr_file = add_suffix(name, 'comp.stderr')
diff_file_name = in_testdir(add_suffix(name, 'comp.diff'))
- if not await compare_outputs(way, 'stderr',
+ if compare_stderr and not await compare_outputs(way, 'stderr',
join_normalisers(getTestOpts().extra_errmsg_normaliser,
normalise_errmsg),
expected_stderr_file, actual_stderr_file,
@@ -1747,7 +1764,8 @@ async def compile_and_run__(name: TestName,
extra_mods: List[str],
extra_hc_opts: str,
backpack: bool=False,
- compile_stderr: bool=False
+ compile_stderr: bool=False,
+ use_extension: bool=True
) -> PassFail:
# print 'Compile and run, extra args = ', extra_hc_opts
@@ -1780,8 +1798,11 @@ async def compile_and_run__(name: TestName,
stderr = diff_file_name.read_text()
diff_file_name.unlink()
return failBecause('ghc.stderr mismatch', stderr=stderr)
-#
- cmd = './' + name + exe_extension()
+
+ opts = getTestOpts()
+ extension = exe_extension() if not opts.ignore_extension else ""
+
+ cmd = './' + name + extension
# we don't check the compiler's stderr for a compile-and-run test
return await simple_run( name, way, cmd, getTestOpts().extra_run_opts )
@@ -1789,6 +1810,9 @@ async def compile_and_run__(name: TestName,
async def compile_and_run( name, way, extra_hc_opts ):
return await compile_and_run__( name, way, None, [], extra_hc_opts)
+async def backpack_run( name, way, extra_hc_opts ):
+ return await compile_and_run__( name, way, None, [], extra_hc_opts, backpack=True )
+
async def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
return await compile_and_run__( name, way, top_mod, [], extra_hc_opts)
@@ -2296,8 +2320,8 @@ def write_file(f: Path, s: str) -> None:
# operate on bytes.
async def check_hp_ok(name: TestName) -> bool:
- actual_name = name + exe_extension()
opts = getTestOpts()
+ actual_name = name + exe_extension() if not opts.ignore_extension else name
# do not qualify for hp2ps because we should be in the right directory
hp2psCmd = 'cd "{opts.testdir}" && {{hp2ps}} {actual_name}'.format(**locals())
diff --git a/testsuite/tests/hpc/function/test.T b/testsuite/tests/hpc/function/test.T
index 24dfa70f83b8..31f6c39e7980 100644
--- a/testsuite/tests/hpc/function/test.T
+++ b/testsuite/tests/hpc/function/test.T
@@ -5,5 +5,6 @@ hpc_prefix = "perl hpcrun.pl --clear --exeext={exeext} --hpc={hpc}"
test('tough',
[extra_files(['../hpcrun.pl']),
cmd_prefix(hpc_prefix),
+ ignore_extension,
when(arch('wasm32'), fragile(23243))],
compile_and_run, ['-fhpc'])
diff --git a/testsuite/tests/hpc/function2/test.T b/testsuite/tests/hpc/function2/test.T
index 3740e0e7b075..d0899f873987 100644
--- a/testsuite/tests/hpc/function2/test.T
+++ b/testsuite/tests/hpc/function2/test.T
@@ -10,6 +10,7 @@ test('tough2',
[extra_files(['../hpcrun.pl', 'subdir/']),
literate,
cmd_prefix(hpc_prefix),
+ ignore_extension,
omit_ways(ghci_ways + prof_ways), # profile goes in the wrong place
when(arch('wasm32'), fragile(23243)) ],
multimod_compile_and_run, ['subdir/tough2.lhs', '-fhpc'])
diff --git a/testsuite/tests/hpc/simple/test.T b/testsuite/tests/hpc/simple/test.T
index cd7cd8b09495..38f4876c7f26 100644
--- a/testsuite/tests/hpc/simple/test.T
+++ b/testsuite/tests/hpc/simple/test.T
@@ -3,6 +3,7 @@ setTestOpts([omit_ghci, when(fast(), skip), js_skip])
hpc_prefix = "perl hpcrun.pl --clear --exeext={exeext} --hpc={hpc}"
test('hpc001', [extra_files(['../hpcrun.pl']), cmd_prefix(hpc_prefix),
- when(arch('wasm32'), fragile(23243))
+ when(arch('wasm32'), fragile(23243)),
+ ignore_extension
],
compile_and_run, ['-fhpc'])
diff --git a/testsuite/tests/perf/size/all.T b/testsuite/tests/perf/size/all.T
index 969185e438da..3b1b46e78ad4 100644
--- a/testsuite/tests/perf/size/all.T
+++ b/testsuite/tests/perf/size/all.T
@@ -1,3 +1,6 @@
test('size_hello_obj', [collect_size(5, 'size_hello_obj.o')], compile, [''])
+test('size_hello_artifact', [collect_size(5, 'size_hello_artifact' + exe_extension())],
+ compile_artifact, [''])
+
test('libdir',[collect_size_dir(10, config.libdir)], static_stats, [] )
diff --git a/testsuite/tests/perf/size/size_hello_artifact.hs b/testsuite/tests/perf/size/size_hello_artifact.hs
new file mode 100644
index 000000000000..d38500d168f7
--- /dev/null
+++ b/testsuite/tests/perf/size/size_hello_artifact.hs
@@ -0,0 +1,4 @@
+-- same as size_hello_obj but we test the size of the resulting executable.
+module Main where
+
+main = print "Hello World!"
diff --git a/testsuite/tests/profiling/should_run/all.T b/testsuite/tests/profiling/should_run/all.T
index e09976508e0d..a113645e229d 100644
--- a/testsuite/tests/profiling/should_run/all.T
+++ b/testsuite/tests/profiling/should_run/all.T
@@ -4,6 +4,7 @@ setTestOpts(js_skip) # JS backend doesn't support profiling yet
test('heapprof002',
[extra_files(['heapprof001.hs']),
+ ignore_extension,
pre_cmd('cp heapprof001.hs heapprof002.hs'), extra_ways(['normal_h']),
extra_run_opts('7')],
compile_and_run, [''])
diff --git a/testsuite/tests/typecheck/testeq1/test.T b/testsuite/tests/typecheck/testeq1/test.T
index 58dfc33e166f..c9bc9d411528 100644
--- a/testsuite/tests/typecheck/testeq1/test.T
+++ b/testsuite/tests/typecheck/testeq1/test.T
@@ -1,6 +1,7 @@
test('typecheck.testeq1', [ extra_files(['FakePrelude.hs', 'Main.hs', 'TypeCast.hs', 'TypeEq.hs'])
, when(fast(), skip)
+ , ignore_extension
, js_broken(22355)
# https://gitlab.haskell.org/ghc/ghc/-/issues/23238
, when(arch('wasm32'), skip)
--
GitLab