From 9bae34d87f6c978e03031c549920071857c9080c Mon Sep 17 00:00:00 2001
From: doyougnu <jeffrey.young@iohk.io>
Date: Fri, 5 Apr 2024 11:43:49 -0400
Subject: [PATCH] testsuite: expand size testing infrastructure
- closes #24191
- adds windows_skip, wasm_skip, wasm_arch, find_so, _find_so
- path_from_ghcPkg, collect_size_ghc_pkg, collect_object_size, find_non_inplace functions to testsuite
- adds on_windows and req_dynamic_ghc predicate to testsuite
The design is to not make the testsuite too smart and simply offload to
ghc-pkg for locations of object files and directories.
---
testsuite/driver/testglobals.py | 4 +
testsuite/driver/testlib.py | 133 +++++++++++++++++++++++++++++---
testsuite/tests/perf/size/all.T | 75 +++++++++++++++++-
3 files changed, 202 insertions(+), 10 deletions(-)
diff --git a/testsuite/driver/testglobals.py b/testsuite/driver/testglobals.py
index 5b2e59b7ba5e..8617841a5573 100644
--- a/testsuite/driver/testglobals.py
+++ b/testsuite/driver/testglobals.py
@@ -221,6 +221,10 @@ class TestConfig:
# I have no idea what this does
self.package_conf_cache_file = None # type: Optional[Path]
+ # the libdir for the test compiler. Set by hadrian, see
+ # Setting.Builders.RunTest
+ self.libdir = ''
+
# The extra hadrian dependencies we need for all configured tests
self.hadrian_deps = set() # type: Set[str]
diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py
index 35c97581a2f5..db157ae0f277 100644
--- a/testsuite/driver/testlib.py
+++ b/testsuite/driver/testlib.py
@@ -143,6 +143,15 @@ def js_skip( name, opts ):
if js_arch():
skip(name,opts)
+# disable test on WASM arch
+def wasm_skip( name, opts ):
+ if wasm_arch():
+ skip(name,opts)
+
+def windows_skip(name,opts):
+ if on_windows():
+ skip(name,opts)
+
# expect broken for the JS backend
def js_broken( bug: IssueNumber ):
if js_arch():
@@ -239,6 +248,15 @@ def req_dynamic_hs( name, opts ):
if not config.supports_dynamic_hs:
opts.expect = 'fail'
+def req_dynamic_ghc( name, opts ):
+ '''
+ Require that the GHC is dynamically linked, if static then skip.
+ See tests/perf/size/all.T, specifically foo.so tests for use case
+ and example
+ '''
+ if not config.ghc_dynamic:
+ skip(name,opts)
+
def req_interp( name, opts ):
if not config.have_interp or needsTargetWrapper():
opts.expect = 'fail'
@@ -611,15 +629,24 @@ def collect_size ( deviation, path ):
def get_dir_size(path):
total = 0
- with os.scandir(path) as it:
- for entry in it:
- if entry.is_file():
- total += entry.stat().st_size
- elif entry.is_dir():
- total += get_dir_size(entry.path)
- return total
+ try:
+ with os.scandir(path) as it:
+ for entry in it:
+ if entry.is_file():
+ total += entry.stat().st_size
+ elif entry.is_dir():
+ total += get_dir_size(entry.path)
+ return total
+ except FileNotFoundError:
+ print("Exception: Could not find: " + path)
def collect_size_dir ( deviation, path ):
+
+ ## os.path.join joins the path with slashes (not backslashes) on windows
+ ## CI...for some reason, so we manually detect it here
+ sep = r"/"
+ if on_windows():
+ sep = r"\\"
return collect_generic_stat ( 'size', deviation, lambda way: get_dir_size(path) )
# Read a number from a specific file
@@ -636,6 +663,91 @@ def collect_generic_stats ( metric_info ):
return _collect_generic_stat(name, opts, metric_info)
return f
+# wrap the call to collect_size_dir with path_from_ghcPkg in a function. Python
+# is call-by-value so if we placed the call in an all.T file then the python
+# interpreter would evaluate the call to path_from_ghcPkg
+def collect_size_ghc_pkg (deviation, library):
+ return collect_size_dir(deviation, path_from_ghcPkg(library, "library-dirs"))
+
+# same for collect_size and find_so
+def collect_object_size (deviation, library, use_non_inplace=False):
+ if use_non_inplace:
+ return collect_size(deviation, find_non_inplace_so(library))
+ else:
+ return collect_size(deviation, find_so(library))
+
+def path_from_ghcPkg (library, field):
+ """Find the field as a path for a library via a call to ghc-pkg. This is a
+ testsuite wrapper around a call to ghc-pkg field {library} {field}.
+ """
+
+ ### example output from ghc-pkg:
+ ### $ ./ghc-pkg field Cabal library-dirs
+ ### library-dirs: /home/doyougnu/programming/haskell/ghc/_build/stage1/lib/../lib/x86_64-linux-ghc-9.11.20240424/Cabal-3.11.0.0-inplace
+ ### so we split the string and drop the 'library-dirs'
+ ghcPkgCmd = fr"{config.ghc_pkg} field {library} {field}"
+
+ try:
+ result = subprocess.run(ghcPkgCmd, capture_output=True, shell=True)
+
+ # check_returncode throws an exception if the return code is not 0.
+ result.check_returncode()
+
+ # if we get here then the call worked and we have the path we split by
+ # whitespace and then return the path which becomes the second element
+ # in the array
+ return re.split(r'\s+', result.stdout.decode("utf-8"))[1]
+ except Exception as e:
+ message = f"""
+ Attempt to find {field} of {library} using ghc-pkg failed.
+ ghc-pkg path: {config.ghc_pkg}
+ error" {e}
+ """
+ print(message)
+
+
+def _find_so(lib, directory, in_place):
+ """Find a shared object file (.so) for lib in directory. We deliberately
+ keep the regex simple, just removing the ghc version and project version.
+ Example:
+
+ _find_so("Cabal-syntax-3.11.0.0", path-from-ghc-pkg, True) ==>
+ /builds/ghc/ghc/_build/install/lib/ghc-9.11.20240410/lib/x86_64-linux-ghc-9.11.20240410/libHSCabal-syntax-3.11.0.0-inplace-ghc9.11.20240410.so
+ """
+
+ # produce the suffix for the CI operating system
+ suffix = "so"
+ if config.os == "mingw32":
+ suffix = "dll"
+ elif config.os == "darwin":
+ suffix = "dylib"
+
+ # Most artfacts are of the form foo-inplace, except for the rts.
+ if in_place:
+ to_match = r'libHS{}-\d+(\.\d+)+-inplace-\S+\.' + suffix
+ else:
+ to_match = r'libHS{}-\d+(\.\d+)+\S+\.' + suffix
+
+ matches = []
+ # wrap this in some exception handling, hadrian test will error out because
+ # these files don't exist yet, so we pass when this occurs
+ try:
+ for f in os.listdir(directory):
+ if f.endswith(suffix):
+ pattern = re.compile(to_match.format(re.escape(lib)))
+ match = re.match(pattern, f)
+ if match:
+ matches.append(match.group())
+ return os.path.join(directory, matches[0])
+ except:
+ failBecause('Could not find shared object file: ' + lib)
+
+def find_so(lib):
+ return _find_so(lib,path_from_ghcPkg(lib, "dynamic-library-dirs"),True)
+
+def find_non_inplace_so(lib):
+ return _find_so(lib,path_from_ghcPkg(lib, "dynamic-library-dirs"),False)
+
# Define the a generic stat test, which computes the statistic by calling the function
# given as the third argument.
def collect_generic_stat ( metric, deviation, get_stat ):
@@ -804,9 +916,9 @@ KNOWN_OPERATING_SYSTEMS = set([
])
def exe_extension() -> str:
- if config.arch == 'wasm32':
+ if wasm_arch():
return '.wasm'
- elif config.os == "mingw32":
+ elif on_windows():
return '.exe'
return ''
@@ -829,6 +941,9 @@ def cygwin( ) -> bool:
def js_arch() -> bool:
return arch("javascript");
+def on_windows() -> bool:
+ return config.os == "mingw32"
+
def wasm_arch() -> bool:
return arch("wasm32")
diff --git a/testsuite/tests/perf/size/all.T b/testsuite/tests/perf/size/all.T
index 3b1b46e78ad4..647c51d48ee4 100644
--- a/testsuite/tests/perf/size/all.T
+++ b/testsuite/tests/perf/size/all.T
@@ -3,4 +3,77 @@ test('size_hello_obj', [collect_size(5, 'size_hello_obj.o')], compile, [''])
test('size_hello_artifact', [collect_size(5, 'size_hello_artifact' + exe_extension())],
compile_artifact, [''])
-test('libdir',[collect_size_dir(10, config.libdir)], static_stats, [] )
+test('array_dir' ,[collect_size_ghc_pkg(5 , 'array')] , static_stats , [] )
+test('base_dir' ,[collect_size_ghc_pkg(5 , 'base')] , static_stats , [] )
+test('binary_dir' ,[collect_size_ghc_pkg(5 , 'binary')] , static_stats , [] )
+test('bytestring_dir' ,[collect_size_ghc_pkg(5 , 'bytestring')] , static_stats , [] )
+test('cabal_dir' ,[collect_size_ghc_pkg(5 , 'Cabal')] , static_stats , [] )
+test('cabal_syntax_dir' ,[collect_size_ghc_pkg(5 , 'Cabal-syntax')] , static_stats , [] )
+test('containers_dir' ,[collect_size_ghc_pkg(5 , 'containers')] , static_stats , [] )
+test('deepseq_dir' ,[collect_size_ghc_pkg(5 , 'deepseq')] , static_stats , [] )
+test('directory_dir' ,[collect_size_ghc_pkg(5 , 'directory')] , static_stats , [] )
+test('exceptions_dir' ,[collect_size_ghc_pkg(5 , 'exceptions')] , static_stats , [] )
+test('ghc_bignum_dir' ,[collect_size_ghc_pkg(5 , 'ghc-bignum')] , static_stats , [] )
+test('ghc_boot_dir' ,[collect_size_ghc_pkg(5 , 'ghc-boot')] , static_stats , [] )
+test('ghc_boot_th_dir' ,[collect_size_ghc_pkg(5 , 'ghc-boot-th')] , static_stats , [] )
+test('ghc_compact_dir' ,[collect_size_ghc_pkg(5 , 'ghc-compact')] , static_stats , [] )
+test('ghc_dir' ,[collect_size_ghc_pkg(5 , 'ghc')] , static_stats , [] )
+test('ghc_experimental_dir',[collect_size_ghc_pkg(5 , 'ghc-experimental')], static_stats , [] )
+test('ghc_heap_dir' ,[collect_size_ghc_pkg(5 , 'ghc-heap')] , static_stats , [] )
+test('ghc_internal_dir' ,[collect_size_ghc_pkg(5 , 'ghc-internal')] , static_stats , [] )
+test('ghc_platform_dir' ,[collect_size_ghc_pkg(5 , 'ghc-platform')] , static_stats , [] )
+test('ghc_prim_dir' ,[collect_size_ghc_pkg(5 , 'ghc-prim')] , static_stats , [] )
+test('ghc_toolchain_dir' ,[collect_size_ghc_pkg(5 , 'ghc-toolchain')] , static_stats , [] )
+test('haskeline_dir' ,[collect_size_ghc_pkg(5 , 'haskeline')] , static_stats , [] )
+test('hpc_dir' ,[collect_size_ghc_pkg(5 , 'hpc')] , static_stats , [] )
+test('integer_gmp_dir' ,[collect_size_ghc_pkg(5 , 'integer-gmp')] , static_stats , [] )
+test('mtl_dir' ,[collect_size_ghc_pkg(5 , 'mtl')] , static_stats , [] )
+test('os_string_dir' ,[collect_size_ghc_pkg(5 , 'os-string')] , static_stats , [] )
+test('parsec_dir' ,[collect_size_ghc_pkg(5 , 'parsec')] , static_stats , [] )
+test('pretty_dir' ,[collect_size_ghc_pkg(5 , 'pretty')] , static_stats , [] )
+test('process_dir' ,[collect_size_ghc_pkg(5 , 'process')] , static_stats , [] )
+test('time_dir' ,[collect_size_ghc_pkg(5 , 'time')] , static_stats , [] )
+test('xhtml_dir' ,[collect_size_ghc_pkg(5 , 'xhtml')] , static_stats , [] )
+
+# size of the entire libdir
+test('libdir' ,[collect_size_dir(10, config.libdir)] , static_stats , [] )
+
+# skip these on windows
+test('unix_dir' ,[windows_skip, collect_size_ghc_pkg(5, 'unix')] , static_stats, [] )
+test('terminfo_dir' ,[windows_skip, js_skip, collect_size_ghc_pkg(5, 'terminfo')], static_stats, [] )
+
+# skip the shared object file tests on windows
+test('array_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "array")] , static_stats, [] )
+test('base_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "base")] , static_stats, [] )
+test('binary_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "binary")] , static_stats, [] )
+test('bytestring_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "bytestring")] , static_stats, [] )
+test('cabal_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "Cabal")] , static_stats, [] )
+test('cabal_syntax_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "Cabal-syntax")] , static_stats, [] )
+test('containers_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "containers")] , static_stats, [] )
+test('deepseq_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "deepseq")] , static_stats, [] )
+test('directory_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "directory")] , static_stats, [] )
+test('exceptions_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "exceptions")] , static_stats, [] )
+test('filepath_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "filepath")] , static_stats, [] )
+test('ghc_bignum_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "ghc-bignum")] , static_stats, [] )
+test('ghc_boot_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "ghc-boot")] , static_stats, [] )
+test('ghc_boot_th_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "ghc-boot-th")] , static_stats, [] )
+test('ghc_experimental_so',[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "ghc-experimental")] , static_stats, [] )
+test('ghc_heap_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "ghc-heap")] , static_stats, [] )
+test('ghc_platform_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "ghc-platform")] , static_stats, [] )
+test('ghc_prim_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "ghc-prim")] , static_stats, [] )
+test('ghc_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "ghc")] , static_stats, [] )
+test('ghc_toolchain_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "ghc-toolchain")] , static_stats, [] )
+test('ghci_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "ghci")] , static_stats, [] )
+test('haskeline_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "haskeline")] , static_stats, [] )
+test('hpc_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "hpc")] , static_stats, [] )
+test('mtl_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "mtl")] , static_stats, [] )
+test('os_string_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "os-string")] , static_stats, [] )
+test('parsec_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "parsec")] , static_stats, [] )
+test('process_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "process")] , static_stats, [] )
+test('rts_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "rts", True)] , static_stats, [] )
+test('template_haskell_so',[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "template-haskell")] , static_stats, [] )
+test('terminfo_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "terminfo")] , static_stats, [] )
+test('text_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "text")] , static_stats, [] )
+test('time_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "time")] , static_stats, [] )
+test('transformers_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "transformers")] , static_stats, [] )
+test('xhtml_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "xhtml")] , static_stats, [] )
--
GitLab