From 7a75a09403264c60a1f513b7466dc9503b966aab Mon Sep 17 00:00:00 2001
From: Alp Mestanogullari <alpmestan@gmail.com>
Date: Fri, 26 Apr 2019 14:25:46 +0200
Subject: [PATCH] testsuite: introduce 'static_stats' tests
They are a particular type of perf tests. This patch introduces a
'stats_files_dir' configuration field in the testsuite driver where all
haddock timing files (and possibly others in the future) are assumed to live.
We also change both the Make and Hadrian build systems to pass respectively
$(TOP)/testsuite/tests/perf/haddock/ and
<build root>/stage1/haddock-timing-files/ as the value of that new
configuration field, and to generate the timing files in those directories
in the first place while generating documentation with haddock.
This new test type can be seen as one dedicated to examining stats files that
are generated while building a GHC distribution. This also lets us get rid of
the 'extra_files' directives in the all.T entries for haddock.base,
haddock.Cabal and haddock.compiler.
---
hadrian/src/Context.hs | 3 +-
hadrian/src/Context/Path.hs | 5 ++++
hadrian/src/Rules/Documentation.hs | 5 ++++
hadrian/src/Settings/Builders/Haddock.hs | 4 +--
hadrian/src/Settings/Builders/RunTest.hs | 6 +++-
rules/haddock.mk | 2 +-
testsuite/driver/testglobals.py | 5 ++++
testsuite/driver/testlib.py | 16 +++++++----
testsuite/mk/test.mk | 2 ++
testsuite/tests/perf/haddock/all.T | 36 ++++++++++++++++--------
10 files changed, 62 insertions(+), 22 deletions(-)
diff --git a/hadrian/src/Context.hs b/hadrian/src/Context.hs
index 7c7bb124ff8f..0676743ee588 100644
--- a/hadrian/src/Context.hs
+++ b/hadrian/src/Context.hs
@@ -8,7 +8,8 @@ module Context (
-- * Paths
contextDir, buildPath, buildDir, pkgInplaceConfig, pkgSetupConfigFile,
pkgHaddockFile, pkgRegisteredLibraryFile, pkgLibraryFile, pkgGhciLibraryFile,
- pkgConfFile, objectPath, contextPath, getContextPath, libPath, distDir
+ pkgConfFile, objectPath, contextPath, getContextPath, libPath, distDir,
+ haddockStatsFilesDir
) where
import Base
diff --git a/hadrian/src/Context/Path.hs b/hadrian/src/Context/Path.hs
index 4bc9d9be34f5..e5a82710ab97 100644
--- a/hadrian/src/Context/Path.hs
+++ b/hadrian/src/Context/Path.hs
@@ -41,3 +41,8 @@ buildPath context = buildRoot <&> (-/- buildDir context)
-- | The expression that evaluates to the build path of the current 'Context'.
getBuildPath :: Expr Context b FilePath
getBuildPath = expr . buildPath =<< getContext
+
+-- | Path to the directory containing haddock timing files, used by
+-- the haddock perf tests.
+haddockStatsFilesDir :: Action FilePath
+haddockStatsFilesDir = (-/- "stage1" -/- "haddock-timing-files") <$> buildRoot
diff --git a/hadrian/src/Rules/Documentation.hs b/hadrian/src/Rules/Documentation.hs
index a70b4d5c9d40..9991f017080d 100644
--- a/hadrian/src/Rules/Documentation.hs
+++ b/hadrian/src/Rules/Documentation.hs
@@ -205,7 +205,12 @@ buildPackageDocumentation = do
-- TODO: Pass the correct way from Rules via Context.
dynamicPrograms <- dynamicGhcPrograms =<< flavour
let haddockWay = if dynamicPrograms then dynamic else vanilla
+ statsFilesDir <- haddockStatsFilesDir
+ createDirectory statsFilesDir
build $ target (context {way = haddockWay}) (Haddock BuildPackage) srcs [file]
+ produces [
+ statsFilesDir </> pkgName (Context.package context) <.> "t"
+ ]
data PkgDocTarget = DotHaddock PackageName | HaddockPrologue PackageName
deriving (Eq, Show)
diff --git a/hadrian/src/Settings/Builders/Haddock.hs b/hadrian/src/Settings/Builders/Haddock.hs
index 65c703160cda..b01b8a245902 100644
--- a/hadrian/src/Settings/Builders/Haddock.hs
+++ b/hadrian/src/Settings/Builders/Haddock.hs
@@ -35,13 +35,13 @@ haddockBuilderArgs = mconcat
output <- getOutput
pkg <- getPackage
root <- getBuildRoot
- path <- getBuildPath
context <- getContext
version <- expr $ pkgVersion pkg
synopsis <- expr $ pkgSynopsis pkg
deps <- getContextData depNames
haddocks <- expr $ haddockDependencies context
hVersion <- expr $ pkgVersion haddock
+ statsDir <- expr $ haddockStatsFilesDir
ghcOpts <- haddockGhcArgs
mconcat
[ arg "--verbosity=0"
@@ -66,6 +66,6 @@ haddockBuilderArgs = mconcat
, pure [ "--optghc=" ++ opt | opt <- ghcOpts, not ("--package-db" `isInfixOf` opt) ]
, getInputs
, arg "+RTS"
- , arg $ "-t" ++ path -/- "haddock.t"
+ , arg $ "-t" ++ (statsDir -/- pkgName pkg ++ ".t")
, arg "--machine-readable"
, arg "-RTS" ] ]
diff --git a/hadrian/src/Settings/Builders/RunTest.hs b/hadrian/src/Settings/Builders/RunTest.hs
index 6cc11f8aef5a..63e3dfd6c951 100644
--- a/hadrian/src/Settings/Builders/RunTest.hs
+++ b/hadrian/src/Settings/Builders/RunTest.hs
@@ -85,11 +85,14 @@ runTestBuilderArgs = builder RunTest ? do
wordsize <- getTestSetting TestWORDSIZE
top <- expr $ topDirectory
ghcFlags <- expr runTestGhcFlags
- timeoutProg <- expr buildRoot <&> (-/- timeoutPath)
cmdrootdirs <- expr (testRootDirs <$> userSetting defaultTestArgs)
let defaultRootdirs = ("testsuite" -/- "tests") : libTests
rootdirs | null cmdrootdirs = defaultRootdirs
| otherwise = cmdrootdirs
+ root <- expr buildRoot
+ let timeoutProg = root -/- timeoutPath
+ statsFilesDir <- expr haddockStatsFilesDir
+
-- See #16087
let ghcBuiltByLlvm = False -- TODO: Implement this check
@@ -134,6 +137,7 @@ runTestBuilderArgs = builder RunTest ? do
, arg "--config", arg $ "gs=gs" -- Use the default value as in test.mk
, arg "--config", arg $ "timeout_prog=" ++ show (top -/- timeoutProg)
+ , arg "--config", arg $ "stats_files_dir=" ++ statsFilesDir
, arg $ "--threads=" ++ show threads
, getTestArgs -- User-provided arguments from command line.
]
diff --git a/rules/haddock.mk b/rules/haddock.mk
index f0d686194fec..a0e4f1724ab7 100644
--- a/rules/haddock.mk
+++ b/rules/haddock.mk
@@ -76,7 +76,7 @@ endif
$$($1_$2_HS_SRCS) \
$$($1_$2_EXTRA_HADDOCK_SRCS) \
$$(EXTRA_HADDOCK_OPTS) \
- +RTS -t"$1/$2/haddock.t" --machine-readable
+ +RTS -t"$$(TOP)/testsuite/tests/perf/haddock/$$($1_PACKAGE).t" --machine-readable
# --no-tmp-comp-dir above is important: it saves a few minutes in a
# validate. This flag lets Haddock use the pre-compiled object files
diff --git a/testsuite/driver/testglobals.py b/testsuite/driver/testglobals.py
index b28477c76904..ababefba1954 100644
--- a/testsuite/driver/testglobals.py
+++ b/testsuite/driver/testglobals.py
@@ -139,6 +139,11 @@ class TestConfig:
# terminal supports colors
self.supports_colors = False
+ # Where to look up runtime stats produced by haddock, needed for
+ # the haddock perf tests in testsuite/tests/perf/haddock/.
+ # See Note [Haddock runtime stats files] at the bottom of this file.
+ self.stats_files_dir = '/please_set_stats_files_dir'
+
global config
config = TestConfig()
diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py
index dc8b1b85f143..d0bd98015dd4 100644
--- a/testsuite/driver/testlib.py
+++ b/testsuite/driver/testlib.py
@@ -67,7 +67,6 @@ def isStatsTest():
opts = getTestOpts()
return opts.is_stats_test
-
# This can be called at the top of a file of tests, to set default test options
# for the following tests.
def setTestOpts( f ):
@@ -1211,7 +1210,11 @@ def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
def stats( name, way, stats_file ):
opts = getTestOpts()
- return check_stats(name, way, stats_file, opts.stats_range_fields)
+ return check_stats(name, way, in_testdir(stats_file), opts.stats_range_fields)
+
+def static_stats( name, way, stats_file ):
+ opts = getTestOpts()
+ return check_stats(name, way, in_statsdir(stats_file), opts.stats_range_fields)
def metric_dict(name, way, metric, value):
return Perf.PerfStat(
@@ -1234,7 +1237,7 @@ def check_stats(name, way, stats_file, range_fields):
result = passed()
if range_fields:
try:
- f = open(in_testdir(stats_file))
+ f = open(stats_file)
except IOError as e:
return failBecause(str(e))
stats_file_contents = f.read()
@@ -1357,7 +1360,7 @@ def simple_build(name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, b
# ToDo: if the sub-shell was killed by ^C, then exit
if isCompilerStatsTest():
- statsResult = check_stats(name, way, stats_file, opts.stats_range_fields)
+ statsResult = check_stats(name, way, in_testdir(stats_file), opts.stats_range_fields)
if badResult(statsResult):
return statsResult
@@ -1442,7 +1445,7 @@ def simple_run(name, way, prog, extra_run_opts):
if check_prof and not check_prof_ok(name, way):
return failBecause('bad profile')
- return check_stats(name, way, stats_file, opts.stats_range_fields)
+ return check_stats(name, way, in_testdir(stats_file), opts.stats_range_fields)
def rts_flags(way):
args = config.way_rts_flags.get(way, [])
@@ -2103,6 +2106,9 @@ def in_testdir(name, suffix=''):
def in_srcdir(name, suffix=''):
return os.path.join(getTestOpts().srcdir, add_suffix(name, suffix))
+def in_statsdir(name, suffix=''):
+ return os.path.join(config.stats_files_dir, add_suffix(name, suffix))
+
# Finding the sample output. The filename is of the form
#
# <test>.stdout[-ws-<wordsize>][-<platform>|-<os>]
diff --git a/testsuite/mk/test.mk b/testsuite/mk/test.mk
index b1d716fd7788..6860974c54e3 100644
--- a/testsuite/mk/test.mk
+++ b/testsuite/mk/test.mk
@@ -278,6 +278,8 @@ RUNTEST_OPTS += \
--config 'gs=$(call quote_path,$(GS))' \
--config 'timeout_prog=$(call quote_path,$(TIMEOUT_PROGRAM))'
+RUNTEST_OPTS += --config 'stats_files_dir=$(TOP)/tests/perf/haddock'
+
RUNTEST_OPTS += -e "config.stage=$(GhcStage)"
ifneq "$(METRICS_FILE)" ""
diff --git a/testsuite/tests/perf/haddock/all.T b/testsuite/tests/perf/haddock/all.T
index fca30366f954..929a7fd972f2 100644
--- a/testsuite/tests/perf/haddock/all.T
+++ b/testsuite/tests/perf/haddock/all.T
@@ -1,27 +1,39 @@
+# Note [Haddock runtime stats files]
+#
+# When one of the build systems builds a complete GHC distribution,
+# haddock gets built and then used to generate .haddock files for each
+# library. For that last step, both build systems pass an extra
+# `+RTS -t<some path>.t` to record runtime statistics to the given path.
+#
+# Those .t files are then used by a few haddock perf tests (which all live
+# under testsuite/tests/perf/haddock/). Since each build system needs to produce
+# those files in different places, the testsuite driver takes the directory
+# under which those files are placed as a configuration parameter,
+# `config.stats_files_dir`. Each individual test then specifies the name of
+# the (runtime statistics) file against which some checks are to be performed,
+# in addition to declaring the test's type to be `static_stats`.
+
# We do not add peak_megabytes_allocated and max_bytes_used to these tests, as
# they are somewhat unreliable, and it is harder to re-run these numbers to
# detect outliers, as described in Note [residency]. See #9556.
test('haddock.base',
- [extra_files(['../../../../libraries/base/dist-install/haddock.t']),
- unless(in_tree_compiler(), skip), req_haddock
+ [unless(in_tree_compiler(), skip), req_haddock
,collect_stats('bytes allocated',5)
],
- stats,
- ['haddock.t'])
+ static_stats,
+ ['base.t'])
test('haddock.Cabal',
- [extra_files(['../../../../libraries/Cabal/Cabal/dist-install/haddock.t']),
- unless(in_tree_compiler(), skip), req_haddock
+ [unless(in_tree_compiler(), skip), req_haddock
,collect_stats('bytes allocated',5)
],
- stats,
- ['haddock.t'])
+ static_stats,
+ ['Cabal.t'])
test('haddock.compiler',
- [extra_files(['../../../../compiler/stage2/haddock.t']),
- unless(in_tree_compiler(), skip), req_haddock
+ [unless(in_tree_compiler(), skip), req_haddock
,collect_stats('bytes allocated',10)
],
- stats,
- ['haddock.t'])
+ static_stats,
+ ['ghc.t'])
--
GitLab