Commit 3e07ea8d authored by Ben Gamari's avatar Ben Gamari 🐢

testsuite: Use small allocation area when measuring residency

As suggested in #17387; this helps reduce the variance in our residency
sampling.

Metric Increase:
    T10370
    T3586
    lazy-bs-alloc

Metric Decrease 'compile_time/peak_megabytes_allocated':
    T1969

Metric Decrease 'runtime/bytes allocated':
    space_leak_001

Metric Increase 'compile_time/bytes allocated':
    T1969

Metric Increase 'runtime/peak_megabytes_allocated':
    space_leak_001

Metric Decrease:
    T3064
    T9675
parent a9467f4f
......@@ -556,6 +556,50 @@ def llvm_build ( ) -> bool:
# ---
# Note [Measuring residency]
# ~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Residency (peak_megabytes_allocated and max_bytes_used) is sensitive
# to when the major GC runs, which makes it inherently inaccurate.
# Sometime an innocuous change somewhere can shift things around such
# that the samples occur at a different time, and the residency
# appears to change (up or down) when the underlying profile hasn't
# really changed. To further minimize this effect we run with a single
# generation (meaning we get a residency sample on every GC) with a small
# allocation area (as suggested in #17387).
#
# However, please don't just ignore changes in residency. If you see
# a change in one of these figures, please check whether it is real or
# not as follows:
#
# * Run the test with old and new compilers, adding +RTS -h -i0.01
# (you don't need to compile anything for profiling or enable profiling
# libraries to get a heap profile).
# * view the heap profiles, read off the maximum residency. If it has
# really changed, then you know there's an issue.
RESIDENCY_OPTS = '+RTS -A256k -i0 -h -RTS'
# See Note [Measuring residency].
def collect_runtime_residency(tolerance_pct: float):
return [
collect_compiler_stats(['peak_megabytes_allocated', 'max_bytes_used'], tolerance_pct),
extra_run_opts(RESIDENCY_OPTS),
# The nonmoving collector does not support -G1
omit_ways([WayName(name) for name in ['nonmoving', 'nonmoving_thr', 'nonmoving_thr_ghc']])
]
# See Note [Measuring residency].
def collect_compiler_residency(tolerance_pct: float):
return [
collect_compiler_stats(['peak_megabytes_allocated', 'max_bytes_used'], tolerance_pct),
extra_hc_opts(RESIDENCY_OPTS),
# The nonmoving collector does not support -G1
omit_ways([WayName('nonmoving_thr_ghc')])
]
# ---
def high_memory_usage(name, opts):
opts.alone = True
......
......@@ -7,39 +7,16 @@ def no_lint(name, opts):
setTestOpts(no_lint)
# Note [residency]
#
# Residency (peak_megabytes_allocated and max_bytes_used) is sensitive
# to when the major GC runs, which makes it inherently inaccurate.
# Sometime an innocuous change somewhere can shift things around such
# that the samples occur at a different time, and the residency
# appears to change (up or down) when the underlying profile hasn't
# really changed.
#
# However, please don't just ignore changes in residency. If you see
# a change in one of these figures, please check whether it is real or
# not as follows:
#
# * Run the test with old and new compilers, adding +RTS -h -i0.01
# (you don't need to compile anything for profiling or enable profiling
# libraries to get a heap profile).
# * view the heap profiles, read off the maximum residency. If it has
# really changed, then you know there's an issue.
test('T1969',
[# expect_broken(12437),
collect_compiler_stats(['peak_megabytes_allocated','max_bytes_used'],15),
collect_compiler_residency(15),
collect_compiler_stats('bytes allocated', 1),
only_ways(['normal']),
extra_hc_opts('-dcore-lint -static'),
# Leave -dcore-lint on for this one test, so that we have something
# that will catch a regression in -dcore-lint performance.
# Use `+RTS -G1` for more stable residency measurements. Note [residency].
extra_hc_opts('+RTS -G1 -RTS')
],
],
compile,
[''])
......@@ -56,44 +33,26 @@ else:
conf_3294 = skip
test('T3294',
[
collect_compiler_stats('max_bytes_used',15),
[collect_compiler_residency(15),
collect_compiler_stats('bytes allocated', 1),
conf_3294,
# Use `+RTS -G1` for more stable residency measurements. Note [residency].
extra_hc_opts('+RTS -G1 -RTS')
],
compile,
[''])
test('T4801',
[# collect_compiler_stats('peak_megabytes_allocated',1),
# expect_broken(5224),
# temporarily unbroken (#5227)
# deactivated for now, as this metric became too volatile recently
collect_compiler_stats('bytes allocated',2),
# collect_compiler_stats('max_bytes_used',5),
[collect_compiler_stats('bytes allocated',2),
collect_compiler_residency(2),
only_ways(['normal']),
extra_hc_opts('-static'),
# Use `+RTS -G1` for more stable residency measurements. Note [residency].
extra_hc_opts('+RTS -G1 -RTS')
],
compile,
[''])
test('T3064',
[collect_compiler_stats('peak_megabytes_allocated',20),
[collect_compiler_residency(20),
collect_compiler_stats('bytes allocated',2),
# deactivated for now, as this metric became too volatile recently
# collect_compiler_stats('max_bytes_used',20)
only_ways(['normal']),
# Use `+RTS -G1` for more stable residency measurements. Note [residency].
extra_hc_opts('+RTS -G1 -RTS')
only_ways(['normal']),
],
compile,
[''])
......@@ -168,12 +127,8 @@ test('T9020',
test('T9675',
[ only_ways(['optasm']),
# Note [residency]
collect_compiler_stats(['max_bytes_used','peak_megabytes_allocated'],15),
collect_compiler_residency(15),
collect_compiler_stats('bytes allocated',2),
# Use `+RTS -G1` for more stable residency measurements. Note [residency].
extra_hc_opts('+RTS -G1 -RTS')
],
compile,
[''])
......@@ -221,10 +176,7 @@ test('T9233',
test('T10370',
[ only_ways(['optasm']),
collect_compiler_stats(['max_bytes_used','peak_megabytes_allocated'], 15),
# Use `+RTS -G1` for more stable residency measurements. Note [residency].
extra_hc_opts('+RTS -G1 -RTS')
collect_compiler_residency(15),
],
compile,
[''])
......@@ -381,13 +333,8 @@ test('Naperian',
[''])
test ('T9630',
[ collect_compiler_stats('max_bytes_used',15), # Note [residency]
extra_clean(['T9630a.hi', 'T9630a.o']),
# Use `+RTS -G1` for more stable residency measurements. Note [residency].
extra_hc_opts('+RTS -G1 -RTS'),
# The nonmoving collector does not support -G1
omit_ways(['nonmoving', 'nonmoving_thr', 'nonmoving_thr_ghc'])
[ collect_compiler_residency(15),
extra_clean(['T9630a.hi', 'T9630a.o']),
],
multimod_compile,
['T9630', '-v0 -O'])
......
......@@ -15,7 +15,7 @@
# We do not add peak_megabytes_allocated and max_bytes_used to these tests, as
# they are somewhat unreliable, and it is harder to re-run these numbers to
# detect outliers, as described in Note [residency]. See #9556.
# detect outliers, as described in Note [Measuring residency]. See #9556.
test('haddock.base',
[unless(in_tree_compiler(), skip), req_haddock
......
......@@ -3,15 +3,15 @@
# See Note [Solving from instances when interacting Dicts]
test('T5835',
[collect_stats('max_bytes_used',10),
only_ways(['normal'])
],
[collect_runtime_residency(10),
only_ways(['normal'])
],
compile_and_run,
['-O'])
test('T12791',
[collect_stats('max_bytes_used',10),
only_ways(['normal'])
[collect_runtime_residency(10),
only_ways(['normal'])
],
compile_and_run,
['-O'])
......@@ -35,13 +35,9 @@ test('T14955',
# fortunately the values here are mostly independent of the wordsize,
# because the test allocates an unboxed array of doubles.
test('T3586',
[collect_stats('peak_megabytes_allocated',1),
[collect_runtime_residency(2),
collect_stats('bytes allocated', 5),
only_ways(['normal']),
# Use `+RTS -G1` for more stable residency measurements. Note [residency].
# Only 64-bit as we don't have a good 32-bit test environment at the moment
when(wordsize(64), extra_hc_opts('+RTS -G1 -RTS'))
],
compile_and_run,
['-O'])
......@@ -61,15 +57,11 @@ test('T3245', [when(doing_ghci(), extra_hc_opts('-fobject-code'))],
#
test('lazy-bs-alloc',
[extra_files(['../../numeric/should_run/arith011.stdout']),
collect_stats('peak_megabytes_allocated', 1),
collect_stats('bytes allocated',5),
collect_runtime_residency(1),
only_ways(['normal']),
extra_run_opts('arith011.stdout'),
ignore_stdout,
# Use `+RTS -G1` for more stable residency measurements. Note [residency].
# Only 64-bit as we don't have a good 32-bit test environment at the moment
when(wordsize(64), extra_hc_opts('+RTS -G1 -RTS'))
],
# use a suitably big file, without bloating the repo with a new one:
compile_and_run,
......@@ -374,4 +366,4 @@ test('UniqLoop',
only_ways(['normal'])
],
compile_and_run,
['-O -package ghc'])
\ No newline at end of file
['-O -package ghc'])
......@@ -4,7 +4,7 @@ test('space_leak_001',
# collect_stats('all',5) to test all 3 with
# 5% possible deviation.
[collect_stats(['peak_megabytes_allocated','bytes allocated'],5),
collect_stats('max_bytes_used',15),
collect_runtime_residency(15),
omit_ways(['profasm','profthreaded','threaded1','threaded2','nonmoving_thr']),
reqlib('integer-gmp')
],
......@@ -32,7 +32,8 @@ test('T4018',
compile_and_run, ['-fno-state-hack'])
test('T4029',
[collect_stats(['peak_megabytes_allocated','max_bytes_used'],10),
[collect_stats(['peak_megabytes_allocated'],10),
collect_runtime_residency(10),
extra_hc_opts('+RTS -G1 -RTS' ),
],
ghci_script,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment