testsuite: Use small allocation area when measuring residency

As suggested in #17387; this helps reduce the variance in our residency
sampling.

Metric Increase:
    T10370
    T3586
    lazy-bs-alloc

Metric Decrease 'compile_time/peak_megabytes_allocated':
    T1969

Metric Decrease 'runtime/bytes allocated':
    space_leak_001

Metric Increase 'compile_time/bytes allocated':
    T1969

Metric Increase 'runtime/peak_megabytes_allocated':
    space_leak_001

Metric Decrease:
    T3064
    T9675

testsuite/driver/testlib.py

@@ -556,6 +556,50 @@ def llvm_build ( ) -> bool:
 
 # ---
 
+# Note [Measuring residency]
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~
+#
+# Residency (peak_megabytes_allocated and max_bytes_used) is sensitive
+# to when the major GC runs, which makes it inherently inaccurate.
+# Sometime an innocuous change somewhere can shift things around such
+# that the samples occur at a different time, and the residency
+# appears to change (up or down) when the underlying profile hasn't
+# really changed. To further minimize this effect we run with a single
+# generation (meaning we get a residency sample on every GC) with a small
+# allocation area (as suggested in #17387).
+#
+# However, please don't just ignore changes in residency.  If you see
+# a change in one of these figures, please check whether it is real or
+# not as follows:
+#
+#  * Run the test with old and new compilers, adding +RTS -h -i0.01
+#    (you don't need to compile anything for profiling or enable profiling
+#    libraries to get a heap profile).
+#  * view the heap profiles, read off the maximum residency.  If it has
+#    really changed, then you know there's an issue.
+
+RESIDENCY_OPTS = '+RTS -A256k -i0 -h -RTS'
+
+# See Note [Measuring residency].
+def collect_runtime_residency(tolerance_pct: float):
+    return [
+        collect_compiler_stats(['peak_megabytes_allocated', 'max_bytes_used'], tolerance_pct),
+        extra_run_opts(RESIDENCY_OPTS),
+        # The nonmoving collector does not support -G1
+        omit_ways([WayName(name) for name in ['nonmoving', 'nonmoving_thr', 'nonmoving_thr_ghc']])
+    ]
+
+# See Note [Measuring residency].
+def collect_compiler_residency(tolerance_pct: float):
+    return [
+        collect_compiler_stats(['peak_megabytes_allocated', 'max_bytes_used'], tolerance_pct),
+        extra_hc_opts(RESIDENCY_OPTS),
+        # The nonmoving collector does not support -G1
+        omit_ways([WayName('nonmoving_thr_ghc')])
+    ]
+
+# ---
+
 def high_memory_usage(name, opts):
     opts.alone = True
 

testsuite/tests/perf/compiler/all.T

@@ -7,39 +7,16 @@ def no_lint(name, opts):
 
 setTestOpts(no_lint)
 
-
-# Note [residency]
-#
-# Residency (peak_megabytes_allocated and max_bytes_used) is sensitive
-# to when the major GC runs, which makes it inherently inaccurate.
-# Sometime an innocuous change somewhere can shift things around such
-# that the samples occur at a different time, and the residency
-# appears to change (up or down) when the underlying profile hasn't
-# really changed.
-#
-# However, please don't just ignore changes in residency.  If you see
-# a change in one of these figures, please check whether it is real or
-# not as follows:
-#
-#  * Run the test with old and new compilers, adding +RTS -h -i0.01
-#    (you don't need to compile anything for profiling or enable profiling
-#    libraries to get a heap profile).
-#  * view the heap profiles, read off the maximum residency.  If it has
-#    really changed, then you know there's an issue.
-
 test('T1969',
      [# expect_broken(12437),
-      collect_compiler_stats(['peak_megabytes_allocated','max_bytes_used'],15),
+      collect_compiler_residency(15),
       collect_compiler_stats('bytes allocated', 1),
       only_ways(['normal']),
 
       extra_hc_opts('-dcore-lint -static'),
           # Leave -dcore-lint on for this one test, so that we have something
           # that will catch a regression in -dcore-lint performance.
-
-      # Use `+RTS -G1` for more stable residency measurements. Note [residency].
-      extra_hc_opts('+RTS -G1 -RTS')
-      ],
+     ],
      compile,
      [''])
 
@@ -56,44 +33,26 @@ else:
    conf_3294 = skip
 
 test('T3294',
-     [
-      collect_compiler_stats('max_bytes_used',15),
+     [collect_compiler_residency(15),
       collect_compiler_stats('bytes allocated', 1),
       conf_3294,
-
-      # Use `+RTS -G1` for more stable residency measurements. Note [residency].
-      extra_hc_opts('+RTS -G1 -RTS')
       ],
      compile,
      [''])
 
 test('T4801',
-     [# collect_compiler_stats('peak_megabytes_allocated',1),
-      # expect_broken(5224),
-      # temporarily unbroken (#5227)
-      # deactivated for now, as this metric became too volatile recently
-      collect_compiler_stats('bytes allocated',2),
-      # collect_compiler_stats('max_bytes_used',5),
+     [collect_compiler_stats('bytes allocated',2),
+      collect_compiler_residency(2),
       only_ways(['normal']),
       extra_hc_opts('-static'),
-
-      # Use `+RTS -G1` for more stable residency measurements. Note [residency].
-      extra_hc_opts('+RTS -G1 -RTS')
       ],
      compile,
      [''])
 
 test('T3064',
-     [collect_compiler_stats('peak_megabytes_allocated',20),
+     [collect_compiler_residency(20),
       collect_compiler_stats('bytes allocated',2),
-
-      # deactivated for now, as this metric became too volatile recently
-      # collect_compiler_stats('max_bytes_used',20)
-
-       only_ways(['normal']),
-
-       # Use `+RTS -G1` for more stable residency measurements. Note [residency].
-       extra_hc_opts('+RTS -G1 -RTS')
+      only_ways(['normal']),
       ],
      compile,
      [''])
@@ -168,12 +127,8 @@ test('T9020',
 
 test('T9675',
      [ only_ways(['optasm']),
-       # Note [residency]
-       collect_compiler_stats(['max_bytes_used','peak_megabytes_allocated'],15),
+       collect_compiler_residency(15),
        collect_compiler_stats('bytes allocated',2),
-
-       # Use `+RTS -G1` for more stable residency measurements. Note [residency].
-       extra_hc_opts('+RTS -G1 -RTS')
       ],
      compile,
      [''])
@@ -221,10 +176,7 @@ test('T9233',
 
 test('T10370',
      [ only_ways(['optasm']),
-       collect_compiler_stats(['max_bytes_used','peak_megabytes_allocated'], 15),
-
-       # Use `+RTS -G1` for more stable residency measurements. Note [residency].
-       extra_hc_opts('+RTS -G1 -RTS')
+       collect_compiler_residency(15),
      ],
      compile,
      [''])
@@ -381,13 +333,8 @@ test('Naperian',
      [''])
 
 test ('T9630',
-      [ collect_compiler_stats('max_bytes_used',15), # Note [residency]
-      extra_clean(['T9630a.hi', 'T9630a.o']),
-
-      # Use `+RTS -G1` for more stable residency measurements. Note [residency].
-      extra_hc_opts('+RTS -G1 -RTS'),
-      # The nonmoving collector does not support -G1
-      omit_ways(['nonmoving', 'nonmoving_thr', 'nonmoving_thr_ghc'])
+      [ collect_compiler_residency(15),
+        extra_clean(['T9630a.hi', 'T9630a.o']),
       ],
       multimod_compile,
       ['T9630', '-v0 -O'])

testsuite/tests/perf/haddock/all.T

@@ -15,7 +15,7 @@
 
 # We do not add peak_megabytes_allocated and max_bytes_used to these tests, as
 # they are somewhat unreliable, and it is harder to re-run these numbers to
-# detect outliers, as described in Note [residency]. See #9556.
+# detect outliers, as described in Note [Measuring residency]. See #9556.
 
 test('haddock.base',
      [unless(in_tree_compiler(), skip), req_haddock

testsuite/tests/perf/should_run/all.T

@@ -3,15 +3,15 @@
 # See Note [Solving from instances when interacting Dicts]
 
 test('T5835',
-     [collect_stats('max_bytes_used',10),
-       only_ways(['normal'])
-       ],
+     [collect_runtime_residency(10),
+      only_ways(['normal'])
+      ],
      compile_and_run,
      ['-O'])
 
 test('T12791',
-     [collect_stats('max_bytes_used',10),
-       only_ways(['normal'])
+     [collect_runtime_residency(10),
+      only_ways(['normal'])
        ],
      compile_and_run,
      ['-O'])
@@ -35,13 +35,9 @@ test('T14955',
 # fortunately the values here are mostly independent of the wordsize,
 # because the test allocates an unboxed array of doubles.
 test('T3586',
-     [collect_stats('peak_megabytes_allocated',1),
+     [collect_runtime_residency(2),
       collect_stats('bytes allocated', 5),
       only_ways(['normal']),
-
-      # Use `+RTS -G1` for more stable residency measurements. Note [residency].
-      # Only 64-bit as we don't have a good 32-bit test environment at the moment
-      when(wordsize(64), extra_hc_opts('+RTS -G1 -RTS'))
       ],
      compile_and_run,
      ['-O'])
@@ -61,15 +57,11 @@ test('T3245', [when(doing_ghci(), extra_hc_opts('-fobject-code'))],
 #
 test('lazy-bs-alloc',
      [extra_files(['../../numeric/should_run/arith011.stdout']),
-      collect_stats('peak_megabytes_allocated', 1),
       collect_stats('bytes allocated',5),
+      collect_runtime_residency(1),
       only_ways(['normal']),
       extra_run_opts('arith011.stdout'),
       ignore_stdout,
-
-      # Use `+RTS -G1` for more stable residency measurements. Note [residency].
-      # Only 64-bit as we don't have a good 32-bit test environment at the moment
-      when(wordsize(64), extra_hc_opts('+RTS -G1 -RTS'))
       ],
      # use a suitably big file, without bloating the repo with a new one:
      compile_and_run,
@@ -374,4 +366,4 @@ test('UniqLoop',
       only_ways(['normal'])
       ],
      compile_and_run,
-     ['-O -package ghc'])
\ No newline at end of file
+     ['-O -package ghc'])

testsuite/tests/perf/space_leaks/all.T

@@ -4,7 +4,7 @@ test('space_leak_001',
      # collect_stats('all',5) to test all 3 with
      # 5% possible deviation.
      [collect_stats(['peak_megabytes_allocated','bytes allocated'],5),
-      collect_stats('max_bytes_used',15),
+      collect_runtime_residency(15),
       omit_ways(['profasm','profthreaded','threaded1','threaded2','nonmoving_thr']),
       reqlib('integer-gmp')
       ],
@@ -32,7 +32,8 @@ test('T4018',
      compile_and_run, ['-fno-state-hack'])
 
 test('T4029',
-     [collect_stats(['peak_megabytes_allocated','max_bytes_used'],10),
+     [collect_stats(['peak_megabytes_allocated'],10),
+      collect_runtime_residency(10),
       extra_hc_opts('+RTS -G1 -RTS' ),
       ],
      ghci_script,