gitlab-ci: Use MR base commit as performance baseline

.gitlab-ci.yml

@@ -26,12 +26,28 @@ stages:
   - testing     # head.hackage correctness and compiler performance testing
   - deploy      # push documentation
 
+# Note [The CI Story]
+# ~~~~~~~~~~~~~~~~~~~
+#
+# There are two different types of pipelines:
+#
+#  - marge-bot merges to `master`. Here we perform an exhaustive validation
+#    across all of the platforms which we support. In addition, we push
+#    performance metric notes upstream, providing a persistent record of the
+#    performance characteristics of the compiler.
+#
+#  - merge requests. Here we perform a slightly less exhaustive battery of
+#    testing. Namely we omit some configurations (e.g. the unregisterised job).
+#    These use the merge request's base commit for performance metric
+#    comparisons.
+#
+
 workflow:
-  # N.B.Don't run on wip/ branches, instead on run on merge requests.
+  # N.B. Don't run on wip/ branches, instead on run on merge requests.
   rules:
     - if: $CI_MERGE_REQUEST_ID
     - if: $CI_COMMIT_TAG
-    - if: '$CI_COMMIT_BRANCH == "master"'
+    - if: '$CI_COMMIT_BRANCH == "wip/marge_bot_batch_merge_job"'
     - if: '$CI_COMMIT_BRANCH =~ /ghc-[0.9]+\.[0-9]+/'
     - if: '$CI_PIPELINE_SOURCE == "web"'
 

.gitlab/ci.sh

@@ -363,6 +363,13 @@ function push_perf_notes() {
   "$TOP/.gitlab/test-metrics.sh" push
 }
 
+# Figure out which commit should be used by the testsuite driver as a
+# performance baseline. See Note [The CI Story].
+function determine_metric_baseline() {
+  export PERF_BASELINE_COMMIT="$(git merge-base $CI_MERGE_REQUEST_TARGET_BRANCH_NAME HEAD)"
+  info "Using $PERF_BASELINE_COMMIT for performance metric baseline..."
+}
+
 function test_make() {
   run "$MAKE" test_bindist TEST_PREP=YES
   run "$MAKE" V=0 test \

hadrian/src/Settings/Builders/RunTest.hs

@@ -78,6 +78,7 @@ runTestBuilderArgs = builder RunTest ? do
             <*> (maybe False (=="YES") <$> lookupEnv "OS")
     (testEnv, testMetricsFile) <- expr . liftIO $
         (,) <$> lookupEnv "TEST_ENV" <*> lookupEnv "METRICS_FILE"
+    perfBaseline <- expr . liftIO $ lookupEnv "PERF_BASELINE_COMMIT"
 
     threads     <- shakeThreads <$> expr getShakeOptions
     os          <- getTestSetting TestHostOS
@@ -141,6 +142,9 @@ runTestBuilderArgs = builder RunTest ? do
             , arg "--config", arg $ "timeout_prog=" ++ show (top -/- timeoutProg)
             , arg "--config", arg $ "stats_files_dir=" ++ statsFilesDir
             , arg $ "--threads=" ++ show threads
+            , case perfBaseline of
+                Just commit | not (null commit) -> arg ("--perf-baseline=" ++ show commit)
+                _ -> mempty
             , emitWhenSet testEnv $ \env -> arg ("--test-env=" ++ show env)
             , emitWhenSet testMetricsFile $ \file -> arg ("--metrics-file=" ++ file)
             , getTestArgs -- User-provided arguments from command line.

testsuite/mk/test.mk

@@ -232,6 +232,10 @@ ifneq "$(VERBOSE)" ""
 RUNTEST_OPTS += --verbose=$(VERBOSE)
 endif
 
+ifneq "$(PERF_TEST_BASELINE_COMMIT)" ""
+RUNTEST_OPTS += --perf-baseline=$(PERF_TEST_BASELINE_COMMIT)
+endif
+
 ifeq "$(SKIP_PERF_TESTS)" "YES"
 RUNTEST_OPTS += --skip-perf-tests
 endif