.gitlab-ci.yml

@@ -470,6 +470,8 @@ validate-x86_64-darwin:
     CONFIGURE_ARGS: "--with-intree-gmp"
     TEST_ENV: "x86_64-darwin"
     BUILD_FLAVOUR: "validate"
+    # Due to #19025
+    TEST_ARGS: "--ignore-perf-failures"
   after_script:
     - cp -Rf $HOME/.cabal cabal-cache
     - .gitlab/ci.sh clean
@@ -495,6 +497,8 @@ validate-x86_64-darwin:
     LANG: "en_US.UTF-8"
     CONFIGURE_ARGS: --with-intree-gmp
     TEST_ENV: "x86_64-darwin-hadrian"
+    # Due to #19025
+    TEST_ARGS: "--ignore-perf-failures"
     BIN_DIST_NAME: "ghc-x86_64-apple-darwin"
     BUILD_FLAVOUR: "validate"
   script:

.gitlab/ci.sh

@@ -44,6 +44,7 @@ Environment variables affecting both build systems:
 
   CROSS_TARGET      Triple of cross-compilation target.
   VERBOSE           Set to non-empty for verbose build output
+  TEST_ARGS         Arguments passed to runtest.py
   MSYSTEM           (Windows-only) Which platform to build form (MINGW64 or MINGW32).
 
 Environment variables determining build configuration of Make system:
@@ -443,7 +444,8 @@ function test_make() {
   run "$MAKE" test_bindist TEST_PREP=YES
   run "$MAKE" V=0 test \
     THREADS="$cores" \
-    JUNIT_FILE=../../junit.xml
+    JUNIT_FILE=../../junit.xml \
+    EXTRA_RUNTEST_OPTS="$RUNTEST_ARGS"
 }
 
 function build_hadrian() {
@@ -470,7 +472,8 @@ function test_hadrian() {
   run_hadrian \
     test \
     --summary-junit=./junit.xml \
-    --test-compiler="$TOP"/_build/install/bin/ghc
+    --test-compiler="$TOP"/_build/install/bin/ghc \
+    "runtest.opts+=$RUNTEST_ARGS"
 }
 
 function cabal_test() {

hadrian/doc/user-settings.md

@@ -411,6 +411,9 @@ the right names for them:
   * `cabal.configure.opts` refers to Cabal configure command line. Note that
     package flags can be given by adding `--flags=...` arguments.
 
+  * `runtest.opts` defines extra arguments passed to `runtest.py` when
+    invoked via the `hadrian test` target.
+
 - using a wildcard (`*`) ranges over all possible values for a given "slot";
 - `=` entirely overrides the arguments for a given builder in a given context,
   with the value specified on the right hand side of `=`, while `+=` merely

hadrian/src/Settings.hs

+{-# LANGUAGE TupleSections #-}
+
 module Settings (
     getArgs, getLibraryWays, getRtsWays, flavour, knownPackages,
     findPackageByName, unsafeFindPackageByName, unsafeFindPackageByPath,
@@ -68,7 +70,10 @@ flavour = do
     flavourName <- fromMaybe userDefaultFlavour <$> cmdFlavour
     kvs <- userSetting ([] :: [KeyVal])
     let flavours = hadrianFlavours ++ userFlavours
-        (_settingErrs, tweak) = applySettings kvs
+        (settingErrs, tweak) = applySettings kvs
+
+    when (not $ null settingErrs) $ fail
+      $ "failed to apply key-value settings:" ++ unlines (map (" - " ++) settingErrs)
 
     case parseFlavour flavours flavourTransformers flavourName of
       Left err -> fail err
@@ -190,7 +195,7 @@ applySettings kvs = case partitionEithers (map applySetting kvs) of
 applySetting :: KeyVal -> Either SettingError (Flavour -> Flavour)
 applySetting (KeyVal ks op v) = case runSettingsM ks builderPredicate of
   Left err -> throwError $
-      "error while setting " ++ show ks ++ ": " ++ err
+      "error while setting `" ++ intercalate "`." ks ++ ": " ++ err
   Right pred -> Right $ \flav -> flav
     { args = update (args flav) pred }
 
@@ -229,10 +234,12 @@ builderPredicate :: SettingsM Predicate
 builderPredicate = builderSetting <&> (\(wstg, wpkg, builderMode) ->
   wildcard (pure True) stage wstg <&&>
   wildcard (pure True) package wpkg <&&>
-  (case builderMode of
-     BM_Ghc ghcMode -> wildcard (builder Ghc) (builder . Ghc) ghcMode
-     BM_Cc  ccMode  -> wildcard (builder Cc) (builder . Cc) ccMode
-     BM_CabalConfigure -> builder (Cabal Setup) )
+    (case builderMode of
+       BM_Ghc ghcMode -> wildcard (builder Ghc) (builder . Ghc) ghcMode
+       BM_Cc  ccMode  -> wildcard (builder Cc) (builder . Cc) ccMode
+       BM_CabalConfigure -> builder (Cabal Setup)
+       BM_RunTest     -> builder RunTest
+    )
   )
 
   where (<&&>) = liftA2 (&&)
@@ -241,6 +248,7 @@ builderPredicate = builderSetting <&> (\(wstg, wpkg, builderMode) ->
 data BuilderMode = BM_Ghc (Wildcard GhcMode)
                  | BM_Cc  (Wildcard CcMode)
                  | BM_CabalConfigure
+                 | BM_RunTest
 
 -- | Interpretation-agnostic description of the builder settings
 --   supported by Hadrian.
@@ -250,6 +258,7 @@ data BuilderMode = BM_Ghc (Wildcard GhcMode)
 --   > (<stage> or *).(<package name> or *).ghc.(<ghc mode> or *).opts
 --   > (<stage> or *).(<package name> or *).cc.(<cc mode> or *).opts
 --   > (<stage> or *).(<package name> or *).cabal.configure.opts
+--   > runtest.opts
 --
 --   where:
 --     - @<stage>@ is one of @stage0@, @stage1@, @stage2@ or @stage3@;
@@ -269,15 +278,19 @@ data BuilderMode = BM_Ghc (Wildcard GhcMode)
 --       appropriate spot.
 builderSetting :: Match f
                => f (Wildcard Stage, Wildcard Package, BuilderMode)
-builderSetting = (,,)
-             <$> wild stages
-             <*> wild pkgs
-             <*> matchOneOf
-                   [ str "ghc" *> fmap BM_Ghc (wild ghcBuilder) <* str "opts"
-                   , str "cc" *> fmap BM_Cc (wild ccBuilder) <* str "opts"
-                   , BM_CabalConfigure <$ str "cabal" <* str "configure" <* str "opts"
-                   ]
-
+builderSetting =
+    matchOneOf
+    [ (,,)
+       <$> wild stages
+       <*> wild pkgs
+       <*> matchOneOf
+             [ str "ghc" *> fmap BM_Ghc (wild ghcBuilder) <* str "opts"
+             , str "cc" *> fmap BM_Cc (wild ccBuilder) <* str "opts"
+             , BM_CabalConfigure <$ str "cabal" <* str "configure" <* str "opts"
+             ]
+    , (Wildcard, Wildcard, BM_RunTest)
+      <$ str "runtest" <* str "opts"
+    ]
   where ghcBuilder =
           [ ("c", CompileCWithGhc)
           , ("cpp", CompileCppWithGhc)

testsuite/driver/runtests.py

@@ -76,6 +76,7 @@ parser.add_argument("--perf-baseline", type=GitRef, metavar='COMMIT', help="Base
 parser.add_argument("--test-package-db", dest="test_package_db", action="append", help="Package db providing optional packages used by the testsuite.")
 perf_group.add_argument("--skip-perf-tests", action="store_true", help="skip performance tests")
 perf_group.add_argument("--only-perf-tests", action="store_true", help="Only do performance tests")
+perf_group.add_argument("--ignore-perf-failures", action="store_true", help="Don't fail due to out-of-tolerance perf tests")
 
 args = parser.parse_args()
 
@@ -151,6 +152,7 @@ if args.verbose is not None:
 forceSkipPerfTests = not hasMetricsFile and not inside_git_repo()
 config.skip_perf_tests = args.skip_perf_tests or forceSkipPerfTests
 config.only_perf_tests = args.only_perf_tests
+config.ignore_perf_failures = args.ignore_perf_failures
 
 if args.test_env:
     config.test_env = args.test_env

testsuite/driver/testglobals.py

@@ -31,6 +31,9 @@ class TestConfig:
         self.run_only_some_tests = False
         self.only = set()
 
+        # Don't fail on out-of-tolerance stat failures
+        self.ignore_perf_failures = False
+
         # Accept new output which differs from the sample?
         self.accept = False
         self.accept_platform = False

testsuite/driver/testlib.py

@@ -1539,7 +1539,7 @@ def check_stats(name: TestName,
             # If any metric fails then the test fails.
             # Note, the remaining metrics are still run so that
             # a complete list of changes can be presented to the user.
-            if metric_result.passFail == 'fail':
+            if metric_result.passFail == 'fail' and not config.ignore_perf_failures:
                 result = metric_result
 
     return result

testsuite/tests/perf/compiler/all.T

@@ -2,10 +2,6 @@
 # See testsuite/driver/testlib.py.
 setTestOpts(no_lint)
 
-# The allocations of some tests seem to be remarkably sensitive to which Darwin
-# runner they run on (#19025). Disable them.
-fragile_on_darwin = when(opsys('darwin'), skip)
-
 test('T1969',
      [# expect_broken(12437),
       collect_compiler_residency(20),
@@ -116,8 +112,7 @@ test('T5837',
 
 test('T6048',
      [ only_ways(['optasm']),
-       fragile_on_darwin,
-       collect_compiler_stats('bytes allocated',2)
+      collect_compiler_stats('bytes allocated',2)
       ],
       compile,[''])
 
@@ -204,7 +199,6 @@ test('T12227',
 
 test('T12425',
      [ only_ways(['optasm']),
-       fragile_on_darwin,
        collect_compiler_stats('bytes allocated', 1)
      ],
      compile,
@@ -212,7 +206,6 @@ test('T12425',
 
 test('T12234',
      [ only_ways(['optasm']),
-       fragile_on_darwin,
        collect_compiler_stats('bytes allocated', 2),
      ],
      compile,
@@ -228,7 +221,6 @@ test('T12545',
 
 test('T13035',
      [ only_ways(['normal']),
-       fragile_on_darwin,
        collect_compiler_stats('bytes allocated', 1),
      ],
      compile,
@@ -254,7 +246,6 @@ test('T12707',
 # to avoid spurious errors.
 test('T12150',
      [ only_ways(['optasm']),
-       fragile_on_darwin,
        collect_compiler_stats('bytes allocated', 2)
      ],
     compile,
@@ -416,7 +407,6 @@ test ('T18140',
       ['-v0 -O'])
 test('T10421',
     [ only_ways(['normal']),
-       fragile_on_darwin,
       collect_compiler_stats('bytes allocated', 1)
     ],
     multimod_compile,
@@ -445,9 +435,7 @@ test ('T18223',
       compile,
       ['-v0 -O'])
 test ('T18923',
-      [ collect_compiler_stats('bytes allocated',2),
-        fragile_on_darwin,
-      ],
+      [ collect_compiler_stats('bytes allocated',2) ],
       compile,
       ['-v0 -O'])
 test ('T16577',