.gitlab-ci.yml

@@ -659,22 +659,6 @@ validate-x86_64-linux-deb9-debug:
     when: always
     expire_in: 2 week
 
-# Disabled to alleviate CI load
-.validate-x86_64-linux-deb9-llvm:
-  extends: .build-x86_64-linux-deb9
-  stage: full-build
-  variables:
-    BUILD_FLAVOUR: perf-llvm
-    TEST_ENV: "x86_64-linux-deb9-llvm"
-
-nightly-x86_64-linux-deb9-llvm:
-  <<: *nightly
-  extends: .build-x86_64-linux-deb9
-  stage: full-build
-  variables:
-    BUILD_FLAVOUR: perf-llvm
-    TEST_ENV: "x86_64-linux-deb9-llvm"
-
 validate-x86_64-linux-deb9-integer-simple:
   extends: .build-x86_64-linux-deb9
   stage: full-build
@@ -759,6 +743,23 @@ release-x86_64-linux-deb10-dwarf:
     TEST_ENV: "x86_64-linux-deb10-dwarf"
     BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-deb10-linux-dwarf.tar.xz"
 
+validate-x86_64-linux-deb10-llvm:
+  extends: .build-x86_64-linux-deb10
+  stage: full-build
+  rules:
+    - if: '$CI_MERGE_REQUEST_LABELS =~ /.*LLVM backend.*/'
+  variables:
+    BUILD_FLAVOUR: perf-llvm
+    TEST_ENV: "x86_64-linux-deb10-llvm"
+
+nightly-x86_64-linux-deb10-llvm:
+  <<: *nightly
+  extends: .build-x86_64-linux-deb10
+  stage: full-build
+  variables:
+    BUILD_FLAVOUR: perf-llvm
+    TEST_ENV: "x86_64-linux-deb10-llvm"
+
 #################################
 # x86_64-linux-ubuntu 20.04
 #################################

compiler/GHC/CmmToLlvm/Base.hs

@@ -476,13 +476,15 @@ ghcInternalFunctions :: LlvmM ()
 ghcInternalFunctions = do
     platform <- getPlatform
     let w = llvmWord platform
+        cint = LMInt $ widthInBits $ cIntWidth platform
+    mk "memcmp" cint [i8Ptr, i8Ptr, w]
     mk "memcpy" i8Ptr [i8Ptr, i8Ptr, w]
     mk "memmove" i8Ptr [i8Ptr, i8Ptr, w]
     mk "memset" i8Ptr [i8Ptr, w, w]
     mk "newSpark" w [i8Ptr, i8Ptr]
   where
     mk n ret args = do
-      let n' = llvmDefLabel $ fsLit n
+      let n' = fsLit n
           decl = LlvmFunctionDecl n' ExternallyVisible CC_Ccc ret
                                  FixedArgs (tysToParams args) Nothing
       renderLlvm $ ppLlvmFunctionDecl decl
@@ -516,7 +518,10 @@ getGlobalPtr llvmLbl = do
   let mkGlbVar lbl ty = LMGlobalVar lbl (LMPointer ty) Private Nothing Nothing
   case m_ty of
     -- Directly reference if we have seen it already
-    Just ty -> return $ mkGlbVar (llvmDefLabel llvmLbl) ty Global
+    Just ty -> do
+      if llvmLbl `elem` (map fsLit ["newSpark", "memmove", "memcpy", "memcmp", "memset"])
+        then return $ mkGlbVar (llvmLbl) ty Global
+        else return $ mkGlbVar (llvmDefLabel llvmLbl) ty Global
     -- Otherwise use a forward alias of it
     Nothing -> do
       saveAlias llvmLbl

compiler/ghc.cabal.in

@@ -63,7 +63,7 @@ Library
                    process    >= 1   && < 1.7,
                    bytestring >= 0.9 && < 0.11,
                    binary     == 0.8.*,
-                   time       >= 1.4 && < 1.10,
+                   time       >= 1.4 && < 1.12,
                    containers >= 0.6.2.1 && < 0.7,
                    array      >= 0.1 && < 0.6,
                    filepath   >= 1   && < 1.5,

ghc/ghc-bin.cabal.in

@@ -63,7 +63,7 @@ Executable ghc
             ghci           == @ProjectVersionMunged@,
             haskeline      == 0.8.*,
             exceptions     == 0.10.*,
-            time           >= 1.8 && < 1.10
+            time           >= 1.8 && < 1.12
         CPP-Options: -DHAVE_INTERNAL_INTERPRETER
         Other-Modules:
             GHCi.Leak

hadrian/doc/flavours.md

@@ -99,16 +99,6 @@ when compiling the `compiler` library, and `hsGhc` when compiling/linking the GH
     <td>-O</td>
     <td>-O2</td>
   </tr>
-  <tr>
-    <th>prof</td>
-    <td>-O0<br>-H64m</td>
-    <td>-O0<br>-H64m</td>
-    <td></td>
-    <td>-O</td>
-    <td>-O2</td>
-    <td>-O</td>
-    <td>-O</td>
-  </tr>
   <tr>
     <th>bench</td>
     <td>-O<br>-H64m</td>
@@ -166,13 +156,66 @@ when compiling the `compiler` library, and `hsGhc` when compiling/linking the GH
   </tr>
 </table>
 
-### LLVM variants
+## Flavour transformers
 
-In addition to the above, there are LLVM variants for the flavours `quick`,
-`prof`, `perf` and `bench`, available by appending a `-llvm` suffix (i.e.,
-`quick-llvm` for the LLVM variant of `quick`). These differ only in that there
-is an additional `-fllvm` flag in `hsDefault` when the stage0 compiler is GHC.
-See `src/Settings/Flavours/Llvm.hs` for details.
+Each of the flavours described above is intended as a starting-point for
+configuring your GHC build. In addition, Hadrian supports a number of "flavour
+transformers" which modify the configuration in various ways.
+
+These can be appended to the flavour name passed via the `--flavour`
+command-line flag, separated by the `+` character. For instance,
+
+```
+hadrian --flavour=perf+thread_sanitizer
+```
+
+The supported transformers are listed below:
+
+<table>
+    <tr>
+        <th>Transformer name</th>
+        <th>Effect</th>
+    </tr>
+    <tr>
+        <td><code>werror</code></td>
+        <td>Use the `-Werror` flag for all stage1+ compilation.</td>
+    </tr>
+    <tr>
+        <td><code>debug_info</code></td>
+        <td>Enable production of native debugging information (via GHC/GCC's `-g3`)
+            during stage1+ compilations.</td>
+    </tr>
+    <tr>
+        <td><code>ticky_ghc</code></td>
+        <td>Compile the GHC executable with Ticky-Ticky profiler support.</td>
+    </tr>
+    <tr>
+        <td><code>split_sections</code></td>
+        <td>Enable section splitting for all libraries (except for the GHC
+            library due to the long linking times that this causes).</td>
+    </tr>
+    <tr>
+        <td><code>thread_sanitizer</code></td>
+        <td>Build the runtime system with ThreadSanitizer support</td>
+    </tr>
+    <tr>
+        <td><code>llvm</code></td>
+        <td>Use GHC's LLVM backend (`-fllvm`) for all stage1+ compilation.</td>
+    </tr>
+    <tr>
+        <td><code>profiled_ghc</code></td>
+        <td>Build the GHC executable with cost-centre profiling support.
+            It is that you use this in conjunction with `no_dynamic_ghc` since
+            GHC does not It is support loading of profiled libraries with the
+            dynamically-linker.</td>
+    </tr>
+    <tr>
+        <td><code>no_dynamic_ghc</code></td>
+        <td>Linked GHC against the statically-linked RTS. This causes GHC to
+            default to loading static rather than dynamic library when,
+            e.g., loading libraries during TemplateHaskell evaluations.</td>
+    </tr>
+</table>
 
 ## Ways
 
@@ -184,7 +227,6 @@ information. The following table lists ways that are built in different flavours
         <th rowspan="2">Flavour</th>
         <th colspan="2">Library ways</th>
         <th colspan="2">RTS ways</th>
-        <th colspan="2">Profiled GHC</th>
     </tr>
     <tr>
         <th>stage0</th>
@@ -195,7 +237,7 @@ information. The following table lists ways that are built in different flavours
         <th>stage1+</th>
     </tr>
     <tr>
-    <th>default<br>perf<br>prof<br>devel1<br>devel2<br>perf-llvm<br>prof-llvm</td>
+    <th>default<br>perf<br>prof<br>devel1<br>devel2</td>
     <td>vanilla</td>
     <td>vanilla<br>profiling<br>dynamic</td>
     <td>logging<br>debug<br>threaded<br>threadedDebug<br>threadedLogging
@@ -208,11 +250,9 @@ information. The following table lists ways that are built in different flavours
         <br>debugDynamic<br>threadedDynamic<br>threadedDebugDynamic
         <br>loggingDynamic<br>threadedLoggingDynamic
     </td>
-    <td>Only in<br>prof<br>flavour</td>
-    <td>Only in<br>prof<br>flavour</td>
 </tr>
 <tr>
-    <th>quick<br>quick-llvm<br>quick-validate<br>quick-debug</th>
+    <th>quick<br>quick-validate<br>quick-debug</th>
     <td>vanilla</td>
     <td>vanilla<br>dynamic</td>
     <td>logging<br>debug<br>threaded<br>threadedDebug<br>threadedLogging
@@ -223,8 +263,6 @@ information. The following table lists ways that are built in different flavours
         <br>debugDynamic<br>threadedDynamic<br>threadedDebugDynamic
         <br>loggingDynamic<br>threadedLoggingDynamic
     </td>
-    <td>No</td>
-    <td>No</td>
 </tr>
 <tr>
     <th>quickest<br>bench</th>
@@ -232,7 +270,5 @@ information. The following table lists ways that are built in different flavours
     <td>vanilla</td>
     <td>vanilla<br>threaded</td>
     <td>vanilla<br>threaded</td>
-    <td>No</td>
-    <td>No</td>
 </tr>
 </table>

hadrian/hadrian.cabal

@@ -106,13 +106,10 @@ executable hadrian
                        , Settings.Flavours.Benchmark
                        , Settings.Flavours.Development
                        , Settings.Flavours.GhcInGhci
-                       , Settings.Flavours.Llvm
                        , Settings.Flavours.Performance
-                       , Settings.Flavours.Profiled
                        , Settings.Flavours.Quick
                        , Settings.Flavours.QuickCross
                        , Settings.Flavours.Quickest
-                       , Settings.Flavours.ThreadSanitizer
                        , Settings.Flavours.Validate
                        , Settings.Packages
                        , Settings.Parser

hadrian/src/Flavour.hs

 module Flavour
   ( Flavour (..), werror
   , DocTargets, DocTarget(..)
+  , parseFlavour
     -- * Flavour transformers
+  , flavourTransformers
   , addArgs
   , splitSections, splitSectionsIf
   , enableThreadSanitizer
   , enableDebugInfo, enableTickyGhc
+  , viaLlvmBackend
+  , enableProfiledGhc
+  , disableDynamicGhcPrograms
   ) where
 
 import Expression
 import Data.Set (Set)
+import Data.Map (Map)
+import qualified Data.Map as M
 import Packages
 
+import Text.Parsec.Prim as P
+import Text.Parsec.Combinator as P
+import Text.Parsec.Char as P
+
 -- Please update doc/{flavours.md, user-settings.md} when changing this file.
 -- | 'Flavour' is a collection of build settings that fully define a GHC build.
 -- Note the following type semantics:
@@ -69,6 +80,58 @@ type DocTargets = Set DocTarget
 data DocTarget = Haddocks | SphinxHTML | SphinxPDFs | SphinxMan | SphinxInfo
   deriving (Eq, Ord, Show, Bounded, Enum)
 
+flavourTransformers :: Map String (Flavour -> Flavour)
+flavourTransformers = M.fromList
+    [ "werror" =: werror
+    , "debug_info" =: enableDebugInfo
+    , "ticky_ghc" =: enableTickyGhc
+    , "split_sections" =: splitSections
+    , "thread_sanitizer" =: enableThreadSanitizer
+    , "llvm" =: viaLlvmBackend
+    , "profiled_ghc" =: enableProfiledGhc
+    , "no_dynamic_ghc" =: disableDynamicGhcPrograms
+    ]
+  where (=:) = (,)
+
+type Parser = Parsec String ()
+
+parseFlavour :: [Flavour]  -- ^ base flavours
+             -> Map String (Flavour -> Flavour) -- ^ modifiers
+             -> String
+             -> Either String Flavour
+parseFlavour baseFlavours transformers str =
+    case P.runParser parser () "" str of
+      Left perr -> Left $ unlines $
+                    [ "error parsing flavour specifier: " ++ show perr
+                    , ""
+                    , "known flavours:"
+                    ] ++
+                    [ "  " ++ name f | f <- baseFlavours ] ++
+                    [ ""
+                    , "known flavour transformers:"
+                    ] ++
+                    [ "  " ++ nm | nm <- M.keys transformers ]
+      Right f -> Right f
+  where
+    parser :: Parser Flavour
+    parser = do
+      base <- baseFlavour
+      transs <- P.many flavourTrans
+      return $ foldr ($) base transs
+
+    baseFlavour :: Parser Flavour
+    baseFlavour =
+        P.choice [ f <$ P.try (P.string (name f))
+                 | f <- baseFlavours
+                 ]
+
+    flavourTrans :: Parser (Flavour -> Flavour)
+    flavourTrans = do
+        void $ P.char '+'
+        P.choice [ trans <$ P.try (P.string nm)
+                 | (nm, trans) <- M.toList transformers
+                 ]
+
 -- | Add arguments to the 'args' of a 'Flavour'.
 addArgs :: Args -> Flavour -> Flavour
 addArgs args' fl = fl { args = args fl <> args' }
@@ -96,7 +159,13 @@ enableTickyGhc =
       [ builder (Ghc CompileHs) ? ticky
       , builder (Ghc LinkHs) ? ticky
       ]
-    ticky = arg "-ticky" <> arg "-ticky-allocd"
+    ticky = mconcat
+      [ arg "-ticky"
+      , arg "-ticky-allocd"
+      -- You generally need STG dumps to interpret ticky profiles
+      , arg "-ddump-to-file"
+      , arg "-ddump-stg-final"
+      ]
 
 -- | Transform the input 'Flavour' so as to build with
 --   @-split-sections@ whenever appropriate. You can
@@ -128,3 +197,17 @@ enableThreadSanitizer = addArgs $ mconcat
     , builder (Cabal Flags) ? arg "thread-sanitizer"
     , builder  RunTest ? arg "--config=have_thread_sanitizer=True"
     ]
+
+-- | Use the LLVM backend in stages 1 and later.
+viaLlvmBackend :: Flavour -> Flavour
+viaLlvmBackend = addArgs $ notStage0 ? builder Ghc ? arg "-fllvm"
+
+-- | Build the GHC executable with profiling enabled. It is also recommended
+-- that you use this with @'dynamicGhcPrograms' = False@ since GHC does not
+-- support loading of profiled libraries with the dynamically-linker.
+enableProfiledGhc :: Flavour -> Flavour
+enableProfiledGhc flavour = flavour { ghcProfiled = True }
+
+-- | Disable 'dynamicGhcPrograms'.
+disableDynamicGhcPrograms :: Flavour -> Flavour
+disableDynamicGhcPrograms flavour = flavour { dynamicGhcPrograms = pure False }

hadrian/src/Settings.hs

@@ -15,13 +15,10 @@ import {-# SOURCE #-} Settings.Default
 import Settings.Flavours.Benchmark
 import Settings.Flavours.Development
 import Settings.Flavours.GhcInGhci
-import Settings.Flavours.Llvm
 import Settings.Flavours.Performance
-import Settings.Flavours.Profiled
 import Settings.Flavours.Quick
 import Settings.Flavours.Quickest
 import Settings.Flavours.QuickCross
-import Settings.Flavours.ThreadSanitizer
 import Settings.Flavours.Validate
 
 import Control.Monad.Except
@@ -54,13 +51,11 @@ stagePackages stage = do
 hadrianFlavours :: [Flavour]
 hadrianFlavours =
     [ benchmarkFlavour, defaultFlavour, developmentFlavour Stage1
-    , developmentFlavour Stage2, performanceFlavour, profiledFlavour
+    , developmentFlavour Stage2, performanceFlavour
     , quickFlavour, quickValidateFlavour, quickDebugFlavour
     , quickestFlavour
-    , quickCrossFlavour, benchmarkLlvmFlavour
-    , performanceLlvmFlavour, profiledLlvmFlavour, quickLlvmFlavour
-    , ghcInGhciFlavour, validateFlavour, slowValidateFlavour
-    , threadSanitizerFlavour ]
+    , quickCrossFlavour
+    , ghcInGhciFlavour, validateFlavour, slowValidateFlavour ]
 
 -- | This action looks up a flavour with the name given on the
 --   command line with @--flavour@, defaulting to 'userDefaultFlavour'
@@ -75,11 +70,9 @@ flavour = do
     let flavours = hadrianFlavours ++ userFlavours
         (_settingErrs, tweak) = applySettings kvs
 
-    return $
-      case filter (\fl -> name fl == flavourName) flavours of
-        []  -> error $ "Unknown build flavour: " ++ flavourName
-        [f] -> tweak f
-        _   -> error $ "Multiple build flavours named " ++ flavourName
+    case parseFlavour flavours flavourTransformers flavourName of
+      Left err -> fail err
+      Right f -> return $ tweak f
 
 -- TODO: switch to Set Package as the order of packages should not matter?
 -- Otherwise we have to keep remembering to sort packages from time to time.

hadrian/src/Settings/Flavours/Llvm.hs

-module Settings.Flavours.Llvm (
-  benchmarkLlvmFlavour,
-  performanceLlvmFlavour,
-  profiledLlvmFlavour,
-  quickLlvmFlavour,
-) where
-
-import Expression
-import Flavour
-
-import Settings.Flavours.Benchmark
-import Settings.Flavours.Performance
-import Settings.Flavours.Profiled
-import Settings.Flavours.Quick
-
--- Please update doc/flavours.md when changing this file.
-benchmarkLlvmFlavour, performanceLlvmFlavour, profiledLlvmFlavour, quickLlvmFlavour :: Flavour
-benchmarkLlvmFlavour   = mkLlvmFlavour benchmarkFlavour
-performanceLlvmFlavour = mkLlvmFlavour performanceFlavour
-profiledLlvmFlavour    = mkLlvmFlavour profiledFlavour
-quickLlvmFlavour       = mkLlvmFlavour quickFlavour
-
--- | Turn a flavour into an LLVM flavour
-mkLlvmFlavour :: Flavour -> Flavour
-mkLlvmFlavour flav = flav
-    { name = name flav ++ "-llvm"
-    , args = mconcat [ args flav
-                     , builder Ghc ? arg "-fllvm" ]
-    }

hadrian/src/Settings/Flavours/Profiled.hs

-module Settings.Flavours.Profiled (profiledFlavour) where
-
-import Expression
-import Flavour
-import {-# SOURCE #-} Settings.Default
-
--- Please update doc/flavours.md when changing this file.
-profiledFlavour :: Flavour
-profiledFlavour = defaultFlavour
-    { name        = "prof"
-    , args        = defaultBuilderArgs <> profiledArgs <> defaultPackageArgs
-    , ghcProfiled = True
-    , dynamicGhcPrograms = pure False }
-
-profiledArgs :: Args
-profiledArgs = sourceArgs SourceArgs
-    { hsDefault  = mconcat
-        [ pure ["-O0", "-H64m"]
-        ]
-    , hsLibrary  = notStage0 ? arg "-O"
-    , hsCompiler = mconcat [stage0 ? arg "-O2", notStage0 ? arg "-O"]
-    , hsGhc      = arg "-O" }

hadrian/src/Settings/Flavours/ThreadSanitizer.hs

-module Settings.Flavours.ThreadSanitizer (threadSanitizerFlavour) where
-
-import Flavour
-import Settings.Flavours.Validate
-
-threadSanitizerFlavour :: Flavour
-threadSanitizerFlavour =
-  enableThreadSanitizer (validateFlavour
-    { name = "thread-sanitizer" })

includes/RtsAPI.h

@@ -18,8 +18,6 @@ extern "C" {
 #include "HsFFI.h"
 #include "rts/Time.h"
 #include "rts/Types.h"
-#include "rts/EventLogWriter.h"
-
 
 /*
  * Running the scheduler
@@ -60,6 +58,9 @@ typedef struct CapabilityPublic_ {
     StgRegTable r;
 } CapabilityPublic;
 
+/* N.B. this needs the Capability declaration above. */
+#include "rts/EventLogWriter.h"
+
 /* ----------------------------------------------------------------------------
    RTS configuration settings, for passing to hs_init_ghc()
    ------------------------------------------------------------------------- */

includes/rts/EventLogWriter.h

@@ -68,3 +68,8 @@ bool startEventLogging(const EventLogWriter *writer);
  * Stop event logging and destroy the current EventLogWriter.
  */
 void endEventLogging(void);
+
+/*
+ * Flush the eventlog. cap can be NULL if one is not held.
+ */
+void flushEventLog(Capability **cap);

Cabal @ d30b8f3e

-Subproject commit 5aea8a4b8463e1ae95272e190a1022764164294f
+Subproject commit d30b8f3ec0b0873b9d2eb245afdd53fabacdb884

libraries/base/Debug/Trace.hs

@@ -37,6 +37,7 @@ module Debug.Trace (
         -- $eventlog_tracing
         traceEvent,
         traceEventIO,
+        flushEventLog,
 
         -- * Execution phase markers
         -- $markers
@@ -319,3 +320,11 @@ traceMarkerIO :: String -> IO ()
 traceMarkerIO msg =
   GHC.Foreign.withCString utf8 msg $ \(Ptr p) -> IO $ \s ->
     case traceMarker# p s of s' -> (# s', () #)
+
+-- | Immediately flush the event log, if enabled.
+--
+-- @since 4.15.0.0
+flushEventLog :: IO ()
+flushEventLog = c_flushEventLog nullPtr
+
+foreign import ccall "flushEventLog" c_flushEventLog :: Ptr () -> IO ()

directory @ 0633b48b

-Subproject commit cecf363bc07004ad314e0297ce34ddba05031c0e
+Subproject commit 0633b48b010093f64f98ee494265436e96456aed

libraries/ghc-boot/GHC/Data/ShortText.hs

-{-# LANGUAGE BangPatterns, MagicHash, UnboxedTuples, GeneralizedNewtypeDeriving, DerivingStrategies #-}
+{-# LANGUAGE BangPatterns, MagicHash, UnboxedTuples, GeneralizedNewtypeDeriving, DerivingStrategies, CPP #-}
 {-# OPTIONS_GHC -O2 -funbox-strict-fields #-}
-
+-- gross hack: we manuvered ourselves into a position where we can't boot GHC with a LLVM based GHC anymore.
+-- LLVM based GHC's fail to compile memcmp ffi calls.  These end up as memcmp$def in the llvm ir, however we
+-- don't have any prototypes and subsequently the llvm toolchain chokes on them.  Since 7fdcce6d, we use
+-- ShortText for the package database.  This however introduces this very module; which through inlining ends
+-- up bringing memcmp_ByteArray from bytestring:Data.ByteString.Short.Internal into scope, which results in
+-- the memcmp call we choke on.
+--
+-- The solution thusly is to force late binding via the linker instead of inlining when comping with the
+-- bootstrap compiler.  This will produce a slower (slightly less optimised) stage1 compiler only.
+--
+-- See issue 18857. hsyl20 deserves credit for coming up with the idea for the soltuion.
+--
+-- This can be removed when we exit the boot compiler window. Thus once we drop GHC-9.2 as boot compiler,
+-- we can drop this code as well.
+#if GHC_STAGE < 1
+{-# OPTIONS_GHC -fignore-interface-pragmas #-}
+#endif
 -- |
 -- An Unicode string for internal GHC use. Meant to replace String
 -- in places where being a lazy linked is not very useful and a more

hpc @ 59e6ba02

-Subproject commit fecf0496a65c4be275d613eb0632fecd5a123b69
+Subproject commit 59e6ba02f3fa5c8f4901b4ce21777c4a9beb14b6

time @ c25d6a76

-Subproject commit 5319bed3b14c21de5410ead88ec8aaa838d7339c
+Subproject commit c25d6a76702b454426e149fb590da5cb69f3bd0a

unix @ e0798237

-Subproject commit 8abd63ea234de02d2b3cb08b5098cd06c1a728f6
+Subproject commit e079823775066bcab56b22842be6cce6e060fb9f