ci: Support splitting build across multiple jobs

This utilizes GitLab CI's `parallel`[1] field to divide the build into
several jobs, each handling a subset of the built packages. The job
count is a bit of a trade-off between build re-use and parallelism. I'm
trying 5 for now.

[1] https://docs.gitlab.com/ee/ci/yaml/#parallel

.gitlab-ci.yml

@@ -14,6 +14,7 @@
 #
 
 stages:
+  - prebuild
   - test
   - update-repo
   - deploy
@@ -72,19 +73,11 @@ build-8.8:
     - branches
     - merge_requests
 
-.build:
-  stage: test
-
+.base:
   tags:
     - x86_64-linux
-
   image: nixos/nix
 
-  cache:
-    key: build-HEAD
-    paths:
-      - store.nar
-
   before_script:
     - |
       if [ -e store.nar ]; then
@@ -99,29 +92,52 @@ build-8.8:
     - echo "Bindist tarball is $GHC_TARBALL"
     - |
       nix build \
-      -f https://github.com/mpickering/ghc-artefact-nix/archive/master.tar.gz \
-      --argstr url $GHC_TARBALL \
-      --out-link ghc \
-      ghcHEAD
+        -f https://github.com/mpickering/ghc-artefact-nix/archive/master.tar.gz \
+        --argstr url $GHC_TARBALL \
+        --out-link ghc \
+        ghcHEAD
+    - export GHC=`pwd`/ghc/bin/ghc
+    - rm -Rf $HOME/.cabal/packages/local
     - export GHC=`pwd`/ghc/bin/ghc
-    - rm -Rf $HOME/.cabal/packages/local ci/run
       # Build CI executable
     - |
       nix-build ./ci -j$CPUS --no-build-output
       nix-store --export \
-        $(nix-store -qR --include-outputs \
-          $(nix-instantiate --quiet ./ci)) \
-      > store.nar
+        $(nix-store -qR $(nix-instantiate --quiet ./ci)) \
+        > store.nar
       # Test it
-    - nix run -f ./ci -c run-ci
+    - nix run -f ./ci -c run-ci $ARGS
+
+prebuild:
+  stage: prebuild
+  extends: .base
+  only:
+    - branches
+    - merge_requests
+  variables:
+    ARGS: "--only=acme-box"
+  artifacts:
+    paths:
+      - store.nar
+      - ci/run
+    expire_in: 1 day
+  cache:
+    key: build-HEAD
+    paths:
+      - store.nar
 
+.build:
+  stage: test
+  extends: .base
+  dependencies:
+    - prebuild
+  parallel: 5
   after_script:
     - ls -lh
     - |
       nix run -f ./ci -c \
       tar -cJf results.tar.xz -C ci/run \
       results.json logs
-
   artifacts:
     when: always
     paths:

ci/TestPatches.hs

@@ -53,6 +53,12 @@ newtype BrokenPackages = BrokenPackages { getBrokenPackageNames :: S.Set PkgName
 failureExpected :: BrokenPackages -> PkgName -> Bool
 failureExpected (BrokenPackages pkgs) name = name `S.member` pkgs
 
+-- | To facilitate splitting the builds across multiple GitLab CI jobs we
+-- support *build partitioning*, where the tested packages are split evenly
+-- into @n@ partitions. A 'BuildPartition' identifies one set in such a
+-- partitioning.
+data BuildPartition = BuildPartition { bpIndex, bpCount :: !Int }
+
 data Config = Config { configPatchDir :: FilePath
                      , configCompiler :: FilePath
                      , configGhcOptions :: [String]
@@ -62,6 +68,7 @@ data Config = Config { configPatchDir :: FilePath
                      , configExtraCabalFragments :: [FilePath]
                      , configExtraPackages :: [(Cabal.PackageName, Version)]
                      , configExpectedBrokenPkgs :: BrokenPackages
+                     , configBuildPartition :: Maybe BuildPartition
                      }
 
 cabalOptions :: Config -> [String]
@@ -82,6 +89,7 @@ config =
     <*> extraCabalFragments
     <*> extraPackages
     <*> expectedBrokenPkgs
+    <*> optional buildPartition
   where
     patchDir = option str (short 'p' <> long "patches" <> help "patch directory" <> value "./patches")
     compiler = option str (short 'w' <> long "with-compiler" <> help "path of compiler")
@@ -98,6 +106,13 @@ config =
       $ option
           (fmap toPkgName pkgName)
           (short 'b' <> long "expect-broken" <> metavar "PKGNAME" <> help "expect the given package to fail to build")
+    buildPartition =
+      f <$> option auto (long "partition" <> metavar "N" <> help "the build partition index")
+        <*> option auto (long "partition-count" <> metavar "N" <> help "the number of build partitions")
+      where
+        f i n
+          | i < n = BuildPartition i n
+          | otherwise = error "partition index must be less than partition count"
 
     pkgVer :: ReadM (Cabal.PackageName, Version)
     pkgVer = str >>= parse . T.pack
@@ -115,11 +130,18 @@ config =
     pkgName :: ReadM Cabal.PackageName
     pkgName = str >>= maybe (fail "invalid package name") pure . simpleParse
 
+takeBuildPartition :: BuildPartition -> [a] -> [a]
+takeBuildPartition (BuildPartition i n) xs =
+    take partSize $ drop (i * partSize) xs
+  where
+    partSize = length xs `div` n + 1
+
 testPatches :: Config -> IO ()
 testPatches cfg = do
   setup cfg
   packages <- findPatchedPackages (configPatchDir cfg)
-  packages <- return (packages ++ configExtraPackages cfg)
+  packages <- return $ maybe id takeBuildPartition (configBuildPartition cfg)
+                     $ (packages ++ configExtraPackages cfg)
   let packages' :: S.Set (Cabal.PackageName, Version)
       packages'
         | Just only <- configOnlyPackages cfg

run-ci

@@ -26,6 +26,10 @@ if [ -n "$EXTRA_HC_OPTS" ]; then
   EXTRA_OPTS="$EXTRA_OPTS --ghc-option=\"$EXTRA_HC_OPTS\""
 fi
 
+if [ -n "$CI_NODE_INDEX" ]; then
+  EXTRA_OPTS="$EXTRA_OPTS --partition=$[$CI_NODE_INDEX-1] --partition-count=$CI_NODE_TOTAL"
+fi
+
 mkdir -p run
 
 echo "" > run/deps.cabal.project