From 1f221ba8b87935595b87d39e2d76f752835f9c56 Mon Sep 17 00:00:00 2001
From: Ben Gamari <ben@smart-cactus.org>
Date: Tue, 5 Nov 2019 13:42:52 -0500
Subject: [PATCH] ci: Support splitting build across multiple jobs
This utilizes GitLab CI's `parallel`[1] field to divide the build into
several jobs, each handling a subset of the built packages. The job
count is a bit of a trade-off between build re-use and parallelism. I'm
trying 5 for now.
[1] https://docs.gitlab.com/ee/ci/yaml/#parallel
---
.gitlab-ci.yml | 54 ++++++++++++++++++++++++++++++-----------------
ci/TestPatches.hs | 24 ++++++++++++++++++++-
run-ci | 4 ++++
3 files changed, 62 insertions(+), 20 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8ea85a23..04f05df4 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,6 +14,7 @@
#
stages:
+ - prebuild
- test
- update-repo
- deploy
@@ -72,19 +73,11 @@ build-8.8:
- branches
- merge_requests
-.build:
- stage: test
-
+.base:
tags:
- x86_64-linux
-
image: nixos/nix
- cache:
- key: build-HEAD
- paths:
- - store.nar
-
before_script:
- |
if [ -e store.nar ]; then
@@ -99,29 +92,52 @@ build-8.8:
- echo "Bindist tarball is $GHC_TARBALL"
- |
nix build \
- -f https://github.com/mpickering/ghc-artefact-nix/archive/master.tar.gz \
- --argstr url $GHC_TARBALL \
- --out-link ghc \
- ghcHEAD
+ -f https://github.com/mpickering/ghc-artefact-nix/archive/master.tar.gz \
+ --argstr url $GHC_TARBALL \
+ --out-link ghc \
+ ghcHEAD
+ - export GHC=`pwd`/ghc/bin/ghc
+ - rm -Rf $HOME/.cabal/packages/local
- export GHC=`pwd`/ghc/bin/ghc
- - rm -Rf $HOME/.cabal/packages/local ci/run
# Build CI executable
- |
nix-build ./ci -j$CPUS --no-build-output
nix-store --export \
- $(nix-store -qR --include-outputs \
- $(nix-instantiate --quiet ./ci)) \
- > store.nar
+ $(nix-store -qR $(nix-instantiate --quiet ./ci)) \
+ > store.nar
# Test it
- - nix run -f ./ci -c run-ci
+ - nix run -f ./ci -c run-ci $ARGS
+
+prebuild:
+ stage: prebuild
+ extends: .base
+ only:
+ - branches
+ - merge_requests
+ variables:
+ ARGS: "--only=acme-box"
+ artifacts:
+ paths:
+ - store.nar
+ - ci/run
+ expire_in: 1 day
+ cache:
+ key: build-HEAD
+ paths:
+ - store.nar
+.build:
+ stage: test
+ extends: .base
+ dependencies:
+ - prebuild
+ parallel: 5
after_script:
- ls -lh
- |
nix run -f ./ci -c \
tar -cJf results.tar.xz -C ci/run \
results.json logs
-
artifacts:
when: always
paths:
diff --git a/ci/TestPatches.hs b/ci/TestPatches.hs
index e50fb3bb..fbfbfb69 100644
--- a/ci/TestPatches.hs
+++ b/ci/TestPatches.hs
@@ -53,6 +53,12 @@ newtype BrokenPackages = BrokenPackages { getBrokenPackageNames :: S.Set PkgName
failureExpected :: BrokenPackages -> PkgName -> Bool
failureExpected (BrokenPackages pkgs) name = name `S.member` pkgs
+-- | To facilitate splitting the builds across multiple GitLab CI jobs we
+-- support *build partitioning*, where the tested packages are split evenly
+-- into @n@ partitions. A 'BuildPartition' identifies one set in such a
+-- partitioning.
+data BuildPartition = BuildPartition { bpIndex, bpCount :: !Int }
+
data Config = Config { configPatchDir :: FilePath
, configCompiler :: FilePath
, configGhcOptions :: [String]
@@ -62,6 +68,7 @@ data Config = Config { configPatchDir :: FilePath
, configExtraCabalFragments :: [FilePath]
, configExtraPackages :: [(Cabal.PackageName, Version)]
, configExpectedBrokenPkgs :: BrokenPackages
+ , configBuildPartition :: Maybe BuildPartition
}
cabalOptions :: Config -> [String]
@@ -82,6 +89,7 @@ config =
<*> extraCabalFragments
<*> extraPackages
<*> expectedBrokenPkgs
+ <*> optional buildPartition
where
patchDir = option str (short 'p' <> long "patches" <> help "patch directory" <> value "./patches")
compiler = option str (short 'w' <> long "with-compiler" <> help "path of compiler")
@@ -98,6 +106,13 @@ config =
$ option
(fmap toPkgName pkgName)
(short 'b' <> long "expect-broken" <> metavar "PKGNAME" <> help "expect the given package to fail to build")
+ buildPartition =
+ f <$> option auto (long "partition" <> metavar "N" <> help "the build partition index")
+ <*> option auto (long "partition-count" <> metavar "N" <> help "the number of build partitions")
+ where
+ f i n
+ | i < n = BuildPartition i n
+ | otherwise = error "partition index must be less than partition count"
pkgVer :: ReadM (Cabal.PackageName, Version)
pkgVer = str >>= parse . T.pack
@@ -115,11 +130,18 @@ config =
pkgName :: ReadM Cabal.PackageName
pkgName = str >>= maybe (fail "invalid package name") pure . simpleParse
+takeBuildPartition :: BuildPartition -> [a] -> [a]
+takeBuildPartition (BuildPartition i n) xs =
+ take partSize $ drop (i * partSize) xs
+ where
+ partSize = length xs `div` n + 1
+
testPatches :: Config -> IO ()
testPatches cfg = do
setup cfg
packages <- findPatchedPackages (configPatchDir cfg)
- packages <- return (packages ++ configExtraPackages cfg)
+ packages <- return $ maybe id takeBuildPartition (configBuildPartition cfg)
+ $ (packages ++ configExtraPackages cfg)
let packages' :: S.Set (Cabal.PackageName, Version)
packages'
| Just only <- configOnlyPackages cfg
diff --git a/run-ci b/run-ci
index 0a02add9..6936d055 100755
--- a/run-ci
+++ b/run-ci
@@ -26,6 +26,10 @@ if [ -n "$EXTRA_HC_OPTS" ]; then
EXTRA_OPTS="$EXTRA_OPTS --ghc-option=\"$EXTRA_HC_OPTS\""
fi
+if [ -n "$CI_NODE_INDEX" ]; then
+ EXTRA_OPTS="$EXTRA_OPTS --partition=$[$CI_NODE_INDEX-1] --partition-count=$CI_NODE_TOTAL"
+fi
+
mkdir -p run
echo "" > run/deps.cabal.project
--
GitLab