Commit 238a969e authored by Ben Gamari's avatar Ben Gamari 🐢

Add GitLab CI

parent 05296be9
Pipeline #8053 passed with stage
in 32 minutes and 10 seconds
# GHC/head.hackage CI support
# ===========================
#
# This is the GitLab CI automation that drives GHC's head.hackage testing.
# The goal is to be able to test GHC by building a (small) subset of Hackage.
# Moreover, we want to be able to collect logs of failed builds as well as
# performance metrics from builds that succeed.
#
# To accomplish this we use head.hackage's native Nix support and the
# ghc-artefact-nix expression to make GHC binary distributions usable from
# within Nix. These components are tied together by ./scripts/build-all.nix,
# which contains the list of packages which we build as well as some simple
# configuration to minimize the cost of the builds.
#
variables:
# Commit of ghc/ci-images repository from which to pull Docker images
DOCKER_REV: 6d19c3adc1f5c28c82aed8c5b1ac40931ac60f3f
# A default for testing
GHC_TARBALL: "https://gitlab.haskell.org/ghc/ghc/-/jobs/111365/artifacts/raw/ghc-x86_64-fedora27-linux.tar.xz"
# Project ID of ghc/ghc
GHC_PROJECT_ID: "1"
# ACCESS_TOKEN provided via protected environment variable
build:
tags:
- x86_64-linux
- head.hackage
image: nixos/nix
cache:
key: build-all
paths:
- store.nar
before_script:
- |
if [ -e store.nar ]; then
echo "Extracting cached Nix store..."
nix-store --import -vv < store.nar || echo "invalid cache"
else
echo "No cache found"
fi
script:
- |
if [ -n "$GHC_PIPELINE_ID" ]; then
job_name="validate-x86_64-linux-fedora27"
job_id=$(nix run -f scripts/build-all.nix find-job \
--arg bindistTarball $GHC_TARBALL \
-c find-job.sh $GHC_PROJECT_ID $GHC_PIPELINE_ID $job_name)
echo "Pulling ${job_name} binary distribution from Pipeline $GHC_PIPELINE_ID (job $job_id)..."
GHC_TARBALL="https://gitlab.haskell.org/ghc/ghc/-/jobs/$job_id/artifacts/raw/ghc-x86_64-fedora27-linux.tar.xz"
fi
- echo "Bindist tarball is $GHC_TARBALL"
- nix-build scripts/build-all.nix -j$CPUS
--no-build-output
-A buildDepends
--arg bindistTarball $GHC_TARBALL
- nix-store --export $(nix-store -qR --include-outputs $(nix-instantiate --quiet scripts/build-all.nix --arg bindistTarball $GHC_TARBALL -A buildDepends -A ghc)) > store.nar
- ret=0
- nix-build scripts/build-all.nix
-j$CPUS --no-build-output
-A testedPackages
--keep-going
--arg bindistTarball $GHC_TARBALL
|| (echo "Build failed!"; ret=1)
- scripts/summarize.py || echo "summarize script failed"
- exit $ret
after_script:
- nix run -f '<nixpkgs>' gnutar -c tar -zcf logs.tar.gz logs
# The following throws a fontconfig error but it appears to be innocuous.
- nix run -f '<nixpkgs>' graphviz -c dot -O -T svg summary.dot
- ls -lh
artifacts:
when: always
paths:
- logs.tar.gz
- summary.json
- summary.dot
- summary.dot.svg
......@@ -140,6 +140,34 @@ build `head.hackage` packages:
```
$ nix build -f ./. --arg ghc "(import ghc-from-source.nix {ghc-path=$GHC_TREE;})"
```
### GitLab CI
[GHC's GitLab instance](https://gitlab.haskell.org/ghc/head.hackage) uses
GitLab CI and `nix` to build a subset of the head.hackage package set using GHC
snapshots.
To run a similar build locally simply download a binary distribution from a
`x86_64-fedora27-linux` CI job and run:
```
$ nix build -f scripts/build-all.nix testedPackages \
--arg bindistTarball ./ghc-x86_64-fedora27-linux.tar.xz
```
This will build the set of packages defined by the `testedPackages` list in
`scripts/build-all.nix`.
After building `testedPackages` (allowing for failures) the CI job runs
`scripts/summarize.py`, which produces a few artifacts:
* a JSON summary, which includes the full dependency graph as well as which
package builds failed
* a DOT graph showing the package depedencies and their build success
* a directory of build logs
You can run `scripts/summarize.py` locally with
```
$ export GHC_TARBALL=./ghc-x86_64-fedora27-linux.tar.xz
$ python3 scripts/summarize.py
```
### Travis CI
......
......@@ -8,12 +8,12 @@
# nix build -f --arg ghc "(import build.nix {ghc-path=$GHC_TREE;})"
#
let
rev = "f2632f5c60f574d787fc5490efb3f43f9e6209b7";
rev = "e0818a15305561ea1ebe36203d914465fd71453d";
baseNixpkgs =
builtins.fetchTarball {
url = "https://github.com/NixOS/nixpkgs/archive/${rev}.tar.gz";
sha256 = "1c36p7w36i38gng3yp1nd5vz0p2dwrax5szjkvnmdxfklggs7knf";
};
fetchTarball {
url = "https://github.com/NixOS/nixpkgs/archive/${rev}.tar.gz";
sha256 = "017j741anrl66nndfmblp6xc1lz7783dh8lf19srmxnk5xk4wn7l";
};
in
# ghc: path to a GHC source tree
......@@ -40,12 +40,12 @@ let
haskellOverrides
(self.lib.composeExtensions patchesOverrides jailbreakOverrides);
baseHaskellPackages = self.callPackage "${baseNixpkgs}/pkgs/development/haskell-modules" {
baseHaskellPackages = self.callPackage "${baseNixpkgs}/pkgs/development/haskell-modules" rec {
haskellLib = import "${baseNixpkgs}/pkgs/development/haskell-modules/lib.nix" {
inherit (self) lib;
pkgs = self;
};
buildHaskellPackages = haskellPackages; #self.buildPackages.haskell.packages.ghc843;
buildHaskellPackages = self.buildPackages.haskell.packages.ghc865;
ghc = ghcHEAD;
compilerConfig = self1: super1: {
# Packages included in GHC's global package database
......@@ -81,10 +81,9 @@ let
doctest = haskellPackages.callHackage "doctest" "0.16.0" {};
http-api-data = haskellPackages.callPackage ./http-api-data.nix {};
tagged = self1.callHackage "tagged" "0.8.6" {};
jailbreak-cabal = self.haskell.packages.ghc802.jailbreak-cabal;
cabal2nix = self.haskell.packages.ghc843.cabal2nix;
jailbreak-cabal = buildHaskellPackages.jailbreak-cabal;
cabal2nix = buildHaskellPackages.cabal2nix;
};
};
in baseHaskellPackages.extend overrides;
......
# GHC/head.hackage CI support
#
# See .gitlab-ci.yml in the root of this repository.
# Expects to be run on x86_64.
# bindistTarball should be a fedora27 tarball.
{ bindistTarball }:
let
# GHC from the given bindist.
ghc =
let
commit = "633c6d7553d0f2c91245bcf47ae539cfdb7aaa13";
src = fetchTarball {
url = "https://github.com/bgamari/ghc-artefact-nix/archive/${commit}.tar.gz";
sha256 = "0yl7dk8drb92ipgph1mxx8my4gy9py27m749zw6ah6np4gvdp9sx";
};
in nixpkgs.callPackage "${src}/artifact.nix" {} {
ncursesVersion = "6";
bindistTarball =
if builtins.typeOf bindistTarball == "path"
then bindistTarball
else builtins.fetchurl bindistTarball;
};
# Build configuration.
nixpkgs = import ../. {
ghc = self: ghc;
haskellOverrides = self: super: rec {
mkDerivation = drv: super.mkDerivation (drv // {
doHaddock = false;
enableLibraryProfiling = false;
enableExecutableProfiling = false;
hyperlinkSource = false;
configureFlags = (drv.configureFlags or []) ++ [
"--ghc-options=-ddump-timings"
];
});
};
};
# Build dependencies worth caching.
buildDepends = with nixpkgs; [
buildPackages.haskellPackages.jailbreak-cabal
buildPackages.haskellPackages.cabal2nix
];
# Get build-time dependencies of a derivation.
# See https://github.com/NixOS/nix/issues/1245#issuecomment-401642781
getDepends = pkg: let
drv = builtins.readFile pkg.drvPath;
storeDirRe = lib.replaceStrings [ "." ] [ "\\." ] builtins.storeDir;
storeBaseRe = "[0-9a-df-np-sv-z]{32}-[+_?=a-zA-Z0-9-][+_?=.a-zA-Z0-9-]*";
re = "(${storeDirRe}/${storeBaseRe}\\.drv)";
inputs = lib.concatLists (lib.filter lib.isList (builtins.split re drv));
in map import inputs;
# The packages which we are here to test
testedPackages = with nixpkgs.haskell.lib; with nixpkgs.haskellPackages; {
inherit lens aeson criterion scotty;
# servant: Don't distribute Sphinx documentation
servant = overrideCabal servant { postInstall = ""; };
# singletons: Disable testsuite since it often breaks due to spurious
# TH-pretty-printing changes
singletons = dontCheck singletons;
};
inherit (nixpkgs) lib;
transHaskellDeps = drv:
let
haskellDeps = drv.passthru.getBuildInputs.haskellBuildInputs or [];
in [{
haskellDeps = map (dep: dep.name) haskellDeps;
drvName = drv.name;
drvPath = drv.drvPath;
out = drv.outPath;
name = drv.passthru.pname;
version = drv.passthru.version;
}] ++ lib.concatMap transHaskellDeps haskellDeps;
summary = {
pkgs = lib.concatMap transHaskellDeps (lib.attrValues testedPackages);
roots = map (drv: drv.name) (lib.attrValues testedPackages);
};
# Find Job ID of the given job name in the given pipeline
find-job = nixpkgs.writeScriptBin "find-job.sh" ''
set -e
project_id=$1
pipeline_id=$2
job_name=$3
# Access token is a protected environment variable in the head.hackage project and
# is necessary for this query to succeed. Sadly job tokens only seem to
# give us access to the project being built.
${nixpkgs.curl}/bin/curl \
--silent --show-error \
-H "Private-Token: $ACCESS_TOKEN" \
"https://gitlab.haskell.org/api/v4/projects/$project_id/pipelines/$pipeline_id/jobs?scope[]=success" \
> resp.json
job_id=$(${nixpkgs.jq}/bin/jq ". | map(select(.name == \"$job_name\")) | .[0].id" < resp.json)
if [ "$job_id" = "null" ]; then
echo "Error finding job $job_name for $pipeline_id in project $project_id:" >&2
cat resp.json >&2
rm resp.json
exit 1
else
rm resp.json
echo -n "$job_id"
fi
'';
in {
inherit nixpkgs ghc testedPackages buildDepends;
inherit (nixpkgs) haskellPackages lib;
testedPackageNames = nixpkgs.lib.attrNames testedPackages;
inherit summary find-job;
}
#!/usr/bin/env nix-shell
#!nix-shell -p python3 -i python3
import os
from pathlib import Path
import subprocess
import json
def read_summary():
summary = subprocess.check_output(
['nix', 'eval', '--json',
'--arg', 'bindistTarball', os.environ['GHC_TARBALL'],
'-f', 'scripts/build-all.nix',
'summary'],
encoding = 'UTF-8')
summary = json.loads(summary)
for pkg in summary['pkgs']:
pkg['failed'] = not Path(pkg['out']).exists()
del pkg['out']
return summary
def export_logs(summary):
logs = Path('logs')
logs.mkdir(exist_ok=True)
# Deduplicate
pkgs = { pkg['drvPath']: pkg for pkg in summary['pkgs'] }
for pkg in pkgs.values():
print(f'Collecting log for {pkg["drvPath"]}...')
p = subprocess.run(['nix', 'log', pkg['drvPath']],
stdout=open(logs / f'{pkg["drvName"]}.log', 'wb'))
if p.returncode != 0:
print(f'Error exporting log for {pkg["drvPath"]}')
def export_dot(summary):
pkgsDict = { pkg['drvName']: pkg for pkg in summary['pkgs'] }
edges = {
(pkg['drvName'], dep)
for pkg in summary['pkgs']
for dep in pkg['haskellDeps']
}
s = 'digraph {\n'
s += ' {overlap=prism};\n'
s += ' subgraph cluster_roots {\n'
s += ' {fillcolor=blue style=filled};\n'
for pkg in summary['roots']:
s += f' "{pkg}";\n'
s += ' }\n'
s += '\n'
for pkg, dep in edges:
s += f' "{pkg}" -> "{dep}";\n'
for pkg in summary['pkgs']:
if pkg['failed']:
s += f' "{pkg["drvName"]}" [ color=indianred style=filled ];\n'
s += '}'
return s
def show_failures(summary, log_excerpt=100):
failed = [pkg
for pkg in summary['pkgs']
if pkg['failed']]
if len(failed) == 0:
print('='*80)
print('No issues encountered.')
print('='*80)
else:
print('='*80)
print('These packages failed to build:')
print()
for pkg in failed:
print('*', pkg['name'])
print()
proc = subprocess.run(['nix', 'log', pkg['drvPath']],
stdout=subprocess.PIPE,
encoding='UTF-8')
if proc.returncode != 0:
print(f' Error: Failed to fetch log')
continue
print(f' ---- Last {log_excerpt} lines of log follow ----')
lines = proc.stdout.split('\n')
if len(lines) > log_excerpt:
print(' ⋮')
print('\n '.join(lines[-log_excerpt:]))
print(' ---- End of log ----------------------------')
print()
print()
print('='*80)
if __name__ == "__main__":
assert os.environ['GHC_TARBALL'] != None
summary = read_summary()
export_logs(summary)
json.dump(summary, Path('summary.json').open('w'))
show_failures(summary)
Path('summary.dot').write_text(export_dot(summary))
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment