Merge branch 'gitlab-ci-nix' into 'master'

Nix-based CI

See merge request !2

.gitignore

+dist-newstyle
+packages
+cabal.project.local
\ No newline at end of file

.gitlab-ci.yml

+# GHC/head.hackage CI support
+# ===========================
+#
+# This is the GitLab CI automation that drives GHC's head.hackage testing.
+# The goal is to be able to test GHC by building a (small) subset of Hackage.
+# Moreover, we want to be able to collect logs of failed builds as well as
+# performance metrics from builds that succeed.
+#
+# To accomplish this we use head.hackage's native Nix support and the
+# ghc-artefact-nix expression to make GHC binary distributions usable from
+# within Nix. These components are tied together by ./scripts/build-all.nix,
+# which contains the list of packages which we build as well as some simple
+# configuration to minimize the cost of the builds.
+#
+
+variables:
+  # Commit of ghc/ci-images repository from which to pull Docker images
+  DOCKER_REV: 6d19c3adc1f5c28c82aed8c5b1ac40931ac60f3f
+
+  # A default for testing
+  GHC_TARBALL: "https://gitlab.haskell.org/ghc/ghc/-/jobs/111365/artifacts/raw/ghc-x86_64-fedora27-linux.tar.xz"
+
+  # Project ID of ghc/ghc
+  GHC_PROJECT_ID: "1"
+
+  # ACCESS_TOKEN provided via protected environment variable
+
+build:
+  tags:
+    - x86_64-linux
+    - head.hackage
+
+  image: nixos/nix
+
+  cache:
+    key: build-all
+    paths:
+      - store.nar
+
+  before_script:
+    - |
+      if [ -e store.nar ]; then
+        echo "Extracting cached Nix store..."
+        nix-store --import -vv < store.nar || echo "invalid cache"
+      else
+        echo "No cache found"
+      fi
+
+  script:
+    - |
+      if [ -n "$GHC_PIPELINE_ID" ]; then
+        job_name="validate-x86_64-linux-fedora27"
+        job_id=$(nix run -f scripts/build-all.nix find-job \
+          --arg bindistTarball $GHC_TARBALL \
+          -c find-job.sh $GHC_PROJECT_ID $GHC_PIPELINE_ID $job_name)
+        echo "Pulling ${job_name} binary distribution from Pipeline $GHC_PIPELINE_ID (job $job_id)..."
+        GHC_TARBALL="https://gitlab.haskell.org/ghc/ghc/-/jobs/$job_id/artifacts/raw/ghc-x86_64-fedora27-linux.tar.xz"
+      fi
+
+    - echo "Bindist tarball is $GHC_TARBALL"
+    - nix-build scripts/build-all.nix -j$CPUS
+        --no-build-output
+        -A buildDepends
+        --arg bindistTarball $GHC_TARBALL
+    - nix-store --export $(nix-store -qR --include-outputs $(nix-instantiate --quiet scripts/build-all.nix --arg bindistTarball $GHC_TARBALL -A buildDepends -A ghc)) > store.nar
+    - ret=0
+    - nix-build scripts/build-all.nix
+        -j$CPUS --no-build-output
+        -A testedPackages
+        --keep-going
+        --arg bindistTarball $GHC_TARBALL
+        || (echo "Build failed!"; ret=1)
+    - scripts/summarize.py || echo "summarize script failed"
+    - exit $ret
+
+  after_script:
+    - nix run -f '<nixpkgs>' gnutar -c tar -zcf logs.tar.gz logs
+      # The following throws a fontconfig error but it appears to be innocuous.
+    - nix run -f '<nixpkgs>' graphviz -c dot -O -T svg summary.dot
+    - ls -lh
+
+  artifacts:
+    when: always
+    paths:
+    - logs.tar.gz
+    - summary.json
+    - summary.dot
+    - summary.dot.svg

README.md

@@ -140,6 +140,34 @@ build `head.hackage` packages:
 ```
 $ nix build -f ./. --arg ghc "(import ghc-from-source.nix {ghc-path=$GHC_TREE;})"
 ```
+### GitLab CI
+
+[GHC's GitLab instance](https://gitlab.haskell.org/ghc/head.hackage) uses
+GitLab CI and `nix` to build a subset of the head.hackage package set using GHC
+snapshots.
+
+To run a similar build locally simply download a binary distribution from a
+`x86_64-fedora27-linux` CI job and run:
+```
+$ nix build -f scripts/build-all.nix testedPackages \
+    --arg bindistTarball ./ghc-x86_64-fedora27-linux.tar.xz
+```
+This will build the set of packages defined by the `testedPackages` list in
+`scripts/build-all.nix`.
+
+After building `testedPackages` (allowing for failures) the CI job runs
+`scripts/summarize.py`, which produces a few artifacts:
+
+ * a JSON summary, which includes the full dependency graph as well as which
+   package builds failed
+ * a DOT graph showing the package depedencies and their build success
+ * a directory of build logs
+
+You can run `scripts/summarize.py` locally with
+```
+$ export GHC_TARBALL=./ghc-x86_64-fedora27-linux.tar.xz
+$ python3 scripts/summarize.py
+```
 
 ### Travis CI
 

default.nix

@@ -8,16 +8,18 @@
 #       nix build -f --arg ghc "(import build.nix {ghc-path=$GHC_TREE;})"
 #
 let
-  rev = "f2632f5c60f574d787fc5490efb3f43f9e6209b7";
+  rev = "e0818a15305561ea1ebe36203d914465fd71453d";
   baseNixpkgs =
-    builtins.fetchTarball {
-    url = "https://github.com/NixOS/nixpkgs/archive/${rev}.tar.gz";
-    sha256 = "1c36p7w36i38gng3yp1nd5vz0p2dwrax5szjkvnmdxfklggs7knf";
-  };
+    fetchTarball {
+      url = "https://github.com/NixOS/nixpkgs/archive/${rev}.tar.gz";
+      sha256 = "017j741anrl66nndfmblp6xc1lz7783dh8lf19srmxnk5xk4wn7l";
+    };
 in
 
 # ghc: path to a GHC source tree
-{ ghc ? import ./ghc-prerelease.nix }:
+{ ghc ? import ./ghc-prerelease.nix
+, haskellOverrides ? (self: super: self)
+}:
 
 let
   jailbreakOverrides = self: super: {
@@ -33,14 +35,17 @@ let
     haskellPackages =
       let patchesOverrides = self.callPackage patches {};
           patches = self.callPackage (import ./scripts/overrides.nix) { patches = ./patches; };
-          overrides = self.lib.composeExtensions patchesOverrides jailbreakOverrides;
+          overrides =
+            self.lib.composeExtensions
+              haskellOverrides
+              (self.lib.composeExtensions patchesOverrides jailbreakOverrides);
 
-          baseHaskellPackages = self.callPackage "${baseNixpkgs}/pkgs/development/haskell-modules" {
+          baseHaskellPackages = self.callPackage "${baseNixpkgs}/pkgs/development/haskell-modules" rec {
             haskellLib = import "${baseNixpkgs}/pkgs/development/haskell-modules/lib.nix" {
               inherit (self) lib;
               pkgs = self;
             };
-            buildHaskellPackages = haskellPackages; #self.buildPackages.haskell.packages.ghc843;
+            buildHaskellPackages = self.buildPackages.haskell.packages.ghc865;
             ghc = ghcHEAD;
             compilerConfig = self1: super1: {
               # Packages included in GHC's global package database
@@ -76,10 +81,9 @@ let
 
               doctest = haskellPackages.callHackage "doctest" "0.16.0" {};
               http-api-data = haskellPackages.callPackage ./http-api-data.nix {};
-              tagged = self1.callHackage "tagged" "0.8.6" {};
 
-              jailbreak-cabal = self.haskell.packages.ghc802.jailbreak-cabal;
-              cabal2nix = self.haskell.packages.ghc843.cabal2nix;
+              jailbreak-cabal = buildHaskellPackages.jailbreak-cabal;
+              cabal2nix = buildHaskellPackages.cabal2nix;
             };
           };
       in baseHaskellPackages.extend overrides;

scripts/build-all.nix

+# GHC/head.hackage CI support
+#
+# See .gitlab-ci.yml in the root of this repository.
+
+# Expects to be run on x86_64.
+# bindistTarball should be a fedora27 tarball.
+{ bindistTarball }:
+
+let
+  # GHC from the given bindist.
+  ghc =
+    let
+      commit = "633c6d7553d0f2c91245bcf47ae539cfdb7aaa13";
+      src = fetchTarball {
+        url = "https://github.com/bgamari/ghc-artefact-nix/archive/${commit}.tar.gz";
+        sha256 = "0yl7dk8drb92ipgph1mxx8my4gy9py27m749zw6ah6np4gvdp9sx";
+      };
+    in nixpkgs.callPackage "${src}/artifact.nix" {} {
+      ncursesVersion = "6";
+      bindistTarball =
+        if builtins.typeOf bindistTarball == "path"
+        then bindistTarball
+        else builtins.fetchurl bindistTarball;
+    };
+
+  # Build configuration.
+  nixpkgs = import ../. {
+    ghc = self: ghc;
+    haskellOverrides = self: super: rec {
+      mkDerivation = drv: super.mkDerivation (drv // {
+        doHaddock = false;
+        enableLibraryProfiling = false;
+        enableExecutableProfiling = false;
+        hyperlinkSource = false;
+        configureFlags = (drv.configureFlags or []) ++ [
+          "--ghc-options=-ddump-timings"
+        ];
+      });
+    };
+  };
+
+  # Build dependencies worth caching.
+  buildDepends = with nixpkgs; [
+    buildPackages.haskellPackages.jailbreak-cabal
+    buildPackages.haskellPackages.cabal2nix
+  ];
+
+  # Get build-time dependencies of a derivation.
+  # See https://github.com/NixOS/nix/issues/1245#issuecomment-401642781
+  getDepends = pkg: let
+    drv = builtins.readFile pkg.drvPath;
+    storeDirRe = lib.replaceStrings [ "." ] [ "\\." ] builtins.storeDir;
+    storeBaseRe = "[0-9a-df-np-sv-z]{32}-[+_?=a-zA-Z0-9-][+_?=.a-zA-Z0-9-]*";
+    re = "(${storeDirRe}/${storeBaseRe}\\.drv)";
+    inputs = lib.concatLists (lib.filter lib.isList (builtins.split re drv));
+  in map import inputs;
+
+  # The packages which we are here to test
+  testedPackages = with nixpkgs.haskell.lib; with nixpkgs.haskellPackages; {
+    inherit lens aeson criterion scotty;
+    # servant: Don't distribute Sphinx documentation
+    servant = overrideCabal servant { postInstall = ""; };
+    # singletons: Disable testsuite since it often breaks due to spurious
+    # TH-pretty-printing changes
+    singletons = dontCheck singletons;
+  };
+
+  inherit (nixpkgs) lib;
+
+  transHaskellDeps = drv:
+    let
+      haskellDeps = drv.passthru.getBuildInputs.haskellBuildInputs or [];
+    in [{
+      haskellDeps = map (dep: dep.name) haskellDeps;
+      drvName = drv.name;
+      drvPath = drv.drvPath;
+      out = drv.outPath;
+      name = drv.passthru.pname;
+      version = drv.passthru.version;
+    }] ++ lib.concatMap transHaskellDeps haskellDeps;
+
+  summary = {
+    pkgs = lib.concatMap transHaskellDeps (lib.attrValues testedPackages);
+    roots = map (drv: drv.name) (lib.attrValues testedPackages);
+  };
+
+  # Find Job ID of the given job name in the given pipeline
+  find-job = nixpkgs.writeScriptBin "find-job.sh" ''
+    set -e
+
+    project_id=$1
+    pipeline_id=$2
+    job_name=$3
+
+    # Access token is a protected environment variable in the head.hackage project and
+    # is necessary for this query to succeed. Sadly job tokens only seem to
+    # give us access to the project being built.
+    ${nixpkgs.curl}/bin/curl \
+      --silent --show-error \
+      -H "Private-Token: $ACCESS_TOKEN" \
+      "https://gitlab.haskell.org/api/v4/projects/$project_id/pipelines/$pipeline_id/jobs?scope[]=success" \
+      > resp.json
+
+    job_id=$(${nixpkgs.jq}/bin/jq ". | map(select(.name == \"$job_name\")) | .[0].id" < resp.json)
+    if [ "$job_id" = "null" ]; then
+      echo "Error finding job $job_name for $pipeline_id in project $project_id:" >&2
+      cat resp.json >&2
+      rm resp.json
+      exit 1
+    else
+      rm resp.json
+      echo -n "$job_id"
+    fi
+  '';
+
+in {
+  inherit nixpkgs ghc testedPackages buildDepends;
+  inherit (nixpkgs) haskellPackages lib;
+  testedPackageNames = nixpkgs.lib.attrNames testedPackages;
+  inherit summary find-job;
+}

scripts/generate-nix-overrides.hs

@@ -54,8 +54,8 @@ generateOverride prefix patchDir (patchExtless, patchTypes) = do
   let pname = display (packageName pid)
       version = versionNumbers (packageVersion pid)
   return . (packageName pid,) . (version,) $ 
-    "dontRevise "
-    ++ foldl' (override prefix patchDir patchExtless) ("super."++pname) patchTypes
+    "haskell.lib.doJailbreak (dontRevise "
+    ++ foldl' (override prefix patchDir patchExtless) ("super."++pname) patchTypes ++ ")"
 
 patchFunction :: PatchType -> String
 patchFunction = \case

scripts/summarize.py

+#!/usr/bin/env nix-shell
+#!nix-shell -p python3 -i python3
+
+import os
+from pathlib import Path
+import subprocess
+import json
+
+def read_summary():
+    summary = subprocess.check_output(
+        ['nix', 'eval', '--json',
+         '--arg', 'bindistTarball', os.environ['GHC_TARBALL'],
+         '-f', 'scripts/build-all.nix',
+         'summary'],
+        encoding = 'UTF-8')
+
+    summary = json.loads(summary)
+    for pkg in summary['pkgs']:
+        pkg['failed'] = not Path(pkg['out']).exists()
+        del pkg['out']
+
+    return summary
+
+def export_logs(summary):
+    logs = Path('logs')
+    logs.mkdir(exist_ok=True)
+    # Deduplicate
+    pkgs = { pkg['drvPath']: pkg for pkg in summary['pkgs'] }
+    for pkg in pkgs.values():
+        print(f'Collecting log for {pkg["drvPath"]}...')
+        p = subprocess.run(['nix', 'log', pkg['drvPath']],
+                           stdout=open(logs / f'{pkg["drvName"]}.log', 'wb'))
+        if p.returncode != 0:
+            print(f'Error exporting log for {pkg["drvPath"]}')
+
+def export_dot(summary):
+    pkgsDict = { pkg['drvName']: pkg for pkg in summary['pkgs'] }
+    edges = {
+        (pkg['drvName'], dep)
+        for pkg in summary['pkgs']
+        for dep in pkg['haskellDeps']
+    }
+
+    s = 'digraph {\n'
+    s += '  {overlap=prism};\n'
+
+    s += '  subgraph cluster_roots {\n'
+    s += '    {fillcolor=blue style=filled};\n'
+    for pkg in summary['roots']:
+        s += f'    "{pkg}";\n'
+    s += '  }\n'
+    s += '\n'
+
+    for pkg, dep in edges:
+        s +=  f'  "{pkg}" -> "{dep}";\n'
+
+    for pkg in summary['pkgs']:
+        if pkg['failed']:
+            s += f'  "{pkg["drvName"]}" [ color=indianred style=filled ];\n'
+
+    s += '}'
+    return s
+
+def show_failures(summary, log_excerpt=100):
+    failed = [pkg
+              for pkg in summary['pkgs']
+              if pkg['failed']]
+    if len(failed) == 0:
+        print('='*80)
+        print('No issues encountered.')
+        print('='*80)
+    else:
+        print('='*80)
+        print('These packages failed to build:')
+        print()
+        for pkg in failed:
+            print('*', pkg['name'])
+            print()
+            proc = subprocess.run(['nix', 'log', pkg['drvPath']],
+                                 stdout=subprocess.PIPE,
+                                 encoding='UTF-8')
+            if proc.returncode != 0:
+                print(f'    Error: Failed to fetch log')
+                continue
+
+            print(f'  ---- Last {log_excerpt} lines of log follow ----')
+            lines = proc.stdout.split('\n')
+            if len(lines) > log_excerpt:
+                print('    ⋮')
+
+            print('\n    '.join(lines[-log_excerpt:]))
+            print('  ---- End of log ----------------------------')
+            print()
+            print()
+
+        print('='*80)
+
+if __name__ == "__main__":
+    assert os.environ['GHC_TARBALL'] != None
+    summary = read_summary()
+    export_logs(summary)
+    json.dump(summary, Path('summary.json').open('w'))
+    show_failures(summary)
+    Path('summary.dot').write_text(export_dot(summary))