Deploy Hackage repository

.gitlab-ci.yml

@@ -13,6 +13,10 @@
 # configuration to minimize the cost of the builds.
 #
 
+stages:
+  - test
+  - deploy
+
 variables:
   # Commit of ghc/ci-images repository from which to pull Docker images
   DOCKER_REV: 6d19c3adc1f5c28c82aed8c5b1ac40931ac60f3f
@@ -26,6 +30,8 @@ variables:
   # ACCESS_TOKEN provided via protected environment variable
 
 build:
+  stage: test
+
   tags:
     - x86_64-linux
     - head.hackage
@@ -54,7 +60,6 @@ build:
           --arg bindistTarball $GHC_TARBALL \
           -c find-job.sh $GHC_PROJECT_ID $GHC_PIPELINE_ID $job_name)
         echo "Pulling ${job_name} binary distribution from Pipeline $GHC_PIPELINE_ID (job $job_id)..."
-        GHC_TARBALL="https://gitlab.haskell.org/ghc/ghc/-/jobs/$job_id/artifacts/raw/ghc-x86_64-fedora27-linux.tar.xz"
       fi
 
     - echo "Bindist tarball is $GHC_TARBALL"
@@ -86,3 +91,31 @@ build:
     - summary.json
     - summary.dot
     - summary.dot.svg
+
+# Build and deploy a Hackage repository
+update-repo:
+  stage: deploy
+
+  tags:
+    - x86_64-linux
+    - head.hackage
+
+  image: nixos/nix
+
+  variables:
+    #KEYS_TARBALL: https://downloads.haskell.org/ghc/head.hackage-keys.tar.enc
+    KEYS_TARBALL: http://home.smart-cactus.org/~ben/head.hackage-keys.tar.enc
+    # KEYS_TARBALL_KEY provided by protected variable
+
+  only:
+    - master
+
+  script:
+    - nix-channel --add https://nixos.org/channels/nixpkgs-unstable nixpkgs
+    - nix-channel --update
+    - nix build -f scripts/build-repo.nix
+    - nix run -f scripts/build-repo.nix -c build-repo.sh build-repo
+    - mv repo public
+
+  after_script:
+    - rm -Rf keys

README.md

@@ -169,6 +169,12 @@ $ export GHC_TARBALL=./ghc-x86_64-fedora27-linux.tar.xz
 $ python3 scripts/summarize.py
 ```
 
+### Hackage repository
+
+[GHC's GitLab instance](https://gitlab.haskell.org/ghc/head.hackage) uses
+GitLab CI to deploy a Hackage repository with the patches provided by
+`head.hackage`. See the [repository]() for usage instructions.
+
 ### Travis CI
 
 The [Travis CI script generator](https://github.com/haskell-hvr/multi-ghc-travis) has recently added support for enabling the `HEAD.hackage` repository automatically for jobs using unreleased GHC versions.

scripts/build-repo.nix

+{ nixpkgs ? (import <nixpkgs> {}) }:
+
+with nixpkgs;
+let
+  hackage-repo-tool =
+    let
+      src = fetchFromGitHub {
+        owner = "haskell-vanguard";
+        repo = "hackage-security";
+        rev = "5ce34d42ffa9d760dafcf216a10d6f72bd82a1d3";
+        sha256 = "07a6f9gl4xn4fpc09wypm29cwghx31dw0lzq8421v9fjb5m2r96w";
+      };
+    in haskellPackages.callCabal2nix "hackage-repo-tool" "${src}/hackage-repo-tool" {};
+
+  overlay-tool =
+    let
+      src = fetchFromGitHub {
+        owner = "bgamari";
+        repo = "hackage-overlay-repo-tool";
+        rev = "40282de72ebd4158bfca677b8fa179ed74860e68";
+        sha256 = "1z9yy63a9l149in3cb42cylpp1mw70smqh6hrp7n15n3zjfa1w1x";
+      };
+    in haskellPackages.callCabal2nix "hackage-overlay-repo-tool" src {};
+
+  build-repo =
+    let
+      deps = [
+        bash curl gnutar patch rsync openssl
+        cabal-install ghc
+        hackage-repo-tool overlay-tool
+      ];
+    in runCommand "repo" {
+    nativeBuildInputs = [ makeWrapper ];
+  } ''
+    mkdir -p $out/bin
+    makeWrapper ${./build-repo.sh} $out/bin/build-repo.sh \
+        --prefix PATH : ${stdenv.lib.makeBinPath deps}
+  '';
+in
+  build-repo

scripts/build-repo.sh

+#!/usr/bin/env bash
+
+# Script to generate and deploy Hackage repository for head.hackage patchset.
+# The hackage-security keys are stored in a tarball, $KEYS_TARBALL, encrypted
+# with the key material given in $KEYS_TARBALL_KEY.
+#
+# To test under NixOS: nix run nixpkgs.openssl nixpkgs.pwgen
+
+set -e
+
+cipher=aes-256-cbc
+
+# For use by administrator.
+gen_keys_tarball() {
+  hackage-repo-tool create-keys --keys=./keys
+  pass="$(pwgen 32 1)"
+  tar -c keys | openssl enc -$cipher -e -k "$pass" > keys.tar.enc
+  echo "Wrote ./keys.tar.enc"
+  echo "KEYS_TARBALL_KEY = $pass"
+}
+
+# Helper to decrypt and extract the keys tarball.
+extract_keys_tarball() {
+  if [ -z "$KEYS_TARBALL_KEY" ]; then
+    echo "Can't extract keys tarball: KEYS_TARBALL_KEY not set"
+    exit 1
+  fi
+  if [ -z "$KEYS_TARBALL" ]; then
+    echo "Can't extract keys tarball: KEYS_TARBALL not set"
+    exit 1
+  fi
+
+  curl $KEYS_TARBALL | openssl enc -$cipher -d -k "$KEYS_TARBALL_KEY" | tar -x
+  if [ ! -d ./keys ]; then
+    echo "Key tarball extraction failed"
+    exit 1
+  fi
+}
+
+build_index_html() {
+  keys="$(find keys/root -printf "%f")"
+  cat >repo/cabal.project.local <<EOF
+repository head.hackage
+   url: http://head.hackage.haskell.org/
+   secure: True
+   root-keys: ${keys//.private/ }
+   key-threshold: 3
+EOF
+
+  cat >repo/index.html <<EOF
+<html>
+  <head>
+    <title>head.hackage</title>
+  </head>
+  <body>
+    <h1>head.hackage @ GHC</h1>
+    <p>See the <a href="">head.hackage documentation</a> for details on using this repository.</p>
+    <pre>
+      $(cat repo/cabal.project)
+    </pre>
+  </body>
+</html>
+EOF
+}
+
+# Build the hackage repository
+build_repo() {
+  extract_keys_tarball
+
+  # hackage-repo-tool bootstrap fails unless there is at least one package in the
+  # repo. Seed things with acme-box.
+  cabal update
+  cabal fetch acme-box-0.0.0.0
+  mkdir -p repo/package
+  cp $HOME/.cabal/packages/hackage.haskell.org/acme-box/0.0.0.0/acme-box-0.0.0.0.tar.gz repo/package
+
+  hackage-repo-tool bootstrap --keys=./keys --repo=./repo
+
+  mkdir -p template patches.cache
+  tool \
+    --patches=./patches \
+    --repo-cache=./cache \
+    --keys=./keys \
+    --repo-name=head.hackage \
+    --template=template \
+    ./repo
+}
+
+case $1 in
+  gen-keys) gen_keys_tarball ;;
+  extract-keys) extract_keys_tarball ;;
+  build-repo) build_repo ;;
+  *)
+    echo "Unknown command $1"
+    exit 1
+    ;;
+esac