diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e6269e68e092039817dd5f844bd4f152cbc8b12b..62505cd80f5e7a6e4f6affc1c970279a52582c01 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,6 +13,10 @@ # configuration to minimize the cost of the builds. # +stages: + - test + - deploy + variables: # Commit of ghc/ci-images repository from which to pull Docker images DOCKER_REV: 6d19c3adc1f5c28c82aed8c5b1ac40931ac60f3f @@ -26,6 +30,8 @@ variables: # ACCESS_TOKEN provided via protected environment variable build: + stage: test + tags: - x86_64-linux - head.hackage @@ -54,7 +60,6 @@ build: --arg bindistTarball $GHC_TARBALL \ -c find-job.sh $GHC_PROJECT_ID $GHC_PIPELINE_ID $job_name) echo "Pulling ${job_name} binary distribution from Pipeline $GHC_PIPELINE_ID (job $job_id)..." - GHC_TARBALL="https://gitlab.haskell.org/ghc/ghc/-/jobs/$job_id/artifacts/raw/ghc-x86_64-fedora27-linux.tar.xz" fi - echo "Bindist tarball is $GHC_TARBALL" @@ -86,3 +91,31 @@ build: - summary.json - summary.dot - summary.dot.svg + +# Build and deploy a Hackage repository +update-repo: + stage: deploy + + tags: + - x86_64-linux + - head.hackage + + image: nixos/nix + + variables: + #KEYS_TARBALL: https://downloads.haskell.org/ghc/head.hackage-keys.tar.enc + KEYS_TARBALL: http://home.smart-cactus.org/~ben/head.hackage-keys.tar.enc + # KEYS_TARBALL_KEY provided by protected variable + + only: + - master + + script: + - nix-channel --add https://nixos.org/channels/nixpkgs-unstable nixpkgs + - nix-channel --update + - nix build -f scripts/build-repo.nix + - nix run -f scripts/build-repo.nix -c build-repo.sh build-repo + - mv repo public + + after_script: + - rm -Rf keys diff --git a/README.md b/README.md index 745617e92ec31d9b4af43ccf6b1bb01a2264679c..7dd1c87f5580f39ea44df6e49cedc37dbebd62d3 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,12 @@ $ export GHC_TARBALL=./ghc-x86_64-fedora27-linux.tar.xz $ python3 scripts/summarize.py ``` +### Hackage repository + +[GHC's GitLab instance](https://gitlab.haskell.org/ghc/head.hackage) uses +GitLab CI to deploy a Hackage repository with the patches provided by +`head.hackage`. See the [repository]() for usage instructions. + ### Travis CI The [Travis CI script generator](https://github.com/haskell-hvr/multi-ghc-travis) has recently added support for enabling the `HEAD.hackage` repository automatically for jobs using unreleased GHC versions. diff --git a/scripts/build-repo.nix b/scripts/build-repo.nix new file mode 100644 index 0000000000000000000000000000000000000000..22de29305b2da25abbc2c76c2a809d528cfa4a32 --- /dev/null +++ b/scripts/build-repo.nix @@ -0,0 +1,40 @@ +{ nixpkgs ? (import <nixpkgs> {}) }: + +with nixpkgs; +let + hackage-repo-tool = + let + src = fetchFromGitHub { + owner = "haskell-vanguard"; + repo = "hackage-security"; + rev = "5ce34d42ffa9d760dafcf216a10d6f72bd82a1d3"; + sha256 = "07a6f9gl4xn4fpc09wypm29cwghx31dw0lzq8421v9fjb5m2r96w"; + }; + in haskellPackages.callCabal2nix "hackage-repo-tool" "${src}/hackage-repo-tool" {}; + + overlay-tool = + let + src = fetchFromGitHub { + owner = "bgamari"; + repo = "hackage-overlay-repo-tool"; + rev = "40282de72ebd4158bfca677b8fa179ed74860e68"; + sha256 = "1z9yy63a9l149in3cb42cylpp1mw70smqh6hrp7n15n3zjfa1w1x"; + }; + in haskellPackages.callCabal2nix "hackage-overlay-repo-tool" src {}; + + build-repo = + let + deps = [ + bash curl gnutar patch rsync openssl + cabal-install ghc + hackage-repo-tool overlay-tool + ]; + in runCommand "repo" { + nativeBuildInputs = [ makeWrapper ]; + } '' + mkdir -p $out/bin + makeWrapper ${./build-repo.sh} $out/bin/build-repo.sh \ + --prefix PATH : ${stdenv.lib.makeBinPath deps} + ''; +in + build-repo diff --git a/scripts/build-repo.sh b/scripts/build-repo.sh new file mode 100755 index 0000000000000000000000000000000000000000..a584ddd85772606ae79832a087449565f30e08a9 --- /dev/null +++ b/scripts/build-repo.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash + +# Script to generate and deploy Hackage repository for head.hackage patchset. +# The hackage-security keys are stored in a tarball, $KEYS_TARBALL, encrypted +# with the key material given in $KEYS_TARBALL_KEY. +# +# To test under NixOS: nix run nixpkgs.openssl nixpkgs.pwgen + +set -e + +cipher=aes-256-cbc + +# For use by administrator. +gen_keys_tarball() { + hackage-repo-tool create-keys --keys=./keys + pass="$(pwgen 32 1)" + tar -c keys | openssl enc -$cipher -e -k "$pass" > keys.tar.enc + echo "Wrote ./keys.tar.enc" + echo "KEYS_TARBALL_KEY = $pass" +} + +# Helper to decrypt and extract the keys tarball. +extract_keys_tarball() { + if [ -z "$KEYS_TARBALL_KEY" ]; then + echo "Can't extract keys tarball: KEYS_TARBALL_KEY not set" + exit 1 + fi + if [ -z "$KEYS_TARBALL" ]; then + echo "Can't extract keys tarball: KEYS_TARBALL not set" + exit 1 + fi + + curl $KEYS_TARBALL | openssl enc -$cipher -d -k "$KEYS_TARBALL_KEY" | tar -x + if [ ! -d ./keys ]; then + echo "Key tarball extraction failed" + exit 1 + fi +} + +build_index_html() { + keys="$(find keys/root -printf "%f")" + cat >repo/cabal.project.local <<EOF +repository head.hackage + url: http://head.hackage.haskell.org/ + secure: True + root-keys: ${keys//.private/ } + key-threshold: 3 +EOF + + cat >repo/index.html <<EOF +<html> + <head> + <title>head.hackage</title> + </head> + <body> + <h1>head.hackage @ GHC</h1> + <p>See the <a href="">head.hackage documentation</a> for details on using this repository.</p> + <pre> + $(cat repo/cabal.project) + </pre> + </body> +</html> +EOF +} + +# Build the hackage repository +build_repo() { + extract_keys_tarball + + # hackage-repo-tool bootstrap fails unless there is at least one package in the + # repo. Seed things with acme-box. + cabal update + cabal fetch acme-box-0.0.0.0 + mkdir -p repo/package + cp $HOME/.cabal/packages/hackage.haskell.org/acme-box/0.0.0.0/acme-box-0.0.0.0.tar.gz repo/package + + hackage-repo-tool bootstrap --keys=./keys --repo=./repo + + mkdir -p template patches.cache + tool \ + --patches=./patches \ + --repo-cache=./cache \ + --keys=./keys \ + --repo-name=head.hackage \ + --template=template \ + ./repo +} + +case $1 in + gen-keys) gen_keys_tarball ;; + extract-keys) extract_keys_tarball ;; + build-repo) build_repo ;; + *) + echo "Unknown command $1" + exit 1 + ;; +esac