ci/config.cabal.project

+-- This cabal project file is included in the CI configuration to
+-- preclude the solver from using ancient package versions. See, for instance,
+-- ghc/ghc#23048.
+
+index-state:
+    hackage.haskell.org 2025-05-11T00:00:00Z,
+    head.hackage HEAD
+
+constraints: th-abstraction >= 0.4
+
+-- aeson is now quite old, don't attempt to use it
+constraints: aeson >= 1.0,
+
+-- don't allow uvector, which is deprecated; vector should be used instead
+constraints: uvector == 0.0
+
+-- containers-0.5 generally won't build with any recent compiler
+constraints: containers >= 0.6
+
+-- earlier transformers-compat versions don't include the Eq constraint on Eq1
+-- instances needed by GHC 9.6
+constraints: transformers-compat >= 0.7

ci/config.sh

+# vi: set filetype=sh
+
+# Packages expected not to build due to GHC bugs. This is `source`'d by the CI
+# script and the arguments in BROKEN_ARGS are added to the hackage-ci
+# command-line.
+
+# Mark the named package as broken.
+#
+# Usage:
+#    broken $pkg_name $ghc_ticket_number
+#
+function broken() {
+  pkg_name="$1"
+  ticket="$2"
+  echo "Marking $pkg_name as broken due to #$ticket"
+  EXTRA_OPTS="$EXTRA_OPTS --expect-broken=$pkg_name"
+}
+
+function only_package() {
+  echo "Adding $@ to --only package list"
+  for pkg in $@; do
+    EXTRA_OPTS="$EXTRA_OPTS --only=$pkg"
+  done
+}
+
+function test_package() {
+  echo "Adding $@ to --test-package list"
+  EXTRA_OPTS="$EXTRA_OPTS --test-package=$1=$2"
+}
+
+# Return the version number of the most recent release of the given package
+function latest_version() {
+  pkg=$1
+  curl -s -H "Accept: application/json" -L -X GET http://hackage.haskell.org/package/$pkg/preferred | jq '.["normal-version"] | .[0]' -r
+}
+
+# Add a package to the set of packages that lack patches but are nevertheless
+# tested.
+function extra_package() {
+  pkg_name="$1"
+  version="$2"
+  if [ -z "$version" ]; then
+    version=$(latest_version $pkg_name)
+  fi
+  echo "Adding $pkg_name-$version to extra package set"
+  EXTRA_OPTS="$EXTRA_OPTS --extra-package=$pkg_name==$version"
+}
+
+# Mark a package to be declared with build-tool-depends, not build-depends.
+# This is necessary for packages that do not have a library component.
+function build_tool_package() {
+  pkg_name="$1"
+  echo "Adding $pkg_name as a build-tool package"
+  EXTRA_OPTS="$EXTRA_OPTS --build-tool-package=$pkg_name"
+}
+
+if [ -z "$GHC" ]; then GHC=ghc; fi
+
+function ghc_version() {
+  $GHC --version | sed 's/.*version \([0-9]*\.\([0-9]*\.\)*\)/\1/'
+}
+
+function ghc_commit() {
+  $GHC --print-project-git-commit-id
+}
+
+function ghc_arch() {
+  $GHC --print-host-platform
+}
+
+# ======================================================================
+# Baseline constraints
+#
+# These constraints are applied to preclude the solver from producing build
+# plans using ancient, under-constrained package versions.
+
+EXTRA_OPTS="$EXTRA_OPTS --extra-cabal-fragment=$(pwd)/config.cabal.project"
+
+# ======================================================================
+# The lists begin here
+#
+# For instance:
+#
+#    broken "lens" 17988
+
+version="$(ghc_version)"
+commit="$(ghc_commit)"
+arch="$(ghc_arch)"
+echo "Found GHC $version, commit $commit."
+case $version in
+  9.8.*)
+    #       package                   ticket
+    broken  liquidhaskell-boot        350
+    # singletons-base only supports the latest ghc
+    broken singletons-base            00000
+    ;;
+
+  9.10.*)
+    #       package                   ticket
+    broken  liquidhaskell-boot        350
+    # singletons-base only supports the latest ghc
+    broken singletons-base            00000
+    ;;
+
+  9.12.*)
+    #       package                   ticket
+    broken  liquidhaskell-boot        350
+    ;;
+
+  9.14.*)
+    #       package                   ticket
+    broken  ghcide                    00000
+    broken  Agda                      26154
+    # broken until strict-containers is updated to work with the new containers
+    broken  strict-containers         00000
+    ;;
+
+  9.15.*)
+    #       package                   ticket
+    broken  ghcide                    00000
+    broken  Agda                      26154
+    # broken until strict-containers is updated to work with the new containers
+    broken  strict-containers         00000
+    ;;
+
+  *)
+    echo "No broken packages for GHC $version"
+    ;;
+esac
+
+case $arch in
+  x86_64-*-*)
+  #       package                   ticket
+  ;;
+  aarch64-*-*)
+  # These just don't build on aarch64
+  #       package                   ticket
+  broken  charsetdetect             00000
+  broken  packman                   00000
+  ;;
+  *)
+  echo "$arch is unknown to head.hackage, assuming nothing is broken."
+  ;;
+esac
+
+
+
+# Extra packages
+# ==============
+#
+# These are packages which we don't have patches for but want to test anyways.
+extra_package lens 5.2.3
+extra_package generic-lens 2.2.2.0
+extra_package optics 0.4.2.1
+extra_package aeson 2.2.3.0
+extra_package criterion 1.6.3.0
+extra_package scotty 0.21
+extra_package generic-lens 2.2.2.0
+extra_package microstache 1.0.2.3
+extra_package singletons-base 3.5
+extra_package servant 0.20.1
+extra_package hgmp 0.1.2.1
+extra_package Agda 2.7.0.1
+extra_package mmark 0.0.7.6
+extra_package doctest 0.24.0
+extra_package tasty 1.5.3
+extra_package pandoc 3.1.11.1
+extra_package servant-conduit 0.16
+extra_package servant-machines 0.16
+extra_package linear-generics 0.2.3
+extra_package futhark 0.25.13
+extra_package generic-random 1.5.0.1
+extra_package lame 0.2.2
+extra_package inspection-testing 0.5.0.3
+extra_package ghcide 2.9.0.0
+extra_package ghc-typelits-extra 0.4.7
+
+# This package is affected by https://gitlab.haskell.org/ghc/ghc/-/issues/22912
+extra_package vector-space 0.16
+
+# Build-tool packages
+build_tool_package alex
+build_tool_package happy
+build_tool_package c2hs
+
+# $BUILD_MODE controls how head.hackage runs.
+# ===========================================
+#
+# Four build modes exist: FULL, QUICK, TEST, and COMPAT.
+#
+# FULL.
+# ------
+# Build all patched + extra packages.
+#
+# QUICK.
+# ------
+# Build the "quick" configuration, which builds a small subset of the overall
+# package set. (Also runs tests!) We do this during the merge request validation
+# pipeline. Note: If "$QUICK" is non-null, it is used as a backwards-compat
+# synonym for BUILD_MODE=QUICK.
+#
+# TEST.
+# -----
+# Just build the local test packages and run the tests.
+#
+# COMPAT: FULL + TEST.
+# --------------------
+# Backwards-compat default build mode.
+#
+: ${BUILD_MODE:=COMPAT}
+if [ -n "$QUICK" ]; then
+    BUILD_MODE=QUICK
+fi
+case "$BUILD_MODE" in
+    FULL) ;;
+    QUICK)
+        only_package tasty
+        only_package Cabal
+        only_package microlens
+        only_package free
+        only_package optparse-applicative
+        test_package system-test "$(pwd)/../tests/ghc-debug/**/*.cabal"
+        test_package ghc-tests "$(pwd)/../tests/ghc-tests"
+        ;;
+    TEST)
+        # FIXME: I specify a single "only_package" to prevent all the other
+        # packages from being built. Morally, I really want to say "build
+        # nothing at all besides the tests".
+        only_package tasty
+        test_package system-test "$(pwd)/../tests/ghc-debug/**/*.cabal"
+        test_package ghc-tests "$(pwd)/../tests/ghc-tests"
+        test_package all "$(pwd)/../tests/text"
+        test_package bytestring-tests "$(pwd)/../tests/bytestring"
+        test_package all "$(pwd)/../tests/containers/containers-tests"
+        ;;
+    COMPAT)
+        test_package system-test "$(pwd)/../tests/ghc-debug/**/*.cabal"
+        test_package ghc-tests "$(pwd)/../tests/ghc-tests"
+        ;;
+esac

ci/default.nix

+{ sources ? import ./nix/sources.nix, nixpkgs ? (import sources.nixpkgs.outPath {}) }:
+
+with nixpkgs;
+let
+  haskellPackages = nixpkgs.haskellPackages;
+
+  hackage-repo-tool =
+    let src = sources.hackage-security.outPath;
+    in nixpkgs.haskell.lib.doJailbreak (haskellPackages.callCabal2nix "hackage-repo-tool" "${src}/hackage-repo-tool" {});
+
+  overlay-tool =
+    let src = sources.overlay-tool;
+    in nixpkgs.haskell.lib.doJailbreak (haskellPackages.callCabal2nix "hackage-overlay-repo-tool" src { });
+
+  head-hackage-ci =
+    let
+      src = nixpkgs.nix-gitignore.gitignoreSource [] ./.;
+    in haskellPackages.callCabal2nix "head-hackage-ci" src {};
+
+  buildDeps = import ./build-deps.nix { pkgs = nixpkgs; };
+
+  buildDepsFragment =
+    let
+
+      mkCabalFragment = pkgName: deps:
+        with pkgs.lib;
+        let
+          libDirs = concatStringsSep " " (map (dep: getOutput "lib" dep + "/lib") deps);
+          includeDirs = concatStringsSep " " (map (dep: getOutput "dev" dep + "/include") deps);
+        in ''
+        package ${pkgName}
+          extra-lib-dirs: ${libDirs}
+          extra-include-dirs: ${includeDirs}
+        '';
+    in
+      pkgs.lib.concatStringsSep "\n"
+        (pkgs.lib.mapAttrsToList mkCabalFragment buildDeps);
+
+  buildDepsFile = pkgs.writeText "deps.cabal.project" buildDepsFragment;
+
+  build-repo =
+    let
+      deps = [
+        bash curl gnutar findutils patch rsync openssl
+        haskellPackages.cabal-install haskellPackages.ghc gcc binutils-unwrapped pwgen gnused
+        hackage-repo-tool overlay-tool python3 jq pkg-config
+        git # cabal-install wants this to fetch source-repository-packages
+      ];
+
+      pkg_config_depends = lib.makeSearchPathOutput "dev" "lib/pkgconfig" (lib.concatLists (lib.attrValues buildDeps));
+
+    in
+      runCommand "repo" {
+        nativeBuildInputs = [ makeWrapper ];
+      } ''
+        mkdir -p $out/bin
+        makeWrapper ${head-hackage-ci}/bin/head-hackage-ci $out/bin/head-hackage-ci \
+            --prefix PATH : ${lib.makeBinPath deps}:$out/bin
+
+        makeWrapper ${./build-repo.sh} $out/bin/build-repo.sh \
+            --prefix PATH : ${lib.makeBinPath deps}:$out/bin
+
+        makeWrapper ${./discover_tarball.sh} $out/bin/discover_tarball.sh \
+            --prefix PATH : ${lib.makeBinPath deps}:$out/bin
+
+        makeWrapper ${../run-ci} $out/bin/run-ci \
+            --prefix PATH : ${lib.makeBinPath deps}:$out/bin \
+            --prefix PKG_CONFIG_PATH : ${pkg_config_depends} \
+            --set USE_NIX 1 \
+            --set CI_CONFIG ${./config.sh}
+
+        makeWrapper ${./find-job.sh} $out/bin/find-job \
+            --prefix PATH : ${lib.makeBinPath deps}:$out/bin
+
+        makeWrapper ${./find-latest-job.sh} $out/bin/find-latest-job \
+            --prefix PATH : ${lib.makeBinPath deps}:$out/bin
+
+        makeWrapper ${xz}/bin/xz $out/bin/xz
+        makeWrapper ${curl}/bin/curl $out/bin/curl
+      '';
+in
+  mkShell { 
+    name = "head-hackage-build-env"; 
+    buildInputs = [ build-repo ];   
+    cabalDepsSrc = buildDepsFragment;
+  }

ci/discover_tarball.sh

+#! /usr/bin/env bash
+
+set -e
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
+ARCH="$(uname -m)"
+
+>&2 echo "NIGHTLY=${NIGHTLY}"
+>&2 echo "RELEASE_JOB=${RELEASE_JOB}"
+>&2 echo "SLOW_VALIDATE=${SLOW_VALIDATE}"
+>&2 echo "ARCH=${ARCH}"
+
+# NOTE: If you add a new JOB_NAME here then you also might need to modify the
+# `needs` field in ghc/ghc .gitlab-ci.yml to avoid triggering the downstream job
+# too early.
+
+# Before the update to deb12, we use the deb10 bindists
+case $UPSTREAM_BRANCH_NAME in
+  ghc-9.8|ghc-9.10)
+    case $ARCH in
+      aarch64)
+        if [ -n "$NIGHTLY" ]; then
+          BINDIST_NAME="ghc-aarch64-linux-deb10-validate.tar.xz"
+          JOB_NAME="nightly-aarch64-linux-deb10-validate"
+        elif [ "$RELEASE_JOB" == "yes" ]; then
+          BINDIST_NAME="ghc-aarch64-linux-deb10-release+no_split_sections.tar.xz"
+          JOB_NAME="release-aarch64-linux-deb10-release+no_split_sections"
+        else
+          BINDIST_NAME="ghc-aarch64-linux-deb10-validate.tar.xz"
+          JOB_NAME="aarch64-linux-deb10-validate"
+        fi
+      ;;
+      *)
+        if [ -n "$SLOW_VALIDATE" ]; then
+          BINDIST_NAME="ghc-x86_64-linux-deb10-numa-slow-validate.tar.xz"
+          if [ -n "$NIGHTLY" ]; then
+            JOB_NAME="nightly-x86_64-linux-deb10-numa-slow-validate"
+          elif [ "$RELEASE_JOB" == "yes" ]; then
+            echo "No slow validate build in release job"
+            exit 2
+          else
+            JOB_NAME="x86_64-linux-deb10-numa-slow-validate"
+          fi
+        else
+          BINDIST_NAME="ghc-x86_64-linux-fedora33-release.tar.xz"
+          if [ -n "$NIGHTLY" ]; then
+            JOB_NAME="nightly-x86_64-linux-fedora33-release"
+          elif [ "$RELEASE_JOB" == "yes" ]; then
+            JOB_NAME="release-x86_64-linux-fedora33-release"
+          else
+            JOB_NAME="x86_64-linux-fedora33-release"
+          fi
+        fi
+      ;;
+    esac
+    ;;
+  *) # Post update to use deb12
+    case $ARCH in
+      aarch64)
+        if [ -n "$NIGHTLY" ]; then
+          BINDIST_NAME="ghc-aarch64-linux-deb12-validate.tar.xz"
+          JOB_NAME="nightly-aarch64-linux-deb12-validate"
+        elif [ "$RELEASE_JOB" == "yes" ]; then
+          BINDIST_NAME="ghc-aarch64-linux-deb12-release+no_split_sections.tar.xz"
+          JOB_NAME="release-aarch64-linux-deb12-release+no_split_sections"
+        else
+          BINDIST_NAME="ghc-aarch64-linux-deb12-validate.tar.xz"
+          JOB_NAME="aarch64-linux-deb12-validate"
+        fi
+      ;;
+      *)
+        if [ -n "$SLOW_VALIDATE" ]; then
+          BINDIST_NAME="ghc-x86_64-linux-deb12-numa-slow-validate.tar.xz"
+          if [ -n "$NIGHTLY" ]; then
+            JOB_NAME="nightly-x86_64-linux-deb12-numa-slow-validate"
+          elif [ "$RELEASE_JOB" == "yes" ]; then
+            echo "No slow validate build in release job"
+            exit 2
+          else
+            JOB_NAME="x86_64-linux-deb12-numa-slow-validate"
+          fi
+        else
+          BINDIST_NAME="ghc-x86_64-linux-fedora33-release.tar.xz"
+          if [ -n "$NIGHTLY" ]; then
+            JOB_NAME="nightly-x86_64-linux-fedora33-release"
+          elif [ "$RELEASE_JOB" == "yes" ]; then
+            JOB_NAME="release-x86_64-linux-fedora33-release"
+          else
+            JOB_NAME="x86_64-linux-fedora33-release"
+          fi
+        fi
+      ;;
+    esac
+    ;;
+esac
+
+>&2 echo "BINDIST_NAME=${BINDIST_NAME}"
+>&2 echo "JOB_NAME=${JOB_NAME}"
+
+if [ -n "$UPSTREAM_COMMIT_SHA" ]; then
+  # N.B. We can't use this if the upstream pipeline might be in-progress
+  # since the below URL cannot provide an artifact until a pipeline has
+  # run to completion on the requested branch. This is in general
+  # not the case for GHC pipelines. Consequently, in this case we will
+  # usually rather provide UPSTREAM_PIPELINE_ID.
+  >&2 echo "Pulling binary distribution from commit $UPSTREAM_COMMIT_SHA of project $UPSTREAM_PROJECT_PATH..."
+  GHC_TARBALL="https://gitlab.haskell.org/$UPSTREAM_PROJECT_PATH/-/jobs/artifacts/$UPSTREAM_COMMIT_SHA/raw/$BINDIST_NAME?job=$JOB_NAME"
+elif [ -n "$UPSTREAM_PIPELINE_ID" ]; then
+  job_name=$JOB_NAME
+  >&2 echo "Pulling ${job_name} binary distribution from Pipeline $UPSTREAM_PIPELINE_ID..."
+  job_id=$(find-job $UPSTREAM_PROJECT_ID $UPSTREAM_PIPELINE_ID $job_name)
+  >&2 echo "Using job $job_id..."
+  echo "https://gitlab.haskell.org/$UPSTREAM_PROJECT_PATH/-/jobs/$job_id/artifacts/raw/$BINDIST_NAME"
+elif [ -n "$UPSTREAM_BRANCH_NAME" ]; then
+  job_name=$JOB_NAME
+  >&2 echo "Finding ${job_name} binary distribution from $UPSTREAM_BRANCH_NAME..."
+  job_id=$(find-latest-job "$UPSTREAM_PROJECT_ID" "$UPSTREAM_BRANCH_NAME" "$JOB_NAME")
+  >&2 echo "Using job $job_id..."
+  echo "https://gitlab.haskell.org/$UPSTREAM_PROJECT_PATH/-/jobs/$job_id/artifacts/raw/$BINDIST_NAME"
+fi

ci/find-job.sh

+#!/usr/bin/env bash
+
+set -e
+
+project_id=$1
+pipeline_id=$2
+job_name=$3
+
+resp=$(mktemp)
+
+# Access token is a protected environment variable in the head.hackage project and
+# is necessary for this query to succeed. Sadly job tokens only seem to
+# give us access to the project being built.
+curl \
+  --silent --show-error \
+  -H "Private-Token: $ACCESS_TOKEN" \
+  "https://gitlab.haskell.org/api/v4/projects/$project_id/pipelines/$pipeline_id/jobs?scope[]=success&per_page=50" \
+  > "$resp"
+
+job_id=$(jq ". | map(select(.name == \"$job_name\")) | .[0].id" < "$resp")
+if [ "$job_id" = "null" ]; then
+  echo "Error finding job $job_name for $pipeline_id in project $project_id:" >&2
+  cat "$resp" >&2
+  rm "$resp"
+  exit 1
+else
+  rm "$resp"
+  echo -n "$job_id"
+fi

ci/find-latest-job.sh

+#!/usr/bin/env bash
+
+set -e
+
+project_id=$1
+branch_name=$2
+job_name=$3
+
+resp=$(mktemp)
+
+# Access token is a protected environment variable in the head.hackage project and
+# is necessary for this query to succeed. Sadly job tokens only seem to
+# give us access to the project being built.
+curl \
+  --silent --show-error \
+  -H "Private-Token: $ACCESS_TOKEN" \
+  "https://gitlab.haskell.org/api/v4/projects/$project_id/pipelines/?ref=$branch_name&scope=finished&per_page=50" \
+  > "$resp"
+
+job_ids=$(jq ". | map(.id) " < "$resp")
+if [ "$job_ids" = "null" ]; then
+  echo "Error finding job $job_name for $pipeline_id in project $project_id:" >&2
+  cat "$resp" >&2
+  rm "$resp"
+  exit 1
+else
+  for i in $(echo $job_ids | jq '.[]'); do
+    if find-job $project_id $i $job_name; then
+      exit 0
+    fi
+  done
+  echo "Error finding job $job_name for $branch_name project $project_id:" >&2
+  exit 1
+fi

ci/generate-pipeline.sh

+#!/bin/sh
+
+PIPELINE_TYPE=validation
+
+if [ -n "$PIPELINE_OVERRIDE"  ]; then
+    PIPELINE_TYPE="$PIPELINE_OVERRIDE"
+# Triggered by GHC
+elif [ "$CI_PIPELINE_SOURCE" = "pipeline" -a '(' -n "$UPSTREAM_COMMIT_SHA" -o -n "$UPSTREAM_PIPELINE_ID" ')' ]; then
+    PIPELINE_TYPE=downstream
+# Nightly repo update, to be scheduled
+#
+# SCHEDULE_TYPE must be set when creating the scheduled job. It is used to
+# explicitly identify which schedule we want.
+elif [ "$CI_PIPELINE_SOURCE" = "schedule" -a "$SCHEDULE_TYPE" = "update-repo" ]; then
+    PIPELINE_TYPE=update-repo
+fi
+
+mk_pipeline () {
+    echo "Generating $PIPELINE_TYPE pipeline"
+    cp -v $1 gitlab-generated-pipeline.yml
+}
+
+case "$PIPELINE_TYPE" in
+    validation)
+        mk_pipeline ci/pipelines/validation.yml
+        ;;
+    downstream)
+        mk_pipeline ci/pipelines/downstream.yml
+        ;;
+    update-repo)
+        mk_pipeline ci/pipelines/update-repo.yml
+        ;;
+    update-branch)
+        mk_pipeline ci/pipelines/update-branch.yml
+        ;;
+    *)
+        echo "Unknown pipeline type: $PIPELINE_TYPE"
+        exit 1
+        ;;
+esac

ci/ghc-from-artifact.nix

+let
+  sources = import ./nix/sources.nix;
+  nixpkgs = import sources.nixpkgs {};
+in
+
+{ ghcTarball }:
+let
+  nixpkgsSrc = sources.nixpkgs;
+  ghc-artefact-nix =
+    import sources.ghc-artefact-nix {
+      url = ghcTarball;
+      nixpkgsSrc = sources.nixpkgs;
+    };
+in ghc-artefact-nix.ghcHEAD

ci/head-hackage-ci.cabal

+cabal-version:       >=1.10
+name:                head-hackage-ci
+version:             0.1.0.0
+synopsis:            Test driver for head.hackage
+-- description:
+-- bug-reports:
+license:             BSD3
+license-file:        LICENSE
+author:              Ben Gamari
+maintainer:          ben@smart-cactus.org
+copyright:           (c) 2019 Ben Gamari
+build-type:          Simple
+
+executable head-hackage-ci
+  main-is:             Main.hs
+  other-modules:       Types, Utils, TestPatches, MakeConstraints, LatestVersion
+  build-depends:
+    Cabal,
+    aeson,
+    ansi-wl-pprint,
+    async,
+    base >=4.11 && <5,
+    bytestring,
+    cabal-plan,
+    containers,
+    directory,
+    filepath,
+    lens,
+    neat-interpolation,
+    optparse-applicative,
+    stm,
+    temporary,
+    text,
+    typed-process,
+    wreq
+  ghc-options: -threaded
+  default-language:    Haskell2010

ci/logging-ghc

+#!/usr/bin/env bash
+
+# A wrapper for GHC that enables eventlogging when compiling packages with
+# --make.
+
+
+: ${GHC:?GHC must be set}
+: ${EVENTLOG_DIR:?EVENTLOG_DIR must be set}
+
+# Parse --make and unit-id out of args
+#
+# Example:
+# -this-unit-id Cabal-3.0.2.0-b925bb4a908ac87dbee8e61089f84aad30e5791bc2f3e6ffef7bdbb944fda648
+args=( "$@" )
+i=0
+unit_id=
+make_mode=
+while :
+do
+    k="${args[$i]}"
+    v="${args[$i+1]}"
+    if [[ $k == "--make" ]]
+    then
+        make_mode="$k"
+    elif [[ -n $k && -n $v ]]
+    then
+        if [[ $k == "-this-unit-id" ]]
+        then
+            unit_id="$v"
+            break
+        fi
+    else
+        break
+    fi
+    (( i++ ))
+done
+
+# Continue with tracing, if applicable
+if [[ -n $make_mode ]]
+then
+    log_args=${unit_id:+ +RTS -l -ol${EVENTLOG_DIR}/${unit_id}.eventlog -RTS}
+fi
+exec $GHC "$@"$log_args

ci/nix/sources.nix

+# extract sources from the flake.lock using flake-compat
+(import
+  (
+    let lock = builtins.fromJSON (builtins.readFile ../../flake.lock); in
+    fetchTarball {
+      url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz";
+      sha256 = lock.nodes.flake-compat.locked.narHash;
+    }
+  )
+  { src = ../../.; }
+).defaultNix.inputs

ci/pipelines/downstream.yml

+# Defines .run-ci
+include: ci/pipelines/lib/run-head-hackage-ci.yml
+
+# A build triggered from a ghc/ghc> pipeline.
+.downstream:
+  extends: .run-ci
+  parallel:
+    matrix:
+      - ARCH: aarch64
+      - ARCH: x86_64
+  tags:
+    - ${ARCH}-linux
+
+build-downstream:
+  extends: .downstream
+  variables:
+    BUILD_MODE: FULL
+
+test-downstream:
+  extends: .downstream
+  variables:
+    BUILD_MODE: TEST
+  allow_failure: true

ci/pipelines/lib/run-head-hackage-ci.yml

+# Main CI script used in all validation jobs.
+.run-ci:
+  stage: test
+  interruptible: true
+
+  tags:
+    - x86_64-linux
+
+  image: "nixos/nix:$DOCKER_TAG"
+
+  script:
+    - source ci/setup-nix-environment.sh
+    - GHC_TARBALL=$(discover_tarball.sh)
+      # Install GHC
+    - echo "Bindist tarball is $GHC_TARBALL"
+    - curl -L "$GHC_TARBALL" > ghc.tar.xz
+    - |
+      nix build \
+      -f ci/ghc-from-artifact.nix \
+      --arg ghcTarball ./ghc.tar.xz \
+      --out-link ghc
+    - export GHC=`pwd`/ghc/bin/ghc
+    - rm -Rf $HOME/.cabal/packages/local ci/run
+      # Test it
+    - run-ci
+
+  after_script:
+    - |
+      nix --quiet develop -f ci -c \
+      tar -cJf results.tar.xz -C ci/run \
+      results.json logs compiler-info eventlogs
+
+  artifacts:
+    when: always
+    paths:
+    - results.tar.xz

ci/pipelines/update-branch.yml

+# This job is for updating a special branch called "upstream-testing" which is used
+# when we want to trigger jobs to run on head.hackage. This branch is not protected so
+# the permissions about who can trigger jobs on upstream-testing are much more liberal than
+# who can trigger jobs which run on master (which have the side-effect of updating the repo).
+#
+# For testing on topic branches, the operator must pass in
+# PROJECT_ACCESS_TOKEN_OVERRIDE since PROJECT_ACCESS_TOKEN is only available to
+# protected branches.
+#
+# PROJECT_ACCESS_TOKEN is CI/CD variable set at [1]. Its value is a Project
+# Access Token created at [2] with scope write_repository.
+#
+# [1]: https://gitlab.haskell.org/ghc/head.hackage/-/settings/ci_cd
+# [2]: https://gitlab.haskell.org/ghc/head.hackage/-/settings/access_tokens
+update-branch:
+  image: "nixos/nix:$DOCKER_TAG"
+  tags:
+    - x86_64-linux
+  script:
+    - |
+      : ${PROJECT_ACCESS_TOKEN:=${PROJECT_ACCESS_TOKEN_OVERRIDE}}
+      if [ -z "${PROJECT_ACCESS_TOKEN:-}" ]; then
+        >&2 echo "PROJECT_ACCESS_TOKEN must be set."
+        exit 1
+      fi
+      git push "https://gitlab-ci-token:${PROJECT_ACCESS_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git" +refs/remotes/origin/master:refs/heads/upstream-testing -o ci.skip

ci/pipelines/update-repo.yml

+# Build a Hackage repository
+
+stages:
+  - update-repo
+  - deploy
+
+update-repo:
+  stage: update-repo
+
+  tags:
+    - x86_64-linux
+
+  image: "nixos/nix:$DOCKER_TAG"
+
+  variables:
+    KEYS_TARBALL: https://downloads.haskell.org/ghc/head.hackage-keys.tar.enc
+    # KEYS_TARBALL_KEY provided by protected variable
+
+  script:
+    - source ci/setup-nix-environment.sh
+    - build-repo.sh extract-keys
+    - build-repo.sh build-repo
+
+  after_script:
+    - rm -Rf keys
+
+  artifacts:
+    paths:
+      - repo
+
+pages:
+  stage: deploy
+  tags:
+    - x86_64-linux
+  image: alpine:latest
+  variables:
+    # We don't need the repo since we're just moving artifacts around.
+    GIT_STRATEGY: none
+    GIT_SUBMODULE_STRATEGY: none
+  script:
+    - mv repo public
+  needs:
+    - update-repo
+  artifacts:
+    paths:
+      - public

ci/pipelines/validation.yml

+# head.hackage's own validation pipeline.
+
+include:
+  # Defines .run-ci
+  - local: ci/pipelines/lib/run-head-hackage-ci.yml
+
+stages:
+  - update-branch
+  - test
+  - update-repo
+
+# Build against named GHC versions
+.build-with-ghc:
+  extends: .run-ci
+  parallel:
+    matrix:
+      - UPSTREAM_BRANCH_NAME: [ghc-9.8, ghc-9.10, ghc-9.12, ghc-9.14]
+        ARCH: [aarch64, x86_64]
+      - UPSTREAM_BRANCH_NAME: [master]
+        ARCH: [aarch64, x86_64]
+        SLOW_VALIDATE: 1
+  tags:
+    - ${ARCH}-linux
+  variables:
+    EXTRA_HC_OPTS: "-dcore-lint"
+
+build-packages:
+  extends: .build-with-ghc
+  variables:
+    BUILD_MODE: FULL
+
+test-packages:
+  extends: .build-with-ghc
+  variables:
+    BUILD_MODE: TEST
+  allow_failure: true
+
+# Build and deploy a Hackage repository
+update-repo:
+  stage: update-repo
+  needs:
+    - job: build-packages
+      parallel:
+        matrix:
+          - UPSTREAM_BRANCH_NAME: [master]
+            ARCH: [aarch64, x86_64]
+            SLOW_VALIDATE: 1
+      artifacts: false
+  rules:
+    - if: $CI_COMMIT_BRANCH == "master" && $CI_PROJECT_NAMESPACE == "ghc"
+  trigger:
+    include: ci/pipelines/update-repo.yml
+    strategy: depend
+    forward:
+      pipeline_variables: true
+
+# This job is for updating a special branch called "upstream-testing" which is used
+# when we want to trigger jobs to run on head.hackage. This branch is not protected so
+# the permissions about who can trigger jobs on upstream-testing are much more liberal than
+# who can trigger jobs which run on master (which have the side-effect of updating the repo).
+update-branch:
+  stage: update-branch
+  rules:
+    - if: $CI_COMMIT_BRANCH == "master" && $CI_PROJECT_NAMESPACE == "ghc" && $CI_PIPELINE_SOURCE != 'merge_request_event'
+  trigger:
+    include: ci/pipelines/update-branch.yml
+    strategy: depend
+    forward:
+      pipeline_variables: true

ci/setup-nix-environment.sh

+# Bash snippet to be sourced in CI scripts.
+
+cat >/etc/nix/nix.conf <<EOF
+sandbox = false
+build-users-group =
+trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+substituters = https://cache.nixos.org
+experimental-features = nix-command flakes
+EOF
+
+# Equivalent to running `nix develop` but works in CI scripts.
+# We take care to avoid setting the temp dir as this will break unsandboxed builds.
+# See: https://github.com/NixOS/nix/issues/1802
+. <(nix print-dev-env | grep -v "export TE*MP" )

default.nix

-# Build head.hackage packages
-#
-# Usage:
-#   Build using nixpkgs' ghcHEAD:
-#       nix build -f ./.
-#
-#   Build using GHC built from source tree $GHC_TREE:
-#       nix build -f --arg ghc "(import build.nix {ghc-path=$GHC_TREE;})"
-#
-let
-  # To update nixpkgs bump rev and baseNixpkgs's sha256.
-  rev = "2f746bc1fcdc24b103bda3a27ed82c23200cc077";
-  baseNixpkgs =
-    fetchTarball {
-      url = "https://github.com/NixOS/nixpkgs/archive/${rev}.tar.gz";
-      sha256 = "142yl6xakkwmyy8x32h94fci4x09l0yqppq5hwj19a3nmd0zmsjx";
-    };
-in
-
-# ghc: path to a GHC source tree
-{ ghc ? import ./ghc-prerelease.nix
-, haskellOverrides ? (self: super: self)
-}:
-
-let
-  jailbreakOverrides = self: super: {
-    mkDerivation = drv: super.mkDerivation (drv // { jailbreak = true; doCheck = false; });
-  };
-
-  overrides = self: super: rec {
-    all-cabal-hashes = self.fetchurl (import ./all-cabal-hashes.nix);
-
-    # Should this be self?
-    ghcHEAD = ghc super;
-
-    haskellPackages =
-      let patchesOverrides = self.callPackage patches {};
-          patches = self.callPackage (import ./scripts/overrides.nix) { patches = ./patches; };
-          overrides =
-            self.lib.composeExtensions
-              haskellOverrides
-              (self.lib.composeExtensions patchesOverrides jailbreakOverrides);
-
-          baseHaskellPackages = self.callPackage "${baseNixpkgs}/pkgs/development/haskell-modules" rec {
-            haskellLib = import "${baseNixpkgs}/pkgs/development/haskell-modules/lib.nix" {
-              inherit (self) lib;
-              pkgs = self;
-            };
-            buildHaskellPackages = self.buildPackages.haskell.packages.ghc865;
-            ghc = ghcHEAD;
-            compilerConfig = self1: super1: {
-              # Packages included in GHC's global package database
-              Cabal = null;
-              array = null;
-              base = null;
-              binary = null;
-              bytestring = null;
-              containers = null;
-              deepseq = null;
-              directory = null;
-              filepath = null;
-              ghc-boot = null;
-              ghc-boot-th = null;
-              ghc-compact = null;
-              ghc-prim = null;
-              ghci = null;
-              haskeline = null;
-              hpc = null;
-              integer-gmp = null;
-              integer-simple = null;
-              mtl = null;
-              parsec = null;
-              pretty = null;
-              process = null;
-              rts = null;
-              stm = null;
-              template-haskell = null;
-              text = null;
-              time = null;
-              transformers = null;
-              unix = null;
-
-              doctest = haskellPackages.callHackage "doctest" "0.16.0" {};
-              http-api-data = haskellPackages.callPackage ./http-api-data.nix {};
-
-              jailbreak-cabal = buildHaskellPackages.jailbreak-cabal;
-              cabal2nix = buildHaskellPackages.cabal2nix;
-            };
-          };
-      in baseHaskellPackages.extend overrides;
-
-    headHackageScripts = self.stdenv.mkDerivation {
-      name = "head-hackage-scripts";
-      nativeBuildInputs = [ self.makeWrapper ];
-      buildCommand = ''
-        mkdir -p $out/bin
-        makeWrapper ${scripts/patch-tool} $out/bin/patch-tool \
-          --prefix PATH : ${super.haskellPackages.cabal-install}/bin \
-          --prefix PATH : ${self.jq}/bin \
-          --prefix PATH : ${self.curl}/bin
-
-        makeWrapper ${scripts/head.hackage.sh} $out/bin/head.hackage.sh \
-          --set CABAL ${super.haskellPackages.cabal-install}/bin/cabal
-      '';
-    };
-  };
-in import baseNixpkgs { overlays = [ overrides ]; }

flake.lock

+{
+  "nodes": {
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1733328505,
+        "narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "ghc-artefact-nix": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1671037040,
+        "narHash": "sha256-2rPvuEi2V5PD4k6JqbHuhl/ciBrxq50++6XG9NfwVJ0=",
+        "owner": "mpickering",
+        "repo": "ghc-artefact-nix",
+        "rev": "3684936ecde09234f51410e07ccd1c7f48d4f4ac",
+        "type": "github"
+      },
+      "original": {
+        "owner": "mpickering",
+        "repo": "ghc-artefact-nix",
+        "type": "github"
+      }
+    },
+    "hackage-security": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1725697204,
+        "narHash": "sha256-zaLDDeKQpBVaSBRKv/X3WTfeu0rLazy5H8AfJQlsVGM=",
+        "owner": "haskell",
+        "repo": "hackage-security",
+        "rev": "8ada49de74456f3aebd2ae08cf11151acbf8ef05",
+        "type": "github"
+      },
+      "original": {
+        "owner": "haskell",
+        "repo": "hackage-security",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1733392399,
+        "narHash": "sha256-kEsTJTUQfQFIJOcLYFt/RvNxIK653ZkTBIs4DG+cBns=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "d0797a04b81caeae77bcff10a9dde78bc17f5661",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "overlay-tool": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1671663228,
+        "narHash": "sha256-GKeke8FT9w2u6J9aI4eN3mOV0BURHiytQAEir4/4M1Q=",
+        "owner": "bgamari",
+        "repo": "hackage-overlay-repo-tool",
+        "rev": "f60aa0d497961745a503ddea2adb0facb58376e8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "bgamari",
+        "repo": "hackage-overlay-repo-tool",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "ghc-artefact-nix": "ghc-artefact-nix",
+        "hackage-security": "hackage-security",
+        "nixpkgs": "nixpkgs",
+        "overlay-tool": "overlay-tool"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

+{
+  description = "head.hackage";
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+  inputs.ghc-artefact-nix.url = "github:mpickering/ghc-artefact-nix";
+  inputs.ghc-artefact-nix.flake = false;
+  inputs.hackage-security.url = "github:haskell/hackage-security";
+  inputs.hackage-security.flake = false;
+  inputs.overlay-tool.url = "github:bgamari/hackage-overlay-repo-tool";
+  inputs.overlay-tool.flake = false;
+  inputs.flake-compat.url = "github:edolstra/flake-compat";
+  inputs.flake-compat.flake = false;
+  outputs = sources@{nixpkgs, ...}:
+  let
+    env_x86 = import ./ci/default.nix { inherit sources; nixpkgs = nixpkgs.legacyPackages."x86_64-linux"; };
+    env_aarch64 = import ./ci/default.nix { inherit sources; nixpkgs = nixpkgs.legacyPackages."aarch64-linux"; };
+  in {
+    devShells."x86_64-linux".default = env_x86;
+    devShells."aarch64-linux".default = env_aarch64;
+    hydraJobs.env."x86_64-linux" = env_x86;
+    hydraJobs.env."aarch64-linux" = env_aarch64;
+  };
+}