Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • ghc/head.hackage
  • RyanGlScott/head.hackage
  • vaibhavsagar/head.hackage
  • phadej/head.hackage
  • jessoune29/head.hackage
  • alanz/head.hackage
  • clint/head.hackage
  • osa1/head.hackage
  • supersven/head.hackage
  • fendor/head.hackage
  • hsyl20/head.hackage
  • adinapoli/head.hackage
  • alexbiehl/head.hackage
  • mimi.vx/head.hackage
  • Kleidukos/head.hackage
  • wz1000/head.hackage
  • alinab/head.hackage
  • teo/head.hackage
  • duog/head.hackage
  • sheaf/head.hackage
  • expipiplus1/head.hackage
  • drsooch/head.hackage
  • tobias/head.hackage
  • brandonchinn178/head.hackage
  • mpickering/hooks-setup-testing
  • Mikolaj/head.hackage
  • RandomMoonwalker/head.hackage
  • facundominguez/head.hackage
  • trac-fizzixnerd/head.hackage
  • neil.mayhew/head.hackage
  • jappeace/head.hackage
31 results
Show changes
Commits on Source (204)
Showing
with 472 additions and 519 deletions
patches/* -text
......@@ -9,27 +9,28 @@
# To accomplish this we use the ci executable in ./ci. This drives a set of
# cabal v2-build builds and preserves their results.
#
# The compiler to be tested can be taken from a number of sources. The
# build-master and build-9-* jobs form the validation pipeline of the
# head.hackage repository. In addition, other GitLab projects (e.g. ghc/ghc>)
# The execution flow looks something like:
#
# - Gitlab runner
# - (nix run)
# - run-ci
# - ./run-ci (the Nix package just wraps the script)
# - (nix run) (when USE_NIX=1)
# - head-hackage-ci $EXTRA_OPTS (a Cabal project in ci/)
# - ci/Main.hs
# - TestPatches.testPatches <$> TestPatches.config
# - option '--test-package'
# - <something similar for building the packages>
#
# EXTRA_OPTS are injected into the execution flow inside ./run-ci, which in turn
# sources them from ci/config.sh.
#
# The compiler to be tested can be taken from a number of sources.
# head.hackage's own validation pipeline runs against GHC HEAD and the three
# supported major versions. In addition, other GitLab projects (e.g. ghc/ghc>)
# can trigger a multi-project pipeline, specifying a GHC binary distribution
# via either the GHC_TARBALL or UPSTREAM_* variables.
#
# Prevent duplicate workflows
workflow:
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
- if: $CI_PIPELINE_SOURCE == "pipeline" # Triggered by GHC
- if: $CI_PIPELINE_SOURCE == "scheduled"
- if: $CI_PIPELINE_SOURCE == "web"
- if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
when: never
- if: $CI_COMMIT_BRANCH
stages:
- test
- update-repo
- deploy
variables:
# Which nixos/nix Docker image tag to use
......@@ -54,7 +55,7 @@ variables:
# Multi-project pipeline variables:
#
# These are set by the "upstream" pipeline for `build-pipeline` pipelines:
# These are set by the "upstream" pipeline for downstream pipelines:
#
# UPSTREAM_PROJECT_PATH: The path of the upstream project (e.g. `ghc/ghc`)
# UPSTREAM_PIPELINE_ID: The ID of the upstream pipeline
......@@ -64,282 +65,31 @@ variables:
# UPSTREAM_COMMIT_SHA: The ref or commit SHA of the GHC build to be tested
#
# We explictly set the locale to avoid happy chocking up on UTF-8 source code. See #31
# We explictly set the locale to avoid happy choking up on UTF-8 source code. See #31
LANG: "C.UTF-8"
stages:
- generate
- dispatch
# A build triggered from a ghc/ghc> pipeline.
.build-pipeline:
extends: .build
before_script:
- !reference [.build, before_script]
- GHC_TARBALL=$(discover_tarball.sh)
parallel:
matrix:
- ARCH: aarch64
- ARCH: x86_64
tags:
- ${ARCH}-linux
rules:
- if: '$UPSTREAM_COMMIT_SHA || $UPSTREAM_PIPELINE_ID'
when: always
- when: never
build-pipeline:
extends: .build-pipeline
variables:
BUILD_MODE: FULL
test-pipeline:
extends: .build-pipeline
variables:
BUILD_MODE: TEST
allow_failure: true
# Build against the master branch
.build-master:
extends: .build
before_script:
- !reference [.build, before_script]
- GHC_TARBALL=$(discover_tarball.sh)
variables:
UPSTREAM_BRANCH_NAME: master
EXTRA_HC_OPTS: "-dcore-lint"
SLOW_VALIDATE: 1
parallel:
matrix:
- ARCH: aarch64
- ARCH: x86_64
tags:
- ${ARCH}-linux
rules:
- if: '$UPSTREAM_COMMIT_SHA || $UPSTREAM_PIPELINE_ID'
when: never
- when: always
build-master:
extends: .build-master
variables:
BUILD_MODE: FULL
test-master:
extends: .build-master
variables:
BUILD_MODE: TEST
allow_failure: true
# Build against the 9.4 branch
.build-9.4:
extends: .build
before_script:
- !reference [.build, before_script]
- GHC_TARBALL=$(discover_tarball.sh)
variables:
UPSTREAM_BRANCH_NAME: ghc-9.4
EXTRA_HC_OPTS: "-dcore-lint"
rules:
- if: '$UPSTREAM_COMMIT_SHA || $UPSTREAM_PIPELINE_ID'
when: never
- when: always
build-9.4:
extends: .build-9.4
variables:
BUILD_MODE: FULL
test-9.4:
extends: .build-9.4
variables:
BUILD_MODE: TEST
allow_failure: true
# Build against the 9.6 branch
.build-9.6:
extends: .build
before_script:
- !reference [.build, before_script]
- GHC_TARBALL=$(discover_tarball.sh)
parallel:
matrix:
- ARCH: aarch64
- ARCH: x86_64
tags:
- ${ARCH}-linux
variables:
UPSTREAM_BRANCH_NAME: ghc-9.6
EXTRA_HC_OPTS: "-dcore-lint"
rules:
- if: '$UPSTREAM_COMMIT_SHA || $UPSTREAM_PIPELINE_ID'
when: never
- when: always
build-9.6:
extends: .build-9.6
variables:
BUILD_MODE: FULL
test-9.6:
extends: .build-9.6
variables:
BUILD_MODE: TEST
allow_failure: true
# Build against the 9.8 branch
.build-9.8:
extends: .build
before_script:
- !reference [.build, before_script]
- GHC_TARBALL=$(discover_tarball.sh)
parallel:
matrix:
- ARCH: aarch64
- ARCH: x86_64
tags:
- ${ARCH}-linux
variables:
UPSTREAM_BRANCH_NAME: ghc-9.8
EXTRA_HC_OPTS: "-dcore-lint"
rules:
- if: '$UPSTREAM_COMMIT_SHA || $UPSTREAM_PIPELINE_ID'
when: never
- when: always
build-9.8:
extends: .build-9.8
variables:
BUILD_MODE: FULL
test-9.8:
extends: .build-9.8
variables:
BUILD_MODE: TEST
allow_failure: true
.build:
stage: test
interruptible: true
tags:
- x86_64-linux
image: "nixos/nix:$DOCKER_TAG"
before_script:
- |
cat >/etc/nix/nix.conf <<EOF
sandbox = false
build-users-group = nixbld
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= loony-tools:pr9m4BkM/5/eSTZlkQyRt57Jz7OMBxNSUiMC4FkcNfk=
substituters = https://cache.nixos.org https://cache.zw3rk.com
experimental-features = nix-command flakes
cores = 1 # to avoid resource exhaustion. See: <https://gitlab.haskell.org/ghc/head.hackage/-/issues/38>
EOF
# Equivalent to running `nix develop` but works in CI scripts.
# We take care to avoid setting the temp dir as this will break unsandboxed builds.
# See: <https://github.com/NixOS/nix/issues/1802>
- . <(nix print-dev-env | grep -v "export TE*MP" )
script:
# Install GHC
- echo "Bindist tarball is $GHC_TARBALL"
- curl -L "$GHC_TARBALL" > ghc.tar.xz
- |
nix build \
-f ci/ghc-from-artifact.nix \
--arg ghcTarball ./ghc.tar.xz \
--out-link ghc
- export GHC=`pwd`/ghc/bin/ghc
- rm -Rf $HOME/.cabal/packages/local ci/run
# Test it
- run-ci
after_script:
- ls -lh
- |
nix develop -f ci -c \
tar -cJf results.tar.xz -C ci/run \
results.json logs compiler-info eventlogs
artifacts:
when: always
paths:
- results.tar.xz
# Build and deploy a Hackage repository
update-repo:
stage: update-repo
before_script:
- |
cat >/etc/nix/nix.conf <<EOF
sandbox = false
build-users-group = nixbld
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= loony-tools:pr9m4BkM/5/eSTZlkQyRt57Jz7OMBxNSUiMC4FkcNfk=
substituters = https://cache.nixos.org https://cache.zw3rk.com
experimental-features = nix-command flakes
cores = 1 # to avoid resource exhaustion. See: <https://gitlab.haskell.org/ghc/head.hackage/-/issues/38>
EOF
# Equivalent to running `nix develop` but works in CI scripts.
# We take care to avoid setting the temp dir as this will break unsandboxed builds.
# See: <https://github.com/NixOS/nix/issues/1802>
- . <(nix print-dev-env | grep -v "export TE*MP" )
tags:
- x86_64-linux
image: "nixos/nix:$DOCKER_TAG"
generate-pipeline:
variables:
KEYS_TARBALL: https://downloads.haskell.org/ghc/head.hackage-keys.tar.enc
# KEYS_TARBALL_KEY provided by protected variable
rules:
- if: '$CI_COMMIT_BRANCH == "master"'
script:
- build-repo.sh extract-keys
- build-repo.sh build-repo
needs:
- job: build-master
artifacts: false
after_script:
- rm -Rf keys
GIT_SUBMODULE_STRATEGY: none
image: alpine:latest
tags: [x86_64-linux]
stage: generate
script: ./ci/generate-pipeline.sh
artifacts:
paths:
- repo
pages:
stage: deploy
tags:
- x86_64-linux
image: "nixos/nix:$DOCKER_TAG"
script:
- mv repo public
needs:
- update-repo
rules:
- if: '$CI_COMMIT_BRANCH == "master"'
artifacts:
paths:
- public
# This job is for updating a special branch called "upstream-testing" which is used
# when we want to trigger jobs to run on head.hackage. This branch is not protected so
# the permissions about who can trigger jobs on upstream-testing are much more liberal than
# who can trigger jobs which run on master (which have the side-effect of updating the repo).
update-branch:
stage: test
image: "nixos/nix:$DOCKER_TAG"
tags:
- x86_64-linux
script:
- git remote -v
- git reset --hard origin/master
- git push "https://gitlab-ci-token:$PROJECT_ACCESS_TOKEN@$CI_SERVER_HOST/$CI_PROJECT_PATH.git" +HEAD:upstream-testing -o ci.skip
rules:
- if: $CI_COMMIT_BRANCH == "master"
- gitlab-generated-pipeline.yml
run-pipeline:
stage: dispatch
trigger:
strategy: depend
forward:
pipeline_variables: true
include:
- artifact: gitlab-generated-pipeline.yml
job: generate-pipeline
......@@ -10,7 +10,7 @@ existing Hackage package(s).
you submit a PR).
- The patches SHOULD work with at least GHC HEAD and a set of recent stable
released GHC versions (currently this means with GHC 9.4, 9.6, 9.8, and 9.9).
released GHC versions (currently this means with GHC 9.6, 9.8, 9.10, 9.12 and 9.13).
- The patches SHOULD ideally result in the same code being compiled,
as one of the main purposes of these patches is to make regression
......@@ -22,6 +22,12 @@ existing Hackage package(s).
This repo contains `<pkg-id>.patch` files in the
[`patches/`](./patches/) folder (where `<pkg-id>` refers to a specific
release of a package, e.g. `lens-4.15.3`).
Adding a patch forces the system to use that specific version,
so empty patch files may exist to force the system to use that a
newer version, instead of a previous patch if available.
For example consider a patched `th-abstraction-0.5.0`, and an empty patch `th-abstraction-0.6.0`,
if we were to remove the empty patch, `0.6.0`, certain libraries such
as `generics-sop` fail to build, because it's forced to use `0.5.0`.
Once merged to `master`, all package releases whose `<pkg-id>` is
mentioned will enter the *HEAD.hackage* package index; if there is a
......@@ -67,7 +73,7 @@ active-repositories: hackage.haskell.org, head.hackage.ghc.haskell.org
```
Also see
https://cabal.readthedocs.io/en/latest/cabal-project.html#cfg-field-active-repositories.
https://cabal.readthedocs.io/en/3.12/cabal-project-description-file.html#cfg-field-active-repositories.
`HEAD.hackage` doesn't bump the bounds of boot packages + certain other packages to avoid the busywork of bumping them. When using `HEAD.hackage`, you should use `--allow-newer` for these packages. The full list is [here](https://gitlab.haskell.org/ghc/head.hackage/-/blob/90570e1c4606c1d7d3d41797ec1b32d1b984067b/ci/MakeConstraints.hs#L40-49).
......@@ -144,6 +150,17 @@ The `scripts/patch-tool` script is a tool for conveniently authoring and updatin
1. `scripts/patch-tool update-patches`
1. Commit the patch
When contributing a patch, one needs to be mindful of [Hackage revisions].
head.hackage doesn't combine patches with the revisions of a package. Instead,
a patch is applied on the unrevised package (also called revision 0). This
implies that when contributing patches, it might be necessary to additionally
include the changes that are already in some revision. Moreover, this also
implies that if a patch only contains changes that are already present in
revisions, then contributing the patch to head.hackage is useless as the changes
are already available for building.
[Hackage revisions]: https://github.com/haskell-infra/hackage-trustees/blob/master/revisions-information.md
### GitLab CI
GHC's GitLab instance uses GitLab CI and the `head-hackage-ci` tool (contained
......
......@@ -45,8 +45,11 @@ allowNewerPkgs = S.fromList
, "binary"
, "bytestring"
, "Cabal"
, "containers"
, "deepseq"
, "text" ] `S.union` bootPkgs
, "text"
, "ghc-boot"
, "ghc-boot-th" ] `S.union` bootPkgs
constraints :: [String] -> Doc
constraints constraints =
......
......@@ -77,7 +77,7 @@ The below is all orchestrated by `run-ci.sh`:
the outcome of the build
1. Write a JSON report (of type `Types.RunResult ()`) to `result.json`
1. Examine the failed units and determine whether there were any unexpected failures.
1. Examine the failed packages and determine whether there were any unexpected failures.
### Build plans and empty patches
......
......@@ -11,7 +11,7 @@ module TestPatches
import Control.Monad
import Data.Foldable
import Data.List (intercalate)
import Data.List (intercalate, partition)
import Data.Maybe
import Data.Text (Text)
import GHC.Generics
......@@ -245,25 +245,28 @@ resultSummary broken runResult = (ok, msg)
&& null failedTestsBuild
&& null failedUnits
msg = vcat
[ "Total units built:" <+> pshow (length allUnits)
[ "Total packages built:" <+> pshow (length allUnits)
, ""
, pshow (length expectedPlanningErrs) <+> "had no valid install plan (expected):"
, PP.indent 4 $ vcat $ map (uncurry prettyPkgVer) expectedPlanningErrs
, ""
, pshow (length planningErrs) <+> "had no valid install plan:"
, PP.indent 4 $ vcat $ map (uncurry prettyPkgVer) planningErrs
, ""
, pshow (length failedUnits) <+> "units failed to build:"
, pshow (length failedUnits) <+> "packages failed to build:"
, PP.indent 4 $ vcat
[ prettyPkgVer (pkgName binfo) (version binfo)
| (binfo, _) <- M.elems failedUnits ]
, pshow (length expectedFailedUnits) <+> "units failed to build (expected):"
, pshow (length expectedFailedUnits) <+> "packages failed to build (expected):"
, PP.indent 4 $ vcat
[ prettyPkgVer (pkgName binfo) (version binfo)
| (binfo, _) <- M.elems expectedFailedUnits ]
, pshow (length failedTargetUnits) <+> "target units failed to build:"
, pshow (length failedTargetUnits) <+> "target packages failed to build:"
, PP.indent 4 $ vcat
[ prettyPkgVer pkg ver
| (pkg, ver) <- failedTargetUnits ]
, ""
, pshow (length failedDependsUnits) <+> "units failed to build due to unbuildable dependencies."
, pshow (length failedDependsUnits) <+> "packages failed to build due to unbuildable dependencies."
, ""
, pshow (length failedTestsBuild) <+> "testsuites failed build."
, PP.indent 4 $ vcat
......@@ -273,7 +276,8 @@ resultSummary broken runResult = (ok, msg)
[ prettyPkgName pkg_name | pkg_name <- failedTests ]
]
allUnits = runResultUnits runResult
planningErrs = planningErrors runResult
(expectedPlanningErrs, planningErrs) =
partition (failureExpected broken . fst) (planningErrors runResult)
failedTests = [ pkg_name | (TestedPatch pkg_name ver (PackageResult (PackageBuildSucceeded PackageTestsFailed) _)) <- testedTests runResult ]
......
......@@ -173,7 +173,14 @@ build_repo() {
cabal update
cabal fetch acme-box-0.0.0.0
mkdir -p repo/package
cp $HOME/.cabal/packages/hackage.haskell.org/acme-box/0.0.0.0/acme-box-0.0.0.0.tar.gz repo/package
# if ~/.cabal exists cabal-install will use that, otherwise packages go into $XDG_CACHE_HOME/cabal
if [ -d "$HOME/.cabal" ]; then
cp "$HOME/.cabal/packages/hackage.haskell.org/acme-box/0.0.0.0/acme-box-0.0.0.0.tar.gz" repo/package
else
cp "${XDG_CACHE_HOME:-$HOME/.cache}/cabal/packages/hackage.haskell.org/acme-box/0.0.0.0/acme-box-0.0.0.0.tar.gz" repo/package
fi
log "Bootstrapping repository..."
hackage-repo-tool bootstrap --keys=./keys --repo=./repo
......
......@@ -3,7 +3,7 @@
-- ghc/ghc#23048.
index-state:
hackage.haskell.org 2023-07-31T12:18:14Z,
hackage.haskell.org 2025-05-11T00:00:00Z,
head.hackage HEAD
constraints: th-abstraction >= 0.4
......
......@@ -88,23 +88,38 @@ commit="$(ghc_commit)"
arch="$(ghc_arch)"
echo "Found GHC $version, commit $commit."
case $version in
9.4.*)
9.6.*)
# package ticket
broken linear-generics 22546
broken servant-conduit 22915
broken servant-machines 22913
broken liquidhaskell-boot 350
# singletons-base only supports the latest ghc
broken singletons-base 00000
;;
9.6.*)
9.8.*)
# package ticket
broken liquidhaskell-boot 350
# singletons-base only supports the latest ghc
broken singletons-base 00000
;;
9.8.*)
9.10.*)
# package ticket
broken liquidhaskell-boot 350
# singletons-base only supports the latest ghc
broken singletons-base 00000
;;
9.9.*)
9.12.*)
# package ticket
broken liquidhaskell-boot 350
;;
9.13.*)
# package ticket
broken ghcide 00000
broken Agda 26154
# broken until strict-containers is updated to work with the new containers
broken strict-containers 00000
;;
*)
......@@ -133,29 +148,31 @@ esac
# ==============
#
# These are packages which we don't have patches for but want to test anyways.
extra_package lens 5.2.2
extra_package lens 5.2.3
extra_package generic-lens 2.2.2.0
extra_package optics 0.4.2.1
extra_package aeson 2.2.0.0
extra_package criterion 1.6.2.0
extra_package scotty 0.12.1
extra_package aeson 2.2.3.0
extra_package criterion 1.6.3.0
extra_package scotty 0.21
extra_package generic-lens 2.2.2.0
extra_package microstache 1.0.2.3
extra_package singletons-base 3.1.1
extra_package servant 0.20
extra_package singletons-base 3.5
extra_package servant 0.20.1
extra_package hgmp 0.1.2.1
extra_package Agda 2.6.3
extra_package Agda 2.7.0.1
extra_package mmark 0.0.7.6
extra_package doctest 0.22.0
extra_package tasty 1.4.3
extra_package pandoc 3.1.5
extra_package doctest 0.24.0
extra_package tasty 1.5.3
extra_package pandoc 3.1.11.1
extra_package servant-conduit 0.16
extra_package servant-machines 0.16
extra_package linear-generics 0.2.2
extra_package futhark 0.25.2
extra_package linear-generics 0.2.3
extra_package futhark 0.25.13
extra_package generic-random 1.5.0.1
extra_package lame 0.2.1
extra_package inspection-testing 0.5.0.2
extra_package ghcide 2.0.0.1
extra_package lame 0.2.2
extra_package inspection-testing 0.5.0.3
extra_package ghcide 2.9.0.0
extra_package ghc-typelits-extra 0.4.7
# This package is affected by https://gitlab.haskell.org/ghc/ghc/-/issues/22912
extra_package vector-space 0.16
......@@ -201,7 +218,7 @@ case "$BUILD_MODE" in
only_package microlens
only_package free
only_package optparse-applicative
test_package system-test "$(pwd)/../tests/ghc-debug/test/"
test_package system-test "$(pwd)/../tests/ghc-debug/**/*.cabal"
test_package ghc-tests "$(pwd)/../tests/ghc-tests"
;;
TEST)
......@@ -209,14 +226,14 @@ case "$BUILD_MODE" in
# packages from being built. Morally, I really want to say "build
# nothing at all besides the tests".
only_package tasty
test_package system-test "$(pwd)/../tests/ghc-debug/test/"
test_package system-test "$(pwd)/../tests/ghc-debug/**/*.cabal"
test_package ghc-tests "$(pwd)/../tests/ghc-tests"
test_package all "$(pwd)/../tests/text"
test_package bytestring-tests "$(pwd)/../tests/bytestring"
test_package all "$(pwd)/../tests/containers/containers-tests"
;;
COMPAT)
test_package system-test "$(pwd)/../tests/ghc-debug/test/"
test_package system-test "$(pwd)/../tests/ghc-debug/**/*.cabal"
test_package ghc-tests "$(pwd)/../tests/ghc-tests"
;;
esac
......@@ -2,18 +2,7 @@
with nixpkgs;
let
haskellPackages = nixpkgs.haskell.packages.ghc925.override {
all-cabal-hashes = sources.all-cabal-hashes.outPath;
overrides = self: super: {
mkDerivation = args: super.mkDerivation (args // {
enableLibraryProfiling = false;
doCheck = false;
doHoogle = false;
doHaddock = false;
});
};
};
haskellPackages = nixpkgs.haskellPackages;
hackage-repo-tool =
let src = sources.hackage-security.outPath;
......@@ -53,7 +42,7 @@ let
let
deps = [
bash curl gnutar findutils patch rsync openssl
cabal-install haskellPackages.ghc gcc binutils-unwrapped pwgen gnused
haskellPackages.cabal-install haskellPackages.ghc gcc binutils-unwrapped pwgen gnused
hackage-repo-tool overlay-tool python3 jq pkg-config
git # cabal-install wants this to fetch source-repository-packages
];
......
......@@ -15,41 +15,84 @@ ARCH="$(uname -m)"
# `needs` field in ghc/ghc .gitlab-ci.yml to avoid triggering the downstream job
# too early.
case $ARCH in
aarch64)
if [ -n "$NIGHTLY" ]; then
BINDIST_NAME="ghc-aarch64-linux-deb10-validate.tar.xz"
JOB_NAME="nightly-aarch64-linux-deb10-validate"
elif [ "$RELEASE_JOB" == "yes" ]; then
BINDIST_NAME="ghc-aarch64-linux-deb10-release+no_split_sections.tar.xz"
JOB_NAME="release-aarch64-linux-deb10-release+no_split_sections"
else
BINDIST_NAME="ghc-aarch64-linux-deb10-validate.tar.xz"
JOB_NAME="aarch64-linux-deb10-validate"
fi
;;
*)
if [ -n "$SLOW_VALIDATE" ]; then
BINDIST_NAME="ghc-x86_64-linux-deb10-numa-slow-validate.tar.xz"
if [ -n "$NIGHTLY" ]; then
JOB_NAME="nightly-x86_64-linux-deb10-numa-slow-validate"
elif [ "$RELEASE_JOB" == "yes" ]; then
echo "No slow validate build in release job"
exit 2
else
JOB_NAME="x86_64-linux-deb10-numa-slow-validate"
fi
else
BINDIST_NAME="ghc-x86_64-linux-fedora33-release.tar.xz"
if [ -n "$NIGHTLY" ]; then
JOB_NAME="nightly-x86_64-linux-fedora33-release"
elif [ "$RELEASE_JOB" == "yes" ]; then
JOB_NAME="release-x86_64-linux-fedora33-release"
else
JOB_NAME="x86_64-linux-fedora33-release"
fi
fi
;;
# Before the update to deb12, we use the deb10 bindists
case $UPSTREAM_BRANCH_NAME in
ghc-9.6|ghc-9.8|ghc-9.10)
case $ARCH in
aarch64)
if [ -n "$NIGHTLY" ]; then
BINDIST_NAME="ghc-aarch64-linux-deb10-validate.tar.xz"
JOB_NAME="nightly-aarch64-linux-deb10-validate"
elif [ "$RELEASE_JOB" == "yes" ]; then
BINDIST_NAME="ghc-aarch64-linux-deb10-release+no_split_sections.tar.xz"
JOB_NAME="release-aarch64-linux-deb10-release+no_split_sections"
else
BINDIST_NAME="ghc-aarch64-linux-deb10-validate.tar.xz"
JOB_NAME="aarch64-linux-deb10-validate"
fi
;;
*)
if [ -n "$SLOW_VALIDATE" ]; then
BINDIST_NAME="ghc-x86_64-linux-deb10-numa-slow-validate.tar.xz"
if [ -n "$NIGHTLY" ]; then
JOB_NAME="nightly-x86_64-linux-deb10-numa-slow-validate"
elif [ "$RELEASE_JOB" == "yes" ]; then
echo "No slow validate build in release job"
exit 2
else
JOB_NAME="x86_64-linux-deb10-numa-slow-validate"
fi
else
BINDIST_NAME="ghc-x86_64-linux-fedora33-release.tar.xz"
if [ -n "$NIGHTLY" ]; then
JOB_NAME="nightly-x86_64-linux-fedora33-release"
elif [ "$RELEASE_JOB" == "yes" ]; then
JOB_NAME="release-x86_64-linux-fedora33-release"
else
JOB_NAME="x86_64-linux-fedora33-release"
fi
fi
;;
esac
;;
*) # Post update to use deb12
case $ARCH in
aarch64)
if [ -n "$NIGHTLY" ]; then
BINDIST_NAME="ghc-aarch64-linux-deb12-validate.tar.xz"
JOB_NAME="nightly-aarch64-linux-deb12-validate"
elif [ "$RELEASE_JOB" == "yes" ]; then
BINDIST_NAME="ghc-aarch64-linux-deb12-release+no_split_sections.tar.xz"
JOB_NAME="release-aarch64-linux-deb12-release+no_split_sections"
else
BINDIST_NAME="ghc-aarch64-linux-deb12-validate.tar.xz"
JOB_NAME="aarch64-linux-deb12-validate"
fi
;;
*)
if [ -n "$SLOW_VALIDATE" ]; then
BINDIST_NAME="ghc-x86_64-linux-deb12-numa-slow-validate.tar.xz"
if [ -n "$NIGHTLY" ]; then
JOB_NAME="nightly-x86_64-linux-deb12-numa-slow-validate"
elif [ "$RELEASE_JOB" == "yes" ]; then
echo "No slow validate build in release job"
exit 2
else
JOB_NAME="x86_64-linux-deb12-numa-slow-validate"
fi
else
BINDIST_NAME="ghc-x86_64-linux-fedora33-release.tar.xz"
if [ -n "$NIGHTLY" ]; then
JOB_NAME="nightly-x86_64-linux-fedora33-release"
elif [ "$RELEASE_JOB" == "yes" ]; then
JOB_NAME="release-x86_64-linux-fedora33-release"
else
JOB_NAME="x86_64-linux-fedora33-release"
fi
fi
;;
esac
;;
esac
>&2 echo "BINDIST_NAME=${BINDIST_NAME}"
......
#!/bin/sh
PIPELINE_TYPE=validation
if [ -n "$PIPELINE_OVERRIDE" ]; then
PIPELINE_TYPE="$PIPELINE_OVERRIDE"
# Triggered by GHC
elif [ "$CI_PIPELINE_SOURCE" = "pipeline" -a '(' -n "$UPSTREAM_COMMIT_SHA" -o -n "$UPSTREAM_PIPELINE_ID" ')' ]; then
PIPELINE_TYPE=downstream
# Nightly repo update, to be scheduled
#
# SCHEDULE_TYPE must be set when creating the scheduled job. It is used to
# explicitly identify which schedule we want.
elif [ "$CI_PIPELINE_SOURCE" = "schedule" -a "$SCHEDULE_TYPE" = "update-repo" ]; then
PIPELINE_TYPE=update-repo
fi
mk_pipeline () {
echo "Generating $PIPELINE_TYPE pipeline"
cp -v $1 gitlab-generated-pipeline.yml
}
case "$PIPELINE_TYPE" in
validation)
mk_pipeline ci/pipelines/validation.yml
;;
downstream)
mk_pipeline ci/pipelines/downstream.yml
;;
update-repo)
mk_pipeline ci/pipelines/update-repo.yml
;;
update-branch)
mk_pipeline ci/pipelines/update-branch.yml
;;
*)
echo "Unknown pipeline type: $PIPELINE_TYPE"
exit 1
;;
esac
# Defines .run-ci
include: ci/pipelines/lib/run-head-hackage-ci.yml
# A build triggered from a ghc/ghc> pipeline.
.downstream:
extends: .run-ci
parallel:
matrix:
- ARCH: aarch64
- ARCH: x86_64
tags:
- ${ARCH}-linux
build-downstream:
extends: .downstream
variables:
BUILD_MODE: FULL
test-downstream:
extends: .downstream
variables:
BUILD_MODE: TEST
allow_failure: true
# Main CI script used in all validation jobs.
.run-ci:
stage: test
interruptible: true
tags:
- x86_64-linux
image: "nixos/nix:$DOCKER_TAG"
script:
- source ci/setup-nix-environment.sh
- GHC_TARBALL=$(discover_tarball.sh)
# Install GHC
- echo "Bindist tarball is $GHC_TARBALL"
- curl -L "$GHC_TARBALL" > ghc.tar.xz
- |
nix build \
-f ci/ghc-from-artifact.nix \
--arg ghcTarball ./ghc.tar.xz \
--out-link ghc
- export GHC=`pwd`/ghc/bin/ghc
- rm -Rf $HOME/.cabal/packages/local ci/run
# Test it
- run-ci
after_script:
- |
nix --quiet develop -f ci -c \
tar -cJf results.tar.xz -C ci/run \
results.json logs compiler-info eventlogs
artifacts:
when: always
paths:
- results.tar.xz
# This job is for updating a special branch called "upstream-testing" which is used
# when we want to trigger jobs to run on head.hackage. This branch is not protected so
# the permissions about who can trigger jobs on upstream-testing are much more liberal than
# who can trigger jobs which run on master (which have the side-effect of updating the repo).
#
# For testing on topic branches, the operator must pass in
# PROJECT_ACCESS_TOKEN_OVERRIDE since PROJECT_ACCESS_TOKEN is only available to
# protected branches.
#
# PROJECT_ACCESS_TOKEN is CI/CD variable set at [1]. Its value is a Project
# Access Token created at [2] with scope write_repository.
#
# [1]: https://gitlab.haskell.org/ghc/head.hackage/-/settings/ci_cd
# [2]: https://gitlab.haskell.org/ghc/head.hackage/-/settings/access_tokens
update-branch:
image: "nixos/nix:$DOCKER_TAG"
tags:
- x86_64-linux
script:
- |
: ${PROJECT_ACCESS_TOKEN:=${PROJECT_ACCESS_TOKEN_OVERRIDE}}
if [ -z "${PROJECT_ACCESS_TOKEN:-}" ]; then
>&2 echo "PROJECT_ACCESS_TOKEN must be set."
exit 1
fi
git push "https://gitlab-ci-token:${PROJECT_ACCESS_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git" +refs/remotes/origin/master:refs/heads/upstream-testing -o ci.skip
# Build a Hackage repository
stages:
- update-repo
- deploy
update-repo:
stage: update-repo
tags:
- x86_64-linux
image: "nixos/nix:$DOCKER_TAG"
variables:
KEYS_TARBALL: https://downloads.haskell.org/ghc/head.hackage-keys.tar.enc
# KEYS_TARBALL_KEY provided by protected variable
script:
- source ci/setup-nix-environment.sh
- build-repo.sh extract-keys
- build-repo.sh build-repo
after_script:
- rm -Rf keys
artifacts:
paths:
- repo
pages:
stage: deploy
tags:
- x86_64-linux
image: alpine:latest
variables:
# We don't need the repo since we're just moving artifacts around.
GIT_STRATEGY: none
GIT_SUBMODULE_STRATEGY: none
script:
- mv repo public
needs:
- update-repo
artifacts:
paths:
- public
# head.hackage's own validation pipeline.
include:
# Defines .run-ci
- local: ci/pipelines/lib/run-head-hackage-ci.yml
stages:
- update-branch
- test
- update-repo
# Build against named GHC versions
.build-with-ghc:
extends: .run-ci
parallel:
matrix:
- UPSTREAM_BRANCH_NAME: [ghc-9.6, ghc-9.8, ghc-9.10, ghc-9.12]
ARCH: [aarch64, x86_64]
- UPSTREAM_BRANCH_NAME: [master]
ARCH: [aarch64, x86_64]
SLOW_VALIDATE: 1
tags:
- ${ARCH}-linux
variables:
EXTRA_HC_OPTS: "-dcore-lint"
build-packages:
extends: .build-with-ghc
variables:
BUILD_MODE: FULL
test-packages:
extends: .build-with-ghc
variables:
BUILD_MODE: TEST
allow_failure: true
# Build and deploy a Hackage repository
update-repo:
stage: update-repo
needs:
- job: build-packages
parallel:
matrix:
- UPSTREAM_BRANCH_NAME: [master]
ARCH: [aarch64, x86_64]
SLOW_VALIDATE: 1
artifacts: false
rules:
- if: $CI_COMMIT_BRANCH == "master" && $CI_PROJECT_NAMESPACE == "ghc"
trigger:
include: ci/pipelines/update-repo.yml
strategy: depend
forward:
pipeline_variables: true
# This job is for updating a special branch called "upstream-testing" which is used
# when we want to trigger jobs to run on head.hackage. This branch is not protected so
# the permissions about who can trigger jobs on upstream-testing are much more liberal than
# who can trigger jobs which run on master (which have the side-effect of updating the repo).
update-branch:
stage: update-branch
rules:
- if: $CI_COMMIT_BRANCH == "master" && $CI_PROJECT_NAMESPACE == "ghc" && $CI_PIPELINE_SOURCE != 'merge_request_event'
trigger:
include: ci/pipelines/update-branch.yml
strategy: depend
forward:
pipeline_variables: true
# Bash snippet to be sourced in CI scripts.
cat >/etc/nix/nix.conf <<EOF
sandbox = false
build-users-group =
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
substituters = https://cache.nixos.org
experimental-features = nix-command flakes
EOF
# Equivalent to running `nix develop` but works in CI scripts.
# We take care to avoid setting the temp dir as this will break unsandboxed builds.
# See: https://github.com/NixOS/nix/issues/1802
. <(nix print-dev-env | grep -v "export TE*MP" )
# Build head.hackage packages
#
# Usage:
# Build using nixpkgs' ghcHEAD:
# nix build -f ./.
#
# Build using GHC built from source tree $GHC_TREE:
# nix build -f --arg ghc "(import build.nix {ghc-path=$GHC_TREE;})"
#
let
# To update nixpkgs bump rev and baseNixpkgs's sha256.
rev = "33e0d99cbedf2acfd7340d2150837fbb28039a64";
baseNixpkgs =
fetchTarball {
url = "https://github.com/NixOS/nixpkgs/archive/${rev}.tar.gz";
sha256 = "042yl6xakkwmyy8x32h94fci4x09l0yqppq5hwj19a3nmd0zmsjx";
};
in
# ghc: path to a GHC source tree
{ ghc ? import ./ghc-prerelease.nix
, haskellOverrides ? (self: super: self)
}:
let
jailbreakOverrides = self: super: {
mkDerivation = drv: super.mkDerivation (drv // { jailbreak = true; doCheck = false; });
};
overrides = self: super: rec {
# Use scripts/update-cabal-hashes.sh to update this
all-cabal-hashes =
let
versions = builtins.fromJSON (builtins.readFile ./scripts/cabal-hashes.json);
fetch = { owner, repo, rev, sha256, ... }: self.fetchurl {
inherit sha256;
url = "https://github.com/${owner}/${repo}/archive/${rev}.tar.gz";
};
in fetch versions;
# Should this be self?
ghcHEAD = ghc super;
haskellPackages =
let patchesOverrides = self.callPackage patches {};
patches = self.callPackage (import ./scripts/overrides.nix) { patches = ./patches; };
overrides =
self.lib.composeExtensions
haskellOverrides
(self.lib.composeExtensions patchesOverrides jailbreakOverrides);
baseHaskellPackages = self.callPackage "${baseNixpkgs}/pkgs/development/haskell-modules" rec {
haskellLib = import "${baseNixpkgs}/pkgs/development/haskell-modules/lib.nix" {
inherit (self) lib;
pkgs = self;
};
buildHaskellPackages = self.buildPackages.haskell.packages.ghc865;
ghc = ghcHEAD;
compilerConfig = self1: super1: {
# Packages included in GHC's global package database
Cabal = null;
array = null;
base = null;
binary = null;
bytestring = null;
containers = null;
deepseq = null;
directory = null;
filepath = null;
ghc-boot = null;
ghc-boot-th = null;
ghc-compact = null;
ghc-prim = null;
ghci = null;
haskeline = null;
hpc = null;
integer-gmp = null;
integer-simple = null;
mtl = null;
parsec = null;
pretty = null;
process = null;
rts = null;
stm = null;
template-haskell = null;
text = null;
time = null;
transformers = null;
unix = null;
doctest = haskellPackages.callHackage "doctest" "0.16.0" {};
http-api-data = haskellPackages.callPackage ./http-api-data.nix {};
jailbreak-cabal = buildHaskellPackages.jailbreak-cabal;
cabal2nix = buildHaskellPackages.cabal2nix;
};
};
in baseHaskellPackages.extend overrides;
headHackageScripts = self.stdenv.mkDerivation {
name = "head-hackage-scripts";
nativeBuildInputs = [ self.makeWrapper ];
buildCommand = ''
mkdir -p $out/bin
makeWrapper ${scripts/patch-tool} $out/bin/patch-tool \
--prefix PATH : ${super.haskellPackages.cabal-install}/bin \
--prefix PATH : ${self.jq}/bin \
--prefix PATH : ${self.curl}/bin
makeWrapper ${scripts/head.hackage.sh} $out/bin/head.hackage.sh \
--set CABAL ${super.haskellPackages.cabal-install}/bin/cabal
'';
};
};
in import baseNixpkgs { overlays = [ overrides ]; }
{
"nodes": {
"all-cabal-hashes": {
"flake": false,
"locked": {
"lastModified": 1494499108,
"narHash": "sha256-vD3K/chhEGRJwSZy3WPnKnVtY+SrgypAzMkOUy8ubu8=",
"owner": "commercialhaskell",
"repo": "all-cabal-hashes",
"rev": "56a8e992cb300266eb6d1a181884bd765309d9be",
"type": "github"
},
"original": {
"owner": "commercialhaskell",
"repo": "all-cabal-hashes",
"type": "github"
}
},
"flake-compat": {
"flake": false,
"locked": {
"lastModified": 1673956053,
"narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
"lastModified": 1733328505,
"narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
"rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec",
"type": "github"
},
"original": {
......@@ -51,11 +35,11 @@
"hackage-security": {
"flake": false,
"locked": {
"lastModified": 1668891326,
"narHash": "sha256-3Oeaq4ZhHe8USes4BjIyk5VWzRTT7itvyvMVyBbj81M=",
"lastModified": 1725697204,
"narHash": "sha256-zaLDDeKQpBVaSBRKv/X3WTfeu0rLazy5H8AfJQlsVGM=",
"owner": "haskell",
"repo": "hackage-security",
"rev": "c37c91c48e0d21b587360534da8a852c5cdf4a74",
"rev": "8ada49de74456f3aebd2ae08cf11151acbf8ef05",
"type": "github"
},
"original": {
......@@ -66,17 +50,17 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1670841420,
"narHash": "sha256-mSEia1FzrsHbfqjorMyYiX8NXdDVeR1Pw1k55jMJlJY=",
"lastModified": 1733392399,
"narHash": "sha256-kEsTJTUQfQFIJOcLYFt/RvNxIK653ZkTBIs4DG+cBns=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "33e0d99cbedf2acfd7340d2150837fbb28039a64",
"rev": "d0797a04b81caeae77bcff10a9dde78bc17f5661",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-unstable",
"repo": "nixpkgs",
"rev": "33e0d99cbedf2acfd7340d2150837fbb28039a64",
"type": "github"
}
},
......@@ -98,7 +82,6 @@
},
"root": {
"inputs": {
"all-cabal-hashes": "all-cabal-hashes",
"flake-compat": "flake-compat",
"ghc-artefact-nix": "ghc-artefact-nix",
"hackage-security": "hackage-security",
......