.ghcid

@@ -2,4 +2,4 @@
 --reload compiler
 --reload ghc
 --reload includes
---restart hadrian/
+--restart hadrian/ghci

.gitignore

@@ -97,6 +97,7 @@ _darcs/
 # -----------------------------------------------------------------------------
 # specific generated files
 
+/.gitlab/jobs-metadata.json
 /bindist-list
 /bindist-list.uniq
 /bindistprep/
@@ -110,11 +111,13 @@ _darcs/
 /compiler/ClosureTypes.h
 /compiler/FunTypes.h
 /compiler/MachRegs.h
+/compiler/MachRegs
 /compiler/ghc-llvm-version.h
 /compiler/ghc.cabal
 /compiler/ghc.cabal.old
 /distrib/configure.ac
 /distrib/ghc.iss
+/docs/index.html
 /docs/man
 /docs/users_guide/.log
 /docs/users_guide/users_guide
@@ -181,8 +184,8 @@ _darcs/
 /linter.log
 /mk/are-validating.mk
 /mk/build.mk
-/mk/config.h
-/mk/config.h.in
+/mk/unused.h
+/mk/unused.h.in
 /mk/config.mk
 /mk/config.mk.old
 /mk/system-cxx-std-lib-1.0.conf
@@ -238,6 +241,7 @@ ghc.nix/
 
 # direnv
 .envrc
+.direnv
 
 # Visual Studio Code
 .vscode

.gitlab-ci.yml

@@ -2,11 +2,11 @@ variables:
   GIT_SSL_NO_VERIFY: "1"
 
   # Commit of ghc/ci-images repository from which to pull Docker images
-  DOCKER_REV: dd01591a50ea4e2aa3c106cf50ca54d38663f912
+  DOCKER_REV: a55d4ae0f9da0a2fb3bc72a13f356e2511a4c4fd
 
   # Sequential version number of all cached things.
   # Bump to invalidate GitLab CI cache.
-  CACHE_REV: 9
+  CACHE_REV: 10
 
   # Disable shallow clones; they break our linting rules
   GIT_DEPTH: 0
@@ -57,35 +57,56 @@ stages:
 # Note [The CI Story]
 # ~~~~~~~~~~~~~~~~~~~
 #
-# There are two different types of pipelines:
+# There are a few different types of pipelines. Among them:
 #
-#  - marge-bot merges to `master`. Here we perform an exhaustive validation
+# 1. marge-bot merges to `master`. Here we perform an exhaustive validation
 #    across all of the platforms which we support. In addition, we push
 #    performance metric notes upstream, providing a persistent record of the
 #    performance characteristics of the compiler.
 #
-#  - merge requests. Here we perform a slightly less exhaustive battery of
+# 2. merge requests. Here we perform a slightly less exhaustive battery of
 #    testing. Namely we omit some configurations (e.g. the unregisterised job).
 #    These use the merge request's base commit for performance metric
 #    comparisons.
 #
-
+# These and other pipelines are defined implicitly by the rules of individual
+# jobs.
+#
+# At the top level, however, we can declare that pipelines (of whatever type)
+# only run when:
+#
+# 1. Processing a merge request (as mentioned above)
+#
+# 2. Processing a tag
+#
+# 3. Pushing to master on the root ghc/ghc repo (as mentioned above)
+#
+# 4. Pushing to a release branch on the root ghc/ghc repo
+#
+# 5. Somebody manually triggers a pipeline from the GitLab UI
+#
+# In particular, note that pipelines don't automatically run just when changes
+# are pushed to a feature branch.
 workflow:
-  # N.B. Don't run on wip/ branches, instead on run on merge requests.
   rules:
     - if: $CI_MERGE_REQUEST_ID
     - if: $CI_COMMIT_TAG
-    - if: '$CI_COMMIT_BRANCH == "master"'
-    - if: '$CI_COMMIT_BRANCH =~ /ghc-[0-9]+\.[0-9]+/'
+    # N.B.: If we weren't explicit about CI_PROJECT_ID, the following rule would
+    # cause a duplicate pipeline for merge requests coming from the master
+    # branch of a fork.
+    - if: $CI_PROJECT_ID == "1" && $CI_COMMIT_BRANCH == "master"
+    - if: $CI_PROJECT_ID == "1" && $CI_COMMIT_BRANCH =~ /ghc-[0-9]+\.[0-9]+/
     - if: '$CI_PIPELINE_SOURCE == "web"'
 
 # which versions of GHC to allow bootstrap with
 .bootstrap_matrix : &bootstrap_matrix
   matrix:
-    - GHC_VERSION: 9.2.5
-      DOCKER_IMAGE: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb10-ghc9_2:$DOCKER_REV"
-    - GHC_VERSION: 9.4.3
-      DOCKER_IMAGE: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb10:$DOCKER_REV"
+    - GHC_VERSION: 9.4.7
+      DOCKER_IMAGE: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb10-ghc9_4:$DOCKER_REV"
+    - GHC_VERSION: 9.6.3
+      DOCKER_IMAGE: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb10-ghc9_6:$DOCKER_REV"
+    - GHC_VERSION: 9.8.1
+      DOCKER_IMAGE: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb10-ghc9_8:$DOCKER_REV"
 
 # Allow linters to fail on draft MRs.
 # This must be explicitly transcluded in lint jobs which
@@ -123,6 +144,12 @@ workflow:
   rules:
     - if: '$RELEASE_JOB == "yes"'
 
+.full-ci: &full-ci
+  - if: '$CI_MERGE_REQUEST_LABELS =~ /.*full-ci.*/'
+  - if: '$CI_MERGE_REQUEST_LABELS =~ /.*marge_bot_batch_merge_job.*/'
+  - if: '$CI_COMMIT_BRANCH == "master"'
+  - if: '$CI_COMMIT_BRANCH =~ /ghc-[0-9]+\.[0-9]+/'
+
 ############################################################
 # Runner Tags
 ############################################################
@@ -264,7 +291,7 @@ lint-author:
     - *drafts-can-fail-lint
 
 lint-ci-config:
-  image: nixos/nix:2.12.0
+  image: nixos/nix:2.14.1
   extends: .lint
   # We don't need history/submodules in this job
   variables:
@@ -272,11 +299,17 @@ lint-ci-config:
     GIT_SUBMODULE_STRATEGY: none
   before_script:
     - echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf
-    - nix-channel --update
   script:
-    - .gitlab/generate_jobs
+    - nix run .gitlab/generate-ci#generate-jobs
     # 1 if .gitlab/generate_jobs changed the output of the generated config
     - nix shell nixpkgs#git -c git diff --exit-code
+    # And run this to generate the .gitlab/jobs-metadata.json
+    - nix run .gitlab/generate-ci#generate-job-metadata
+  artifacts:
+    when: always
+    paths:
+      - .gitlab/jobs-metadata.json
+      - .gitlab/jobs.yaml
   dependencies: []
 
 lint-submods:
@@ -329,7 +362,8 @@ lint-submods-branch:
     paths:
       - cabal-cache
 
-hlint-ghc-and-base:
+# Disabled due to #22830
+.hlint-ghc-and-base:
   extends: .lint-params
   image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
   variables:
@@ -337,6 +371,8 @@ hlint-ghc-and-base:
   script:
     - .gitlab/ci.sh setup
     - .gitlab/ci.sh configure
+    - .gitlab/ci.sh run_hadrian lint:ghc-internal
+    - .gitlab/ci.sh run_hadrian lint:ghc-experimental
     - .gitlab/ci.sh run_hadrian lint:base
     - .gitlab/ci.sh run_hadrian lint:compiler
 
@@ -362,6 +398,9 @@ hadrian-ghc-in-ghci:
     - git clean -xdf && git submodule foreach git clean -xdf
     - .gitlab/ci.sh setup
     - .gitlab/ci.sh configure
+    # Enable -Werror when building hadrian
+    - "echo 'package hadrian'  > hadrian/cabal.project.local"
+    - "echo '  ghc-options: -Werror'  >> hadrian/cabal.project.local"
     # Load ghc-in-ghci then immediately exit and check the modules loaded
     - echo ":q" | hadrian/ghci -j`mk/detect-cpu-count.sh`| tail -n2 | grep "Ok,"
   after_script:
@@ -391,7 +430,7 @@ hadrian-multi:
     # workaround for docker permissions
     - sudo chown ghc:ghc -R .
   variables:
-    GHC_FLAGS: -Werror
+    GHC_FLAGS: "-Werror -Wwarn=deprecations"
     CONFIGURE_ARGS: --enable-bootstrap-with-devel-snapshot
   tags:
     - x86_64-linux
@@ -423,7 +462,7 @@ hadrian-multi:
     paths:
       - cabal-cache
   rules:
-    - if: '$CI_MERGE_REQUEST_LABELS !~ /.*fast-ci.*/'
+    - *full-ci
 
 ############################################################
 # stack-hadrian-build
@@ -453,6 +492,7 @@ test-cabal-reinstall-x86_64-linux-deb10:
     TEST_ENV: "x86_64-linux-deb10-cabal-install"
   rules:
     - if: $NIGHTLY
+    - if: '$CI_MERGE_REQUEST_LABELS =~ /.*test-reinstall.*/'
 
 ########################################
 # Testing ABI is invariant across builds
@@ -478,6 +518,9 @@ abi-test-nightly:
     paths:
       - out
   rules:
+    # This job is broken. Disabling it until some kind soul can finish its
+    # implementation. #23269
+    - when: never
     - if: $NIGHTLY
 
 ############################################################
@@ -498,7 +541,7 @@ doc-tarball:
       optional: true
     - job: nightly-x86_64-windows-validate
       optional: true
-    - job: release-x86_64-windows-release+no_split_sections
+    - job: release-x86_64-windows-release
       optional: true
 
   tags:
@@ -522,7 +565,7 @@ doc-tarball:
         || mv "ghc-x86_64-linux-deb10-release.tar.xz" "$LINUX_BINDIST" \
         || true
       mv "ghc-x86_64-windows-validate.tar.xz" "$WINDOWS_BINDIST" \
-        || mv "ghc-x86_64-windows-release+no_split_sections.tar.xz" "$WINDOWS_BINDIST" \
+        || mv "ghc-x86_64-windows-release.tar.xz" "$WINDOWS_BINDIST" \
         || true
       if [ ! -f "$LINUX_BINDIST" ]; then
         echo "Error: $LINUX_BINDIST does not exist. Did the Debian 9 job fail?"
@@ -590,6 +633,7 @@ source-tarball:
     - if: $NIGHTLY
     - if: '$RELEASE_JOB == "yes"'
     - if: '$CI_MERGE_REQUEST_LABELS =~ /.*test-bootstrap.*/'
+    - *full-ci
 
 generate-hadrian-bootstrap-sources:
   stage: full-build
@@ -608,6 +652,7 @@ generate-hadrian-bootstrap-sources:
     - if: $NIGHTLY
     - if: '$RELEASE_JOB == "yes"'
     - if: '$CI_MERGE_REQUEST_LABELS =~ /.*test-bootstrap.*/'
+    - *full-ci
 
 
 package-hadrian-bootstrap-sources:
@@ -625,6 +670,7 @@ package-hadrian-bootstrap-sources:
     - if: $NIGHTLY
     - if: '$RELEASE_JOB == "yes"'
     - if: '$CI_MERGE_REQUEST_LABELS =~ /.*test-bootstrap.*/'
+    - *full-ci
 
 test-bootstrap:
   stage: full-build
@@ -661,6 +707,7 @@ test-bootstrap:
   rules:
     - if: $NIGHTLY
     - if: '$CI_MERGE_REQUEST_LABELS =~ /.*test-bootstrap.*/'
+    - *full-ci
     - if: '$RELEASE_JOB == "yes"'
       when: always
       variables:
@@ -709,6 +756,12 @@ hackage-lint:
     - job: nightly-x86_64-linux-deb10-numa-slow-validate
       optional: true
       artifacts: false
+    - job: nightly-aarch64-linux-deb10-validate
+      optional: true
+      artifacts: false
+    - job: aarch64-linux-deb10-validate
+      optional: true
+      artifacts: false
   extends: .hackage
   variables:
     SLOW_VALIDATE: 1
@@ -724,6 +777,9 @@ hackage-label-lint:
     - job: x86_64-linux-deb10-numa-slow-validate
       optional: true
       artifacts: false
+    - job: aarch64-linux-deb10-validate
+      optional: true
+      artifacts: false
   extends: .hackage
   variables:
     SLOW_VALIDATE: 1
@@ -738,6 +794,9 @@ nightly-hackage-lint:
     - job: nightly-x86_64-linux-deb10-numa-slow-validate
       optional: true
       artifacts: false
+    - job: nightly-aarch64-linux-deb10-validate
+      optional: true
+      artifacts: false
   rules:
     - if: $NIGHTLY
       variables:
@@ -752,6 +811,9 @@ nightly-hackage-perf:
     - job: nightly-x86_64-linux-fedora33-release
       optional: true
       artifacts: false
+    - job: nightly-aarch64-linux-deb10-validate
+      optional: true
+      artifacts: false
   rules:
     - if: $NIGHTLY
       variables:
@@ -768,6 +830,9 @@ release-hackage-lint:
     - job: release-x86_64-linux-fedora33-release
       optional: true
       artifacts: false
+    - job: release-aarch64-linux-deb10-release+no_split_sections
+      optional: true
+      artifacts: false
   rules:
     - if: '$RELEASE_JOB == "yes"'
   extends: .hackage
@@ -775,6 +840,59 @@ release-hackage-lint:
     # No slow-validate bindist on release pipeline
     EXTRA_HC_OPTS: "-dlint"
 
+############################################################
+# Testing via test-primops
+############################################################
+
+# Triggering jobs in the ghc/test-primops project
+
+.test-primops:
+  stage: testing
+  variables:
+    UPSTREAM_PROJECT_PATH: "$CI_PROJECT_PATH"
+    UPSTREAM_PROJECT_ID: "$CI_PROJECT_ID"
+    UPSTREAM_PIPELINE_ID: "$CI_PIPELINE_ID"
+  trigger:
+    project: "ghc/test-primops"
+    branch: "upstream-testing"
+    strategy: "depend"
+
+.test-primops-validate-template:
+  needs:
+    - job: x86_64-linux-deb10-validate+debug_info
+      artifacts: false
+    - job: aarch64-linux-deb10-validate
+      artifacts: false
+    - job: aarch64-darwin-validate
+      artifacts: false
+    - job: x86_64-darwin-validate
+      artifacts: false
+  extends: .test-primops
+
+test-primops-label:
+  extends: .test-primops-validate-template
+  rules:
+    - if: '$CI_MERGE_REQUEST_LABELS =~ /.*test-primops.*/'
+
+test-primops-nightly:
+  extends: .test-primops
+  needs:
+    - job: nightly-x86_64-linux-deb10-validate
+      artifacts: false
+    - job: nightly-aarch64-linux-deb10-validate
+      artifacts: false
+    - job: nightly-aarch64-darwin-validate
+      artifacts: false
+    - job: nightly-x86_64-darwin-validate
+      artifacts: false
+  rules:
+    - if: $NIGHTLY
+
+test-primops-release:
+  extends: .test-primops
+  rules:
+    - if: '$RELEASE_JOB == "yes"'
+
 ############################################################
 # Nofib testing
 # (Disabled: See #21859)
@@ -796,10 +914,7 @@ perf-nofib:
   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-fedora33:$DOCKER_REV"
   rules:
     - when: never
-    - if: $CI_MERGE_REQUEST_ID
-    - if: '$CI_COMMIT_BRANCH == "master"'
-    - if: '$CI_COMMIT_BRANCH =~ /ghc-[0.9]+\.[0-9]+/'
-    - if: '$CI_MERGE_REQUEST_LABELS !~ /.*fast-ci.*/'
+    - *full-ci
   tags:
     - x86_64-linux
   before_script:
@@ -841,10 +956,6 @@ perf:
       optional: true
   dependencies: null
   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-fedora33:$DOCKER_REV"
-  rules:
-    - if: $CI_MERGE_REQUEST_ID
-    - if: '$CI_COMMIT_BRANCH == "master"'
-    - if: '$CI_COMMIT_BRANCH =~ /ghc-[0.9]+\.[0-9]+/'
   tags:
     - x86_64-linux-perf
   script:
@@ -866,7 +977,7 @@ perf:
     paths:
       - out
   rules:
-    - if: '$CI_MERGE_REQUEST_LABELS !~ /.*fast-ci.*/'
+    - *full-ci
 
 ############################################################
 # ABI testing
@@ -906,7 +1017,7 @@ abi-test:
     paths:
       - out
   rules:
-    - if: '$CI_MERGE_REQUEST_LABELS !~ /.*fast-ci.*/'
+    - *full-ci
 
 
 ############################################################
@@ -950,9 +1061,6 @@ pages:
 #############################################################
 
 
-# TODO: MP: This way of determining the project version is sadly very slow.
-# It seems overkill to have to setup a complete environment, and build hadrian to get
-# it to generate a single file containing the version information.
 project-version:
   stage: packaging
   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb10:$DOCKER_REV"
@@ -965,7 +1073,6 @@ project-version:
     - sudo chown ghc:ghc -R .
     - .gitlab/ci.sh setup
     - .gitlab/ci.sh configure
-    - .gitlab/ci.sh run_hadrian VERSION
     - echo "ProjectVersion=$(cat VERSION)" > version.sh
 
   needs: []
@@ -973,13 +1080,10 @@ project-version:
   artifacts:
     paths:
       - version.sh
-  rules:
-    - if: '$NIGHTLY'
-    - if: '$RELEASE_JOB == "yes"'
 
 .ghcup-metadata:
   stage: deploy
-  image: "nixos/nix:2.12.0"
+  image: nixos/nix:2.14.1
   dependencies: null
   tags:
     - x86_64-linux
@@ -993,10 +1097,11 @@ project-version:
     # Calculate the project version
     - . ./version.sh
 
-    # Download existing ghcup metadata
-    - nix shell --extra-experimental-features nix-command --extra-experimental-features flakes nixpkgs#wget -c wget "https://raw.githubusercontent.com/haskell/ghcup-metadata/develop/ghcup-0.0.7.yaml"
+    # Download existing ghcup metadata for the correct year
+    - PipelineYear="$(date -d $CI_PIPELINE_CREATED_AT +%Y)"
+    - nix shell nixpkgs#wget -c wget "https://ghc.gitlab.haskell.org/ghcup-metadata/ghcup-nightlies-$PipelineYear-0.0.7.yaml" -O ghcup-0.0.7.yaml
 
-    - .gitlab/generate_job_metadata
+    - nix run .gitlab/generate-ci#generate-job-metadata
 
   artifacts:
     paths:
@@ -1011,17 +1116,23 @@ ghcup-metadata-nightly:
       artifacts: false
     - job: nightly-x86_64-linux-centos7-validate
       artifacts: false
+    - job: nightly-x86_64-linux-ubuntu20_04-validate
+      artifacts: false
+    - job: nightly-x86_64-linux-ubuntu18_04-validate
+      artifacts: false
+    - job: nightly-x86_64-linux-rocky8-validate
+      artifacts: false
     - job: nightly-x86_64-darwin-validate
       artifacts: false
     - job: nightly-aarch64-darwin-validate
       artifacts: false
     - job: nightly-x86_64-windows-validate
       artifacts: false
-    - job: nightly-x86_64-linux-alpine3_12-int_native-validate+fully_static
+    - job: nightly-x86_64-linux-alpine3_12-validate
       artifacts: false
     - job: nightly-x86_64-linux-deb9-validate
       artifacts: false
-    - job: nightly-i386-linux-deb9-validate
+    - job: nightly-i386-linux-deb10-validate
       artifacts: false
     - job: nightly-x86_64-linux-deb10-validate
       artifacts: false
@@ -1033,15 +1144,45 @@ ghcup-metadata-nightly:
       artifacts: false
     - job: project-version
   script:
-    - nix shell --extra-experimental-features nix-command -f .gitlab/rel_eng -c ghcup-metadata --metadata ghcup-0.0.7.yaml --pipeline-id="$CI_PIPELINE_ID" --version="$ProjectVersion" > "metadata_test.yaml"
+    - nix shell --extra-experimental-features nix-command -f .gitlab/rel_eng -c ghcup-metadata --metadata ghcup-0.0.7.yaml --date="$(date -d $CI_PIPELINE_CREATED_AT +%Y-%m-%d)" --pipeline-id="$CI_PIPELINE_ID" --version="$ProjectVersion" > "metadata_test.yaml"
   rules:
     - if: $NIGHTLY
 
+# Update the ghcup metadata with information about this nightly pipeline
+ghcup-metadata-nightly-push:
+  stage: deploy
+  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-fedora33:$DOCKER_REV"
+  dependencies: null
+  tags:
+    - x86_64-linux
+  variables:
+    BUILD_FLAVOUR: default
+    GIT_SUBMODULE_STRATEGY: "none"
+  needs:
+    - job: ghcup-metadata-nightly
+      artifacts: true
+  script:
+    - git clone https://gitlab.haskell.org/ghc/ghcup-metadata.git
+    - PipelineYear="$(date -d $CI_PIPELINE_CREATED_AT +%Y)"
+    - cp metadata_test.yaml "ghcup-metadata/ghcup-nightlies-$PipelineYear-0.0.7.yaml"
+    - cp metadata_test.yaml "ghcup-metadata/ghcup-nightlies-0.0.7.yaml"
+    - cd ghcup-metadata
+    - git config user.email "ghc-ci@gitlab-haskell.org"
+    - git config user.name "GHC GitLab CI"
+    - git remote add gitlab_origin https://oauth2:$PROJECT_PUSH_TOKEN@gitlab.haskell.org/ghc/ghcup-metadata.git
+    - git add .
+    - git commit -m "Update metadata"
+    - git push gitlab_origin HEAD:updates
+  rules:
+    # Only run the update on scheduled nightly pipelines, ie once a day
+    - if: $NIGHTLY && $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "master"
+
+
 ghcup-metadata-release:
   # No explicit needs for release pipeline as we assume we need everything and everything will pass.
   extends: .ghcup-metadata
   script:
-    - nix shell --extra-experimental-features nix-command -f .gitlab/rel_eng -c ghcup-metadata --release-mode --metadata ghcup-0.0.7.yaml --pipeline-id="$CI_PIPELINE_ID" --version="$ProjectVersion" > "metadata_test.yaml"
+    - nix shell --extra-experimental-features nix-command -f .gitlab/rel_eng -c ghcup-metadata --release-mode --metadata ghcup-0.0.7.yaml --date="$(date -d $CI_PIPELINE_CREATED_AT +%Y-%m-%d)" --pipeline-id="$CI_PIPELINE_ID" --version="$ProjectVersion" > "metadata_test.yaml"
   rules:
     - if: '$RELEASE_JOB == "yes"'
 
@@ -1078,4 +1219,3 @@ ghcup-metadata-testing-release:
   rules:
     - if: '$RELEASE_JOB == "yes"'
   when: manual
-

.gitlab/ci.sh

@@ -7,7 +7,7 @@
 set -Eeuo pipefail
 
 # Configuration:
-HACKAGE_INDEX_STATE="2020-12-21T14:48:20Z"
+HACKAGE_INDEX_STATE="2023-10-05T11:38:51Z"
 MIN_HAPPY_VERSION="1.20"
 MIN_ALEX_VERSION="3.2.6"
 
@@ -43,12 +43,13 @@ $0 - GHC continuous integration driver
 
 Common Modes:
 
-  usage         Show this usage message.
-  setup         Prepare environment for a build.
-  configure     Run ./configure.
-  clean         Clean the tree
-  shell         Run an interactive shell with a configured build environment.
-  save_cache    Preserve the cabal cache
+  usage             Show this usage message.
+  setup             Prepare environment for a build.
+  configure         Run ./configure.
+  clean             Clean the tree
+  shell             Run an interactive shell with a configured build environment.
+  save_test_output  Generate unexpected-test-output.tar.gz
+  save_cache        Preserve the cabal cache
 
 Hadrian build system
   build_hadrian Build GHC via the Hadrian build system
@@ -74,16 +75,6 @@ Environment variables affecting both build systems:
                     (either "x86-64-darwin" or "aarch-darwin")
   NO_BOOT           Whether to run ./boot or not, used when testing the source dist
 
-Environment variables determining build configuration of Make system:
-
-  BUILD_FLAVOUR     Which flavour to build.
-  BUILD_SPHINX_HTML Whether to build Sphinx HTML documentation.
-  BUILD_SPHINX_PDF  Whether to build Sphinx PDF documentation.
-  INTEGER_LIBRARY   Which integer library to use (integer-simple or integer-gmp).
-  HADDOCK_HYPERLINKED_SOURCES
-                    Whether to build hyperlinked Haddock sources.
-  TEST_TYPE         Which test rule to run.
-
 Environment variables determining build configuration of Hadrian system:
 
   BUILD_FLAVOUR     Which flavour to build.
@@ -210,21 +201,18 @@ function set_toolchain_paths() {
           *) fail "unknown NIX_SYSTEM" ;;
         esac
         info "Building toolchain for $NIX_SYSTEM"
-        nix-build .gitlab/darwin/toolchain.nix --argstr system "$NIX_SYSTEM" -o toolchain.sh
-        cat toolchain.sh
+        nix-build --quiet .gitlab/darwin/toolchain.nix --argstr system "$NIX_SYSTEM" -o toolchain.sh
       fi
       source toolchain.sh
-      info "--info for GHC for $NIX_SYSTEM"
-      $GHC --info
       ;;
     env)
       # These are generally set by the Docker image but
       # we provide these handy fallbacks in case the
       # script isn't run from within a GHC CI docker image.
-      if [ -z "$GHC" ]; then GHC="$(which ghc)"; fi
-      if [ -z "$CABAL" ]; then CABAL="$(which cabal)"; fi
-      if [ -z "$HAPPY" ]; then HAPPY="$(which happy)"; fi
-      if [ -z "$ALEX" ]; then ALEX="$(which alex)"; fi
+      : ${GHC:=$(which ghc)}
+      : ${CABAL:=$(which cabal)}
+      : ${HAPPY:=$(which happy)}
+      : ${ALEX:=$(which alex)}
       ;;
     *) fail "bad toolchain_source"
   esac
@@ -242,7 +230,7 @@ function set_toolchain_paths() {
 function cabal_update() {
   # In principle -w shouldn't be necessary here but with
   # cabal-install 3.8.1.0 it is, due to cabal#8447.
-  run "$CABAL" update -w "$GHC" --index="$HACKAGE_INDEX_STATE"
+  run "$CABAL" update -w "$GHC" "hackage.haskell.org,${HACKAGE_INDEX_STATE}"
 }
 
 
@@ -274,6 +262,11 @@ function setup() {
   show_tool CABAL
   show_tool HAPPY
   show_tool ALEX
+
+  info "====================================================="
+  info "ghc --info"
+  info "====================================================="
+  $GHC --info
 }
 
 function fetch_ghc() {
@@ -387,26 +380,6 @@ function cleanup_submodules() {
   end_section "clean submodules"
 }
 
-function prepare_build_mk() {
-  if [[ -z "$BUILD_FLAVOUR" ]]; then fail "BUILD_FLAVOUR is not set"; fi
-  if [[ -z ${BUILD_SPHINX_HTML:-} ]]; then BUILD_SPHINX_HTML=YES; fi
-  if [[ -z ${BUILD_SPHINX_PDF:-} ]]; then BUILD_SPHINX_PDF=YES; fi
-
-  cat > mk/build.mk <<EOF
-BIGNUM_BACKEND=${BIGNUM_BACKEND}
-include mk/flavours/${BUILD_FLAVOUR}.mk
-GhcLibHcOpts+=-haddock
-EOF
-
-  if [ -n "${HADDOCK_HYPERLINKED_SOURCES:-}" ]; then
-    echo "EXTRA_HADDOCK_OPTS += --hyperlinked-source --quickjump" >> mk/build.mk
-  fi
-
-
-  info "build.mk is:"
-  cat mk/build.mk
-}
-
 function configure() {
   case "${CONFIGURE_WRAPPER:-}" in
     emconfigure) source "$EMSDK/emsdk_env.sh" ;;
@@ -452,7 +425,7 @@ function push_perf_notes() {
     return
   fi
 
-  if [[ -n "${CROSS_TARGET:-}" ]]; then
+  if [ -n "${CROSS_TARGET:-}" ] && [ "${CROSS_EMULATOR:-}" != "js-emulator" ]; then
     info "Can't test cross-compiled build."
     return
   fi
@@ -507,18 +480,29 @@ function build_hadrian() {
 
   check_release_build
 
+  # Just to be sure, use the same hackage index state when building Hadrian.
+  echo "index-state: $HACKAGE_INDEX_STATE" > hadrian/cabal.project.local
+
   # We can safely enable parallel compression for x64. By the time
   # hadrian calls tar/xz to produce bindist, there's no other build
   # work taking place.
   if [[ "${CI_JOB_NAME:-}" != *"i386"* ]]; then
-    XZ_OPT="${XZ_OPT:-} -T$cores"
+    export XZ_OPT="${XZ_OPT:-} -T$cores"
   fi
 
   if [[ -n "${REINSTALL_GHC:-}" ]]; then
     run_hadrian build-cabal -V
   else
-    run_hadrian test:all_deps binary-dist -V
-    mv _build/bindist/ghc*.tar.xz "$BIN_DIST_NAME.tar.xz"
+    case "$(uname)" in
+        MSYS_*|MINGW*)
+          run_hadrian test:all_deps reloc-binary-dist -V
+          mv _build/reloc-bindist/ghc*.tar.xz "$BIN_DIST_NAME.tar.xz"
+          ;;
+        *)
+          run_hadrian test:all_deps binary-dist -V
+          mv _build/bindist/ghc*.tar.xz "$BIN_DIST_NAME.tar.xz"
+          ;;
+    esac
   fi
 
 }
@@ -601,16 +585,33 @@ function test_hadrian() {
   if [[ "${CROSS_EMULATOR:-}" == "NOT_SET" ]]; then
     info "Cannot test cross-compiled build without CROSS_EMULATOR being set."
     return
+    # special case for JS backend
+  elif [ -n "${CROSS_TARGET:-}" ] && [ "${CROSS_EMULATOR:-}" == "js-emulator" ]; then
+    # The JS backend doesn't support CROSS_EMULATOR logic yet
+    unset CROSS_EMULATOR
+    # run "hadrian test" directly, not using the bindist, even though it did get installed.
+    # This is a temporary solution, See !9515 for the status of hadrian support.
+    run_hadrian \
+      test \
+      --summary-junit=./junit.xml \
+      --test-have-intree-files    \
+      --docs=none                 \
+      "runtest.opts+=${RUNTEST_ARGS:-}" \
+      "runtest.opts+=--unexpected-output-dir=$TOP/unexpected-test-output" \
+      || fail "cross-compiled hadrian main testsuite"
+  elif [[ -n "${CROSS_TARGET:-}" ]] && [[ "${CROSS_TARGET:-}" == *"wasm"* ]]; then
+    run_hadrian \
+      test \
+      --summary-junit=./junit.xml \
+      "runtest.opts+=${RUNTEST_ARGS:-}" \
+      "runtest.opts+=--unexpected-output-dir=$TOP/unexpected-test-output" \
+      || fail "hadrian main testsuite targetting $CROSS_TARGET"
   elif [ -n "${CROSS_TARGET:-}" ]; then
     local instdir="$TOP/_build/install"
     local test_compiler="$instdir/bin/${cross_prefix}ghc$exe"
     install_bindist _build/bindist/ghc-*/ "$instdir"
     echo 'main = putStrLn "hello world"' > expected
     run "$test_compiler" -package ghc "$TOP/.gitlab/hello.hs" -o hello
-    # Despite "-o hello", ghc may output something like hello.exe or
-    # hello.wasm depending on the backend. For the time being let's
-    # just move it to hello before proceeding to running it.
-    mv hello.wasm hello || true
     ${CROSS_EMULATOR:-} ./hello > actual
     run diff expected actual
   elif [[ -n "${REINSTALL_GHC:-}" ]]; then
@@ -620,7 +621,9 @@ function test_hadrian() {
       --test-compiler=stage-cabal \
       --test-root-dirs=testsuite/tests/perf \
       --test-root-dirs=testsuite/tests/typecheck \
-      "runtest.opts+=${RUNTEST_ARGS:-}" || fail "hadrian cabal-install test"
+      "runtest.opts+=${RUNTEST_ARGS:-}" \
+      "runtest.opts+=--unexpected-output-dir=$TOP/unexpected-test-output" \
+      || fail "hadrian cabal-install test"
   else
     local instdir="$TOP/_build/install"
     local test_compiler="$instdir/bin/${cross_prefix}ghc$exe"
@@ -642,7 +645,7 @@ function test_hadrian() {
     then
       test_compiler_backend=$(${test_compiler} -e "GHC.Num.Backend.backendName")
       if [ $test_compiler_backend != "\"$BIGNUM_BACKEND\"" ]; then
-        fail "Test compiler has a different BIGNUM_BACKEND ($test_compiler_backend) thean requested ($BIGNUM_BACKEND)"
+        fail "Test compiler has a different BIGNUM_BACKEND ($test_compiler_backend) than requested ($BIGNUM_BACKEND)"
       fi
     fi
 
@@ -658,12 +661,13 @@ function test_hadrian() {
       --summary-junit=./junit.xml \
       --test-have-intree-files \
       --test-compiler="${test_compiler}" \
-      "runtest.opts+=${RUNTEST_ARGS:-}" || fail "hadrian main testsuite"
+      "runtest.opts+=${RUNTEST_ARGS:-}" \
+      "runtest.opts+=--unexpected-output-dir=$TOP/unexpected-test-output" \
+      || fail "hadrian main testsuite"
 
     info "STAGE2_TEST=$?"
 
-    fi
-
+  fi
 }
 
 function summarise_hi_files() {
@@ -755,6 +759,10 @@ function run_abi_test() {
   check_interfaces out/run1 out/run2 interfaces "Mismatched interface hashes"
 }
 
+function save_test_output() {
+    tar -czf unexpected-test-output.tar.gz unexpected-test-output
+}
+
 function save_cache () {
   info "Storing cabal cache from $CABAL_DIR to $CABAL_CACHE..."
   rm -Rf "$CABAL_CACHE"
@@ -790,7 +798,7 @@ function shell() {
   if [ -z "$cmd" ]; then
     cmd="bash -i"
   fi
-  run "$cmd"
+  run $cmd
 }
 
 function lint_author(){
@@ -899,8 +907,8 @@ determine_metric_baseline
 
 set_toolchain_paths
 
-case $1 in
-  usage) usage ;;
+case ${1:-help} in
+  help|usage) usage ;;
   setup) setup && cleanup_submodules ;;
   configure) time_it "configure" configure ;;
   build_hadrian) time_it "build" build_hadrian ;;
@@ -920,6 +928,7 @@ case $1 in
   lint_author) shift; lint_author "$@" ;;
   compare_interfaces_of) shift; compare_interfaces_of "$@" ;;
   clean) clean ;;
+  save_test_output) save_test_output ;;
   save_cache) save_cache ;;
   shell) shift; shell "$@" ;;
   *) fail "unknown mode $1" ;;

.gitlab/circle-ci-job.sh

-# Circle CI "backend" for Gitlab CI
-# =================================
-#
-# Usage example:
-#   .gitlab/circle-ci-job.sh validate-x86_64-linux
-#
-# There are two things to configure to get artifacts to be
-# uploaded to gitlab properly:
-#
-# - At https://<gitlab host>/admin/application_settings, expand the
-#   Continuous Integration and Deployment section and set the
-#   "Maximum artifacts size (MB)" field to something large enough
-#   to contain the bindists (the test reports are tiny in comparison).
-#   500MB seems to work fine, but 200MB might be sufficient.
-#
-# - If gitlab is exposed behind some form of proxy (e.g nginx), make sure
-#   the maximum client request body size is large enough to contain all the
-#   artifacts of a build. For nginx, this would be the following configuration
-#   option: https://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size
-#   (which can be set with services.nginx.clientMaxBodySize on nixos).
-
-#!/usr/bin/env sh
-
-set -e
-
-GHCCI_URL="localhost:8888"
-
-[ $# -gt 0 ] || (echo You need to pass the Circle CI job type as argument to this script; exit 1)
-[ ${CI_RUNNER_ID:-} ] || (echo "CI_RUNNER_ID is not set"; exit 1)
-[ ${CI_JOB_ID:-} ] || (echo "CI_JOB_ID is not set"; exit 1)
-[ ${CI_COMMIT_SHA:-} ] || (echo "CI_COMMIT_SHA is not set"; exit 1)
-[ ${CI_REPOSITORY_URL:-} ] || (echo "CI_REPOSITORY_URL is not set"; exit 1)
-[ ${CI_PIPELINE_ID:-} ] || (echo "CI_PIPELINE_ID is not set"; exit 1)
-# the first argument to this script is the Circle CI job type:
-# validate-x86_64-linux, validate-i386-linux, ...
-CIRCLE_JOB="circleci-$1"
-
-gitlab_user=$(echo $CI_REPOSITORY_URL | cut -d/ -f4)
-gitlab_repo=$(echo $CI_REPOSITORY_URL | cut -d/ -f5 | cut -d. -f1)
-
-BODY="{ \"jobType\": \"$CIRCLE_JOB\", \"source\": { \"user\": \"$gitlab_user\", \"project\":\"$gitlab_repo\", \"commit\":\"$CI_COMMIT_SHA\" }, \"pipelineID\": $CI_PIPELINE_ID, \"runnerID\": $CI_RUNNER_ID, \"jobID\": $CI_JOB_ID }"
-
-
-RESP=$(curl -s -XPOST -H "Content-Type: application/json" -d "$BODY" \
-	    http://${GHCCI_URL}/job)
-
-if [ $? -eq 0 ]; then
-    build_num=$(echo $RESP | jq '.build_num')
-    circle_url=$(echo $RESP | jq '.url')
-else
-    echo "Couldn't submit job"
-    echo $RESP
-    exit 1
-fi
-
-echo Circle CI build number: $build_num
-echo Circle CI build page: $circle_url
-
-outcome="null"
-STATUS_URL="http://${GHCCI_URL}/job/${build_num}"
-STATUS_RESP=""
-
-while [ "$outcome" == "null" ]; do
-    sleep 30s
-    STATUS_RESP=$(curl -s $STATUS_URL)
-    if [ $? -eq 0 ]; then
-	new_outcome=$(echo $STATUS_RESP | jq '.outcome')
-	jq_exitcode=$?
-	if [ "$new_outcome" == "null" ] && [ $jq_exitcode -ne 0 ]; then
-	    echo "Couldn't read 'outcome' field in JSON:"
-	    echo $STATUS_RESP
-	    echo "Skipping"
-	else
-	    outcome="$new_outcome"
-	fi
-    else
-	echo "curl failed:"
-	echo $STATUS_RESP
-	echo "Skipping"
-    fi
-done
-
-if [ "$outcome" == "\"success\"" ]; then
-    echo The build passed
-    artifactsBody=$(curl -s http://${GHCCI_URL}/job/${build_num}/artifacts)
-    (echo $artifactsBody | jq '.[] | .url' | xargs wget -q) || echo "No artifacts"
-    exit 0
-else
-    echo The build failed
-
-    artifactsBody=$(curl -s http://${GHCCI_URL}/job/${build_num}/artifacts)
-    (echo $artifactsBody | jq '.[] | .url' | xargs wget -q) || echo "No artifacts"
-
-    failing_step=$(echo $STATUS_RESP | jq '.steps | .[] | .actions | .[] | select(.status != "success")')
-    failing_step_name=$(echo $failing_step | jq '.name' | sed -e 's/^"//' -e 's/"$//' -e 's/\\r\\n/\n/')
-    echo "Failing step: $failing_step_name"
-
-    failing_cmds=$(echo $failing_step | jq '.bash_command' | sed -e 's/^"//' -e 's/"$//' -e 's/\\r\\n/\n/')
-    echo "Failing command(s):"
-    echo $failing_cmds
-
-    log_url=$(echo $failing_step | jq '.output_url' | sed -e 's/^"//' -e 's/"$//' -e 's/\\r\\n/\n/')
-    echo "Log url: $log_url"
-
-    last_log_lines=$(curl -s $log_url | gunzip | jq '.[] | select(.type == "out") | .message' | sed -e 's/^"//' -e 's/"$//' -e 's/\\r\\n/\n/' | tail -50)
-    echo End of the build log:
-    echo $last_log_lines
-
-    exit 1
-fi

.gitlab/darwin/toolchain.nix

@@ -113,6 +113,8 @@ pkgs.writeTextFile {
     export CABAL="$CABAL_INSTALL"
 
     sdk_path="$(xcrun --sdk macosx --show-sdk-path)"
-    export CONFIGURE_ARGS="$CONFIGURE_ARGS --with-ffi-libraries=$sdk_path/usr/lib --with-ffi-includes=$sdk_path/usr/include/ffi --build=${targetTriple}"
+    : ''${CONFIGURE_ARGS:=}
+    CONFIGURE_ARGS+="''${CONFIGURE_ARGS:+ }--with-ffi-libraries=$sdk_path/usr/lib --with-ffi-includes=$sdk_path/usr/include/ffi --build=${targetTriple}"
+    export CONFIGURE_ARGS
   '';
 }

.gitlab/gen-ci.cabal

-cabal-version: 3.0
-name:          gen-ci
-version:       0.1.0.0
-build-type:    Simple
-
-common warnings
-  ghc-options: -Wall
-
-executable gen_ci
-  import:           warnings
-  main-is:          gen_ci.hs
-  build-depends:
-    , aeson       >=1.8.1
-    , base
-    , bytestring
-    , containers
-
-  default-language: Haskell2010

.gitlab/generate-ci/LICENSE

+Copyright (c) 2023, The GHC Developers
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+
+    * Neither the name of The GHC Developers nor the names of other
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

.gitlab/generate-ci/README.mkd

+# generate-ci
+
+This is the generator for GHC's GitLab CI infrastructure. In particular, this
+generates two outputs:
+
+ * `.gitlab/jobs.yaml`, which is a YAML (or, strictly speaking, JSON)
+   file which defines the bulk of the validation, nightly, and release jobs of
+   GHC's CI. This is committed to the GHC repository and must be updated
+   whenever `gen_ci.hs` is modified.
+
+ * `.gitlab/jobs-metadata.json`, which is a mapping between platforms and
+   produced binary distribution names used when producing `ghcup` metadata
+   for nightly pipeline artifacts (see the `.ghcup-metadata` job in
+   `/.gitlab-ci.yaml`).
+
+
+## Modifying the CI configuration (nix)
+
+The jobs are defined in `gen_ci.hs`. After modifying this you can run
+```sh
+nix run .gitlab/generate-ci#generate-jobs
+```
+from the top of the GHC repository to update the generated configuration.
+
+
+## Modifying the CI configuration (without nix)
+
+One can run `update-ci` without Nix as follows (assuming one has `jq`,
+`cabal-install`, and GHC installed):
+```sh
+$ cabal build generate-ci
+$ PATH="$(dirname $(cabal list-bin generate-ci)):$PATH"
+$ ./generate-jobs
+```

.gitlab/generate-ci/flake.lock

+{
+  "nodes": {
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1687709756,
+        "narHash": "sha256-Y5wKlQSkgEK2weWdOu4J3riRd+kV/VCgHsqLNTTWQ/0=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "dbabf0ca0c0c4bce6ea5eaf65af5cb694d2082c7",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1687886075,
+        "narHash": "sha256-PeayJDDDy+uw1Ats4moZnRdL1OFuZm1Tj+KiHlD67+o=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "a565059a348422af5af9026b5174dc5c0dcefdae",
+        "type": "github"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "type": "indirect"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

.gitlab/generate-ci/flake.nix

+{
+  description = "GHC CI Generator";
+
+  inputs.flake-utils.url = "github:numtide/flake-utils";
+
+  outputs = { self, nixpkgs, flake-utils }:
+    flake-utils.lib.eachDefaultSystem (system:
+      let pkgs = nixpkgs.legacyPackages.${system}; in
+      {
+        packages = rec {
+          # The Haskell generator executable
+          generate-ci = pkgs.haskellPackages.callCabal2nix "generate-ci" ./. {};
+
+
+          # Wrapper scripts
+          generate-job-metadata = pkgs.runCommand "generate-job-metadata" {
+            nativeBuildInputs = with pkgs; [ makeWrapper ];
+          } ''
+            mkdir -p $out/bin
+            makeWrapper ${./generate-job-metadata} $out/bin/generate-job-metadata \
+              --prefix PATH : ${with pkgs; lib.makeBinPath [ generate-ci gitMinimal ]}
+          '';
+
+          generate-jobs = pkgs.runCommand "generate-jobs" {
+            nativeBuildInputs = with pkgs; [ makeWrapper ];
+          } ''
+            mkdir -p $out/bin
+            makeWrapper ${./generate-jobs} $out/bin/generate-jobs \
+              --prefix PATH : ${with pkgs; lib.makeBinPath [ generate-ci jq gitMinimal ]}
+          '';
+          default = generate-jobs;
+        };
+
+        apps = rec {
+          generate-jobs = flake-utils.lib.mkApp {
+            drv = self.packages.${system}.generate-jobs;
+          };
+
+          generate-job-metadata = flake-utils.lib.mkApp {
+            drv = self.packages.${system}.generate-job-metadata;
+          };
+
+          default = generate-jobs;
+        };
+      }
+    );
+}

.gitlab/gen_ci.hs → .gitlab/generate-ci/gen_ci.hs

@@ -3,9 +3,7 @@
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE DeriveFunctor #-}
 {-# LANGUAGE GeneralizedNewtypeDeriving #-}
-{- cabal:
-build-depends: base, aeson >= 1.8.1, containers, bytestring
--}
+{-# LANGUAGE ViewPatterns #-}
 
 import Data.Aeson as A
 import qualified Data.Map as Map
@@ -13,11 +11,9 @@ import Data.Map (Map)
 import Data.Maybe
 import qualified Data.ByteString.Lazy as B
 import qualified Data.ByteString.Lazy.Char8 as B
-import Data.List (intercalate)
-import Data.Set (Set)
 import qualified Data.Set as S
 import System.Environment
-import Data.Maybe
+import Data.List
 
 {-
 Note [Generating the CI pipeline]
@@ -109,7 +105,20 @@ data Opsys
   | Windows deriving (Eq)
 
 data LinuxDistro
-  = Debian11 | Debian10 | Debian9 | Fedora33 | Ubuntu2004 | Centos7 | Alpine deriving (Eq)
+  = Debian12
+  | Debian11
+  | Debian10
+  | Debian9
+  | Fedora33
+  | Fedora38
+  | Ubuntu2004
+  | Ubuntu1804
+  | Centos7
+  | Alpine312
+  | Alpine318
+  | AlpineWasm
+  | Rocky8
+  deriving (Eq)
 
 data Arch = Amd64 | AArch64 | I386
 
@@ -134,6 +143,7 @@ data BuildConfig
                 , llvmBootstrap  :: Bool
                 , withAssertions :: Bool
                 , withNuma       :: Bool
+                , withZstd       :: Bool
                 , crossTarget    :: Maybe String
                 , crossEmulator  :: CrossEmulator
                 , configureWrapper :: Maybe String
@@ -141,15 +151,18 @@ data BuildConfig
                 , tablesNextToCode :: Bool
                 , threadSanitiser :: Bool
                 , noSplitSections :: Bool
+                , validateNonmovingGc :: Bool
                 }
 
 -- Extra arguments to pass to ./configure due to the BuildConfig
 configureArgsStr :: BuildConfig -> String
 configureArgsStr bc = unwords $
-  ["--enable-unregisterised"| unregisterised bc ]
+     ["--enable-unregisterised"| unregisterised bc ]
   ++ ["--disable-tables-next-to-code" | not (tablesNextToCode bc) ]
   ++ ["--with-intree-gmp" | Just _ <- pure (crossTarget bc) ]
   ++ ["--with-system-libffi" | crossTarget bc == Just "wasm32-wasi" ]
+  ++ ["--enable-ipe-data-compression" | withZstd bc ]
+  ++ ["--enable-strict-ghc-toolchain-check"]
 
 -- Compute the hadrian flavour from the BuildConfig
 mkJobFlavour :: BuildConfig -> Flavour
@@ -159,11 +172,18 @@ mkJobFlavour BuildConfig{..} = Flavour buildFlavour opts
            [Dwarf | withDwarf] ++
            [FullyStatic | fullyStatic] ++
            [ThreadSanitiser | threadSanitiser] ++
-           [NoSplitSections | noSplitSections, buildFlavour == Release ]
+           [NoSplitSections | noSplitSections, buildFlavour == Release ] ++
+           [BootNonmovingGc | validateNonmovingGc ]
 
 data Flavour = Flavour BaseFlavour [FlavourTrans]
 
-data FlavourTrans = Llvm | Dwarf | FullyStatic | ThreadSanitiser | NoSplitSections
+data FlavourTrans =
+      Llvm
+    | Dwarf
+    | FullyStatic
+    | ThreadSanitiser
+    | NoSplitSections
+    | BootNonmovingGc
 
 data BaseFlavour = Release | Validate | SlowValidate deriving Eq
 
@@ -181,6 +201,7 @@ vanilla = BuildConfig
   , llvmBootstrap  = False
   , withAssertions = False
   , withNuma = False
+  , withZstd = False
   , crossTarget = Nothing
   , crossEmulator = NoEmulator
   , configureWrapper = Nothing
@@ -188,6 +209,7 @@ vanilla = BuildConfig
   , tablesNextToCode = True
   , threadSanitiser = False
   , noSplitSections = False
+  , validateNonmovingGc = False
   }
 
 splitSectionsBroken :: BuildConfig -> BuildConfig
@@ -212,6 +234,9 @@ debug = vanilla { buildFlavour = SlowValidate
                 , withNuma = True
                 }
 
+zstdIpe :: BuildConfig
+zstdIpe = vanilla { withZstd = True }
+
 static :: BuildConfig
 static = vanilla { fullyStatic = True }
 
@@ -260,13 +285,19 @@ tags arch opsys _bc = [runnerTag arch opsys] -- Tag for which runners we can use
 -- These names are used to find the docker image so they have to match what is
 -- in the docker registry.
 distroName :: LinuxDistro -> String
+distroName Debian12  = "deb12"
 distroName Debian11  = "deb11"
 distroName Debian10   = "deb10"
 distroName Debian9   = "deb9"
 distroName Fedora33  = "fedora33"
+distroName Fedora38  = "fedora38"
+distroName Ubuntu1804 = "ubuntu18_04"
 distroName Ubuntu2004 = "ubuntu20_04"
 distroName Centos7    = "centos7"
-distroName Alpine     = "alpine3_12"
+distroName Alpine312  = "alpine3_12"
+distroName Alpine318  = "alpine3_18"
+distroName AlpineWasm = "alpine3_17-wasm"
+distroName Rocky8     = "rocky8"
 
 opsysName :: Opsys -> String
 opsysName (Linux distro) = "linux-" ++ distroName distro
@@ -292,23 +323,25 @@ testEnv arch opsys bc = intercalate "-" $
                         ++ ["int_" ++ bignumString (bignumBackend bc) | bignumBackend bc /= Gmp]
                         ++ ["unreg" | unregisterised bc ]
                         ++ ["numa"  | withNuma bc ]
+                        ++ ["zstd"  | withZstd bc ]
                         ++ ["no_tntc"  | not (tablesNextToCode bc) ]
                         ++ ["cross_"++triple  | Just triple <- pure $ crossTarget bc ]
                         ++ [flavourString (mkJobFlavour bc)]
 
 -- | The hadrian flavour string we are going to use for this build
 flavourString :: Flavour -> String
-flavourString (Flavour base trans) = baseString base ++ concatMap (("+" ++) . flavourString) trans
+flavourString (Flavour base trans) = base_string base ++ concatMap (("+" ++) . flavour_string) trans
   where
-    baseString Release = "release"
-    baseString Validate = "validate"
-    baseString SlowValidate = "slow-validate"
+    base_string Release = "release"
+    base_string Validate = "validate"
+    base_string SlowValidate = "slow-validate"
 
-    flavourString Llvm = "llvm"
-    flavourString Dwarf = "debug_info"
-    flavourString FullyStatic = "fully_static"
-    flavourString ThreadSanitiser = "thread_sanitizer"
-    flavourString NoSplitSections = "no_split_sections"
+    flavour_string Llvm = "llvm"
+    flavour_string Dwarf = "debug_info"
+    flavour_string FullyStatic = "fully_static"
+    flavour_string ThreadSanitiser = "thread_sanitizer"
+    flavour_string NoSplitSections = "no_split_sections"
+    flavour_string BootNonmovingGc = "boot_nonmoving_gc"
 
 -- The path to the docker image (just for linux builders)
 dockerImage :: Arch -> Opsys -> Maybe String
@@ -378,11 +411,13 @@ opsysVariables AArch64 (Darwin {}) =
           ]
 opsysVariables Amd64 (Darwin {}) =
   mconcat [ "NIX_SYSTEM" =: "x86_64-darwin"
-          , "MACOSX_DEPLOYMENT_TARGET" =: "10.10"
+          , "MACOSX_DEPLOYMENT_TARGET" =: "10.13"
           -- "# Only Sierra and onwards supports clock_gettime. See #12858"
           , "ac_cv_func_clock_gettime" =: "no"
           -- # Only newer OS Xs support utimensat. See #17895
           , "ac_cv_func_utimensat" =: "no"
+          -- # Only newer OS Xs support futimens. See #22938
+          , "ac_cv_func_futimens" =: "no"
           , "LANG" =: "en_US.UTF-8"
           , "CONFIGURE_ARGS" =: "--with-intree-gmp --with-system-libffi"
           -- Fonts can't be installed on darwin
@@ -397,20 +432,25 @@ opsysVariables _ (Windows {}) =
           , "GHC_VERSION" =: "9.4.3" ]
 opsysVariables _ _ = mempty
 
-
-distroVariables :: LinuxDistro -> Variables
-distroVariables Alpine = mconcat
+alpineVariables = mconcat
   [ -- Due to #20266
     "CONFIGURE_ARGS" =: "--disable-ld-override"
   , "INSTALL_CONFIGURE_ARGS" =: "--disable-ld-override"
   , "HADRIAN_ARGS" =: "--docs=no-sphinx"
     -- encoding004: due to lack of locale support
     -- T10458, ghcilink002: due to #17869
-    -- linker_unload_native: due to musl not supporting any means of probing dynlib dependencies
-    -- (see Note [Object unloading]).
-  , "BROKEN_TESTS" =: "encoding004 T10458 ghcilink002 linker_unload_native"
+  , "BROKEN_TESTS" =: "encoding004 T10458"
   ]
+
+
+distroVariables :: LinuxDistro -> Variables
+distroVariables Alpine312 = alpineVariables
+distroVariables Alpine318 = alpineVariables
 distroVariables Centos7 = mconcat [
+    "HADRIAN_ARGS" =: "--docs=no-sphinx"
+  , "BROKEN_TESTS" =: "T22012" -- due to #23979
+  ]
+distroVariables Rocky8 = mconcat [
   "HADRIAN_ARGS" =: "--docs=no-sphinx"
   ]
 distroVariables Fedora33 = mconcat
@@ -475,21 +515,34 @@ instance ToJSON ArtifactsWhen where
 -----------------------------------------------------------------------------
 
 -- Data structure which records the condition when a job is run.
-data OnOffRules = OnOffRules { rule_set :: Set Rule -- ^ The set of enabled rules
+data OnOffRules = OnOffRules { rule_set :: Rule -- ^ The enabled rules
                              , when :: ManualFlag   -- ^ The additional condition about when to run this job.
                              }
 
--- The initial set of rules where all rules are disabled and the job is always run.
+-- The initial set of rules, which assumes a Validate pipeline which is run with FullCI.
 emptyRules :: OnOffRules
-emptyRules = OnOffRules S.empty OnSuccess
+emptyRules = OnOffRules (ValidateOnly (S.singleton FullCI)) OnSuccess
 
 -- When to run the job
 data ManualFlag = Manual -- ^ Only run the job when explicitly triggered by a user
                 | OnSuccess -- ^ Always run it, if the rules pass (the default)
                 deriving Eq
 
-enableRule :: Rule -> OnOffRules -> OnOffRules
-enableRule r (OnOffRules o m) = OnOffRules (S.insert r o) m
+setRule :: Rule -> OnOffRules -> OnOffRules
+setRule r (OnOffRules _ m) = OnOffRules r m
+
+enableValidateRule :: ValidateRule -> OnOffRules -> OnOffRules
+enableValidateRule r = modifyValidateRules (S.insert r)
+
+onlyValidateRule :: ValidateRule -> OnOffRules -> OnOffRules
+onlyValidateRule r  = modifyValidateRules (const (S.singleton r))
+
+removeValidateRule :: ValidateRule -> OnOffRules -> OnOffRules
+removeValidateRule r = modifyValidateRules (S.delete r)
+
+modifyValidateRules :: (S.Set ValidateRule -> S.Set ValidateRule) -> OnOffRules -> OnOffRules
+modifyValidateRules f (OnOffRules (ValidateOnly rs) m) = OnOffRules (ValidateOnly (f rs)) m
+modifyValidateRules _ r = error $ "Applying validate rule to nightly/release job:" ++ show (rule_set r)
 
 manualRule :: OnOffRules -> OnOffRules
 manualRule rules = rules { when = Manual }
@@ -498,10 +551,19 @@ manualRule rules = rules { when = Manual }
 -- For example, even if you don't explicitly disable a rule it will end up in the
 -- rule list with the OFF state.
 enumRules :: OnOffRules -> [OnOffRule]
-enumRules o = map lkup rules
+enumRules (OnOffRules r _) = rulesList
   where
-    enabled_rules = rule_set o
-    lkup r = OnOffRule (if S.member r enabled_rules then On else Off) r
+    rulesList = case r of
+                  ValidateOnly rs -> [OnOffRule On (ValidateOnly rs)
+                                    , OnOffRule Off ReleaseOnly
+                                    , OnOffRule Off Nightly ]
+                  Nightly -> [ OnOffRule Off (ValidateOnly S.empty)
+                             , OnOffRule Off ReleaseOnly
+                             , OnOffRule On Nightly ]
+                  ReleaseOnly -> [ OnOffRule Off (ValidateOnly S.empty)
+                                 , OnOffRule On ReleaseOnly
+                                 , OnOffRule Off Nightly ]
+
 
 
 data OnOffRule = OnOffRule OnOff Rule
@@ -523,18 +585,31 @@ instance ToJSON OnOffRules where
 
     where
       one_rule (OnOffRule onoff r) = ruleString onoff r
-      parens s = "(" ++ s ++ ")"
-      and_all rs = intercalate " && " (map parens rs)
+
+
+parens :: [Char] -> [Char]
+parens s = "(" ++ s ++ ")"
+and_all :: [[Char]] -> [Char]
+and_all rs = intercalate " && " (map parens rs)
+or_all :: [[Char]] -> [Char]
+or_all rs = intercalate " || " (map parens rs)
 
 -- | A Rule corresponds to some condition which must be satisifed in order to
 -- run the job.
-data Rule = FastCI       -- ^ Run this job when the fast-ci label is set
-          | ReleaseOnly  -- ^ Only run this job in a release pipeline
+data Rule = ReleaseOnly  -- ^ Only run this job in a release pipeline
           | Nightly      -- ^ Only run this job in the nightly pipeline
-          | LLVMBackend  -- ^ Only run this job when the "LLVM backend" label is present
-          | FreeBSDLabel -- ^ Only run this job when the "FreeBSD" label is set.
-          | Disable      -- ^ Don't run this job.
-          deriving (Bounded, Enum, Ord, Eq)
+          | ValidateOnly (S.Set ValidateRule) -- ^ Only run this job in a validate pipeline, when any of these rules are enabled.
+          deriving (Show, Ord, Eq)
+
+data ValidateRule =
+            FullCI       -- ^ Run this job when the "full-ci" label is present.
+          | LLVMBackend  -- ^ Run this job when the "LLVM backend" label is present
+          | JSBackend    -- ^ Run this job when the "javascript" label is present
+          | FreeBSDLabel -- ^ Run this job when the "FreeBSD" label is set.
+          | NonmovingGc  -- ^ Run this job when the "non-moving GC" label is set.
+          | IpeData      -- ^ Run this job when the "IPE" label is set
+          | TestPrimops  -- ^ Run this job when "test-primops" label is set
+          deriving (Show, Enum, Bounded, Ord, Eq)
 
 -- A constant evaluating to True because gitlab doesn't support "true" in the
 -- expression language.
@@ -542,27 +617,44 @@ true :: String
 true =  "\"true\" == \"true\""
 -- A constant evaluating to False because gitlab doesn't support "true" in the
 -- expression language.
-false :: String
-false = "\"disabled\" != \"disabled\""
+_false :: String
+_false = "\"disabled\" != \"disabled\""
 
 -- Convert the state of the rule into a string that gitlab understand.
 ruleString :: OnOff -> Rule -> String
-ruleString On FastCI = true
-ruleString Off FastCI = "$CI_MERGE_REQUEST_LABELS !~ /.*fast-ci.*/"
-ruleString On LLVMBackend = "$CI_MERGE_REQUEST_LABELS =~ /.*LLVM backend.*/"
-ruleString Off LLVMBackend = true
-ruleString On FreeBSDLabel = "$CI_MERGE_REQUEST_LABELS =~ /.*FreeBSD.*/"
-ruleString Off FreeBSDLabel = true
+ruleString On (ValidateOnly vs) =
+  case S.toList vs of
+    [] -> true
+    conds -> or_all (map validateRuleString conds)
+ruleString Off (ValidateOnly {}) = true
 ruleString On ReleaseOnly = "$RELEASE_JOB == \"yes\""
 ruleString Off ReleaseOnly = "$RELEASE_JOB != \"yes\""
 ruleString On Nightly = "$NIGHTLY"
 ruleString Off Nightly = "$NIGHTLY == null"
-ruleString On Disable = false
-ruleString Off Disable = true
 
--- Enumeration of all the rules
-rules :: [Rule]
-rules = [minBound .. maxBound]
+labelString :: String -> String
+labelString s =  "$CI_MERGE_REQUEST_LABELS =~ /.*" ++ s ++ ".*/"
+
+branchStringExact :: String -> String
+branchStringExact s = "$CI_COMMIT_BRANCH == \"" ++ s ++ "\""
+
+branchStringLike :: String -> String
+branchStringLike s = "$CI_COMMIT_BRANCH =~ /" ++ s ++ "/"
+
+
+validateRuleString :: ValidateRule -> String
+validateRuleString FullCI = or_all ([ labelString "full-ci"
+                                    , labelString "marge_bot_batch_merge_job"
+                                    , branchStringExact "master"
+                                    , branchStringLike "ghc-[0-9]+\\.[0-9]+"
+                                    ])
+
+validateRuleString LLVMBackend  = labelString "LLVM backend"
+validateRuleString JSBackend    = labelString "javascript"
+validateRuleString FreeBSDLabel = labelString "FreeBSD"
+validateRuleString NonmovingGc  = labelString "non-moving GC"
+validateRuleString IpeData      = labelString "IPE"
+validateRuleString TestPrimops  = labelString "test-primops"
 
 -- | A 'Job' is the description of a single job in a gitlab pipeline. The
 -- job contains all the information about how to do the build but can be further
@@ -626,7 +718,6 @@ job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} }
         , "bash .gitlab/ci.sh test_hadrian" ]
       | otherwise
       = [ "find libraries -name config.sub -exec cp config.sub {} \\;" | Darwin == opsys ] ++
-        [ "sudo apk del --purge glibc*" | opsys == Linux Alpine, isNothing $ crossTarget buildConfig ] ++
         [ "sudo chown ghc:ghc -R ." | Linux {} <- [opsys]] ++
         [ ".gitlab/ci.sh setup"
         , ".gitlab/ci.sh configure"
@@ -637,10 +728,12 @@ job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} }
     jobAfterScript
       | Windows <- opsys =
       [ "bash .gitlab/ci.sh save_cache"
+      , "bash .gitlab/ci.sh save_test_output"
       , "bash .gitlab/ci.sh clean"
       ]
       | otherwise =
       [ ".gitlab/ci.sh save_cache"
+      , ".gitlab/ci.sh save_test_output"
       , ".gitlab/ci.sh clean"
       , "cat ci_timings" ]
 
@@ -649,7 +742,7 @@ job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} }
     jobDependencies = []
     jobVariables = mconcat
       [ opsysVariables arch opsys
-      ,"TEST_ENV" =: testEnv arch opsys buildConfig
+      , "TEST_ENV" =: testEnv arch opsys buildConfig
       , "BIN_DIST_NAME" =: binDistName arch opsys buildConfig
       , "BUILD_FLAVOUR" =: flavourString jobFlavour
       , "BIGNUM_BACKEND" =: bignumString (bignumBackend buildConfig)
@@ -663,13 +756,19 @@ job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} }
           Emulator s       -> "CROSS_EMULATOR" =: s
           NoEmulatorNeeded -> mempty
       , if withNuma buildConfig then "ENABLE_NUMA" =: "1" else mempty
+      , let runtestArgs =
+                [ "--way=nonmoving --way=nonmoving_thr --way=nonmoving_thr_sanity"
+                | validateNonmovingGc buildConfig
+                ]
+        in "RUNTEST_ARGS" =: unwords runtestArgs
       ]
 
     jobArtifacts = Artifacts
       { junitReport = "junit.xml"
       , expireIn = "2 weeks"
       , artifactPaths = [binDistName arch opsys buildConfig ++ ".tar.xz"
-                        ,"junit.xml"]
+                        ,"junit.xml"
+                        ,"unexpected-test-output.tar.gz"]
       , artifactsWhen = ArtifactsAlways
       }
 
@@ -700,12 +799,28 @@ modifyJobs = fmap
 
 -- | Modify just the validate jobs in a 'JobGroup'
 modifyValidateJobs :: (a -> a) -> JobGroup a -> JobGroup a
-modifyValidateJobs f jg = jg { v = f <$> v jg }
+modifyValidateJobs f jg = jg { v = fmap f <$> v jg }
+
+-- | Modify just the nightly jobs in a 'JobGroup'
+modifyNightlyJobs :: (a -> a) -> JobGroup a -> JobGroup a
+modifyNightlyJobs f jg = jg { n = fmap f <$> n jg }
 
 -- Generic helpers
 
-addJobRule :: Rule -> Job -> Job
-addJobRule r j = j { jobRules = enableRule r (jobRules j) }
+setJobRule :: Rule -> Job -> Job
+setJobRule r j = j { jobRules = setRule r (jobRules j) }
+
+addValidateJobRule :: ValidateRule -> Job -> Job
+addValidateJobRule r = modifyValidateJobRule (enableValidateRule r)
+
+onlyValidateJobRule :: ValidateRule -> Job -> Job
+onlyValidateJobRule r = modifyValidateJobRule (onlyValidateRule r)
+
+removeValidateJobRule :: ValidateRule -> Job -> Job
+removeValidateJobRule r = modifyValidateJobRule (removeValidateRule r)
+
+modifyValidateJobRule :: (OnOffRules -> OnOffRules) -> Job -> Job
+modifyValidateJobRule f j = j { jobRules = f (jobRules j) }
 
 addVariable :: String -> String -> Job -> Job
 addVariable k v j = j { jobVariables = mminsertWith (++) k [v] (jobVariables j) }
@@ -722,17 +837,25 @@ delVariable k j = j { jobVariables = MonoidalMap $ Map.delete k $ unMonoidalMap
 validate :: Arch -> Opsys -> BuildConfig -> NamedJob Job
 validate = job
 
+-- Nightly and release apply the FastCI configuration to all jobs so that they all run in
+-- the pipeline (not conditional on the full-ci label)
+nightlyRule :: Job -> Job
+nightlyRule = setJobRule Nightly
+
+releaseRule :: Job -> Job
+releaseRule = setJobRule ReleaseOnly
+
 -- | Make a normal nightly CI job
 nightly :: Arch -> Opsys -> BuildConfig -> NamedJob Job
 nightly arch opsys bc =
   let NamedJob n j = job arch opsys bc
-  in NamedJob { name = "nightly-" ++ n, jobInfo = addJobRule Nightly . keepArtifacts "8 weeks" . highCompression $ j}
+  in NamedJob { name = "nightly-" ++ n, jobInfo = nightlyRule . keepArtifacts "8 weeks" . highCompression $ j}
 
 -- | Make a normal release CI job
 release :: Arch -> Opsys -> BuildConfig -> NamedJob Job
 release arch opsys bc =
   let NamedJob n j = job arch opsys (bc { buildFlavour = Release })
-  in NamedJob { name = "release-" ++ n, jobInfo = addJobRule ReleaseOnly . keepArtifacts "1 year" . ignorePerfFailures . highCompression $ j}
+  in NamedJob { name = "release-" ++ n, jobInfo = releaseRule . keepArtifacts "1 year" . ignorePerfFailures . useHashUnitIds . highCompression $ j}
 
 -- Specific job modification functions
 
@@ -757,9 +880,13 @@ ignorePerfFailures = addVariable "IGNORE_PERF_FAILURES" "all"
 highCompression :: Job -> Job
 highCompression = addVariable "XZ_OPT" "-9"
 
+useHashUnitIds :: Job -> Job
+useHashUnitIds = addVariable "HADRIAN_ARGS" "--hash-unit-ids"
+
 -- | Mark the validate job to run in fast-ci mode
+-- This is default way, to enable all jobs you have to apply the `full-ci` label.
 fastCI :: JobGroup Job -> JobGroup Job
-fastCI = modifyValidateJobs (addJobRule FastCI)
+fastCI = modifyValidateJobs (removeValidateJobRule FullCI)
 
 -- | Mark a group of jobs as allowed to fail.
 allowFailureGroup :: JobGroup Job -> JobGroup Job
@@ -767,15 +894,19 @@ allowFailureGroup = modifyJobs allowFailure
 
 -- | Add a 'Rule' to just the validate job, for example, only run a job if a certain
 -- label is set.
-addValidateRule :: Rule -> JobGroup Job -> JobGroup Job
-addValidateRule t = modifyValidateJobs (addJobRule t)
+addValidateRule :: ValidateRule -> JobGroup Job -> JobGroup Job
+addValidateRule t = modifyValidateJobs (addValidateJobRule t)
+
+-- | Only run a validate job if a certain rule is enabled
+onlyRule :: ValidateRule -> JobGroup Job -> JobGroup Job
+onlyRule t = modifyValidateJobs (onlyValidateJobRule t)
 
 -- | Don't run the validate job, normally used to alleviate CI load by marking
 -- jobs which are unlikely to fail (ie different linux distros)
 disableValidate :: JobGroup Job -> JobGroup Job
-disableValidate = addValidateRule Disable
+disableValidate st = st { v = Nothing }
 
-data NamedJob a = NamedJob { name :: String, jobInfo :: a } deriving Functor
+data NamedJob a = NamedJob { name :: String, jobInfo :: a } deriving (Show, Functor)
 
 renameJob :: (String -> String) -> NamedJob a -> NamedJob a
 renameJob f (NamedJob n i) = NamedJob (f n) i
@@ -785,31 +916,32 @@ instance ToJSON a => ToJSON (NamedJob a) where
     [ "name" A..= name nj
     , "jobInfo" A..= jobInfo nj ]
 
+
+--data NamedJobGroup a = NamedJobGroup { platform :: String, jg :: JobGroup a }
+
 -- Jobs are grouped into either triples or pairs depending on whether the
 -- job is just validate and nightly, or also release.
-data JobGroup a = StandardTriple { v :: NamedJob a
-                                 , n :: NamedJob a
-                                 , r :: NamedJob a }
-                | ValidateOnly   { v :: NamedJob a
-                                 , n :: NamedJob a } deriving Functor
+data JobGroup a = StandardTriple { v :: Maybe (NamedJob a)
+                                 , n :: Maybe (NamedJob a)
+                                 , r :: Maybe (NamedJob a) } deriving (Functor, Show)
 
 instance ToJSON a => ToJSON (JobGroup a) where
-  toJSON jg = object
-    [ "n" A..= n jg
-    , "r" A..= r jg
+  toJSON StandardTriple{..} = object
+    [ "v" A..= v
+    , "n" A..= n
+    , "r" A..= r
     ]
 
 rename :: (String -> String) -> JobGroup a -> JobGroup a
-rename f (StandardTriple nv nn nr) = StandardTriple (renameJob f nv) (renameJob f nn) (renameJob f nr)
-rename f (ValidateOnly nv nn) = ValidateOnly (renameJob f nv) (renameJob f nn)
+rename f (StandardTriple nv nn nr) = StandardTriple (renameJob f <$> nv) (renameJob f <$> nn) (renameJob f <$> nr)
 
 -- | Construct a 'JobGroup' which consists of a validate, nightly and release build with
 -- a specific config.
 standardBuildsWithConfig :: Arch -> Opsys -> BuildConfig -> JobGroup Job
 standardBuildsWithConfig a op bc =
-  StandardTriple (validate a op bc)
-                 (nightly a op bc)
-                 (release a op bc)
+  StandardTriple (Just (validate a op bc))
+                 (Just (nightly a op bc))
+                 (Just (release a op bc))
 
 -- | Construct a 'JobGroup' which consists of a validate, nightly and release builds with
 -- the 'vanilla' config.
@@ -819,11 +951,12 @@ standardBuilds a op = standardBuildsWithConfig a op vanilla
 -- | Construct a 'JobGroup' which just consists of a validate and nightly build. We don't
 -- produce releases for these jobs.
 validateBuilds :: Arch -> Opsys -> BuildConfig -> JobGroup Job
-validateBuilds a op bc = ValidateOnly (validate a op bc) (nightly a op bc)
+validateBuilds a op bc = StandardTriple { v = Just (validate a op bc)
+                                        , n = Just (nightly a op bc)
+                                        , r = Nothing }
 
 flattenJobGroup :: JobGroup a -> [(String, a)]
-flattenJobGroup (StandardTriple a b c) = map flattenNamedJob [a,b,c]
-flattenJobGroup (ValidateOnly a b) = map flattenNamedJob [a, b]
+flattenJobGroup (StandardTriple a b c) = map flattenNamedJob (catMaybes [a,b,c])
 
 flattenNamedJob :: NamedJob a -> (String, a)
 flattenNamedJob (NamedJob n i) = (n, i)
@@ -831,55 +964,78 @@ flattenNamedJob (NamedJob n i) = (n, i)
 
 -- | Specification for all the jobs we want to build.
 jobs :: Map String Job
-jobs = Map.fromList $ concatMap (filter is_enabled_job . flattenJobGroup) job_groups
-  where
-    is_enabled_job (_, Job {jobRules = OnOffRules {..}}) = not $ Disable `S.member` rule_set
+jobs = Map.fromList $ concatMap (flattenJobGroup) job_groups
 
 job_groups :: [JobGroup Job]
 job_groups =
      [ disableValidate (standardBuilds Amd64 (Linux Debian10))
-     , standardBuildsWithConfig Amd64 (Linux Debian10) dwarf
+     , addValidateRule TestPrimops (standardBuildsWithConfig Amd64 (Linux Debian10) dwarf)
      , validateBuilds Amd64 (Linux Debian10) nativeInt
-     , fastCI (validateBuilds Amd64 (Linux Debian10) unreg)
+     , validateBuilds Amd64 (Linux Debian10) unreg
      , fastCI (validateBuilds Amd64 (Linux Debian10) debug)
-     , modifyValidateJobs manual tsan_jobs
-     , modifyValidateJobs manual (validateBuilds Amd64 (Linux Debian10) noTntc)
-     , addValidateRule LLVMBackend (validateBuilds Amd64 (Linux Debian10) llvm)
-
+     , -- Nightly allowed to fail: #22520
+       modifyNightlyJobs allowFailure
+         (modifyValidateJobs manual tsan_jobs)
+     , -- Nightly allowed to fail: #22343
+       modifyNightlyJobs allowFailure
+        (modifyValidateJobs manual (validateBuilds Amd64 (Linux Debian10) noTntc))
+     , onlyRule LLVMBackend (validateBuilds Amd64 (Linux Debian12) llvm)
      , disableValidate (standardBuilds Amd64 (Linux Debian11))
+     , disableValidate (standardBuilds Amd64 (Linux Debian12))
      -- We still build Deb9 bindists for now due to Ubuntu 18 and Linux Mint 19
      -- not being at EOL until April 2023 and they still need tinfo5.
      , disableValidate (standardBuildsWithConfig Amd64 (Linux Debian9) (splitSectionsBroken vanilla))
+     , disableValidate (standardBuilds Amd64 (Linux Ubuntu1804))
      , disableValidate (standardBuilds Amd64 (Linux Ubuntu2004))
+     , disableValidate (standardBuilds Amd64 (Linux Rocky8))
      , disableValidate (standardBuildsWithConfig Amd64 (Linux Centos7) (splitSectionsBroken vanilla))
      -- Fedora33 job is always built with perf so there's one job in the normal
      -- validate pipeline which is built with perf.
-     , standardBuildsWithConfig Amd64 (Linux Fedora33) releaseConfig
+     , fastCI (standardBuildsWithConfig Amd64 (Linux Fedora33) releaseConfig)
      -- This job is only for generating head.hackage docs
      , hackage_doc_job (disableValidate (standardBuildsWithConfig Amd64 (Linux Fedora33) releaseConfig))
      , disableValidate (standardBuildsWithConfig Amd64 (Linux Fedora33) dwarf)
-     , fastCI (standardBuildsWithConfig Amd64 Windows (splitSectionsBroken vanilla))
-     , disableValidate (standardBuildsWithConfig Amd64 Windows (splitSectionsBroken nativeInt))
-     , standardBuilds Amd64 Darwin
-     , allowFailureGroup (addValidateRule FreeBSDLabel (standardBuilds Amd64 FreeBSD13))
-     , standardBuilds AArch64 Darwin
-     , standardBuildsWithConfig AArch64 (Linux Debian10) (splitSectionsBroken vanilla)
-     , disableValidate (validateBuilds AArch64 (Linux Debian10) llvm)
-     , standardBuildsWithConfig I386 (Linux Debian9) (splitSectionsBroken vanilla)
-     , standardBuildsWithConfig Amd64 (Linux Alpine) (splitSectionsBroken static)
-     , disableValidate (allowFailureGroup (standardBuildsWithConfig Amd64 (Linux Alpine) staticNativeInt))
+     , disableValidate (standardBuilds Amd64 (Linux Fedora38))
+     , fastCI (standardBuildsWithConfig Amd64 Windows vanilla)
+     , disableValidate (standardBuildsWithConfig Amd64 Windows nativeInt)
+     , addValidateRule TestPrimops (standardBuilds Amd64 Darwin)
+     , allowFailureGroup (onlyRule FreeBSDLabel (validateBuilds Amd64 FreeBSD13 vanilla))
+     , fastCI (standardBuilds AArch64 Darwin)
+     , fastCI (standardBuildsWithConfig AArch64 (Linux Debian10) (splitSectionsBroken vanilla))
+     , disableValidate (standardBuildsWithConfig AArch64 (Linux Debian11) (splitSectionsBroken vanilla))
+     , onlyRule LLVMBackend (validateBuilds AArch64 (Linux Debian12) llvm)
+     , standardBuildsWithConfig I386 (Linux Debian10) (splitSectionsBroken vanilla)
+     -- Fully static build, in theory usable on any linux distribution.
+     , fullyStaticBrokenTests (standardBuildsWithConfig Amd64 (Linux Alpine312) (splitSectionsBroken static))
+     -- Dynamically linked build, suitable for building your own static executables on alpine
+     , disableValidate (standardBuildsWithConfig Amd64 (Linux Alpine312) (splitSectionsBroken vanilla))
+     , disableValidate (standardBuildsWithConfig AArch64 (Linux Alpine318) (splitSectionsBroken vanilla))
+     , fullyStaticBrokenTests (disableValidate (allowFailureGroup (standardBuildsWithConfig Amd64 (Linux Alpine312) staticNativeInt)))
      , validateBuilds Amd64 (Linux Debian11) (crossConfig "aarch64-linux-gnu" (Emulator "qemu-aarch64 -L /usr/aarch64-linux-gnu") Nothing)
-     , validateBuilds Amd64 (Linux Debian11) (crossConfig "js-unknown-ghcjs" NoEmulatorNeeded (Just "emconfigure")
-        )
-        { bignumBackend = Native
-        }
+
+     , addValidateRule JSBackend (validateBuilds Amd64 (Linux Debian11) javascriptConfig)
+
      , make_wasm_jobs wasm_build_config
-     , disableValidate $ make_wasm_jobs wasm_build_config { bignumBackend = Native }
-     , disableValidate $ make_wasm_jobs wasm_build_config { unregisterised = True }
+     , modifyValidateJobs manual $
+         make_wasm_jobs wasm_build_config {bignumBackend = Native}
+     , modifyValidateJobs manual $
+         make_wasm_jobs wasm_build_config {unregisterised = True}
+     , onlyRule NonmovingGc (standardBuildsWithConfig Amd64 (Linux Debian11) vanilla {validateNonmovingGc = True})
+     , onlyRule IpeData (validateBuilds Amd64 (Linux Debian10) zstdIpe)
      ]
 
   where
+    javascriptConfig = (crossConfig "javascript-unknown-ghcjs" (Emulator "js-emulator") (Just "emconfigure"))
+                         { bignumBackend = Native }
+
+    -- ghcilink002 broken due to #17869
+    --
+    -- linker_unload_native: due to musl not supporting any means of probing dynlib dependencies
+    -- (see Note [Object unloading]).
+    fullyStaticBrokenTests = modifyJobs (addVariable "BROKEN_TESTS" "ghcilink002 linker_unload_native")
+
     hackage_doc_job = rename (<> "-hackage") . modifyJobs (addVariable "HADRIAN_ARGS" "--haddock-base-url")
+
     tsan_jobs =
       modifyJobs
         ( addVariable "TSAN_OPTIONS" "suppressions=$CI_PROJECT_DIR/rts/.tsan-suppressions"
@@ -894,7 +1050,7 @@ job_groups =
             . setVariable "HADRIAN_ARGS" "--docs=none"
             . delVariable "INSTALL_CONFIGURE_ARGS"
         )
-        $ validateBuilds Amd64 (Linux Alpine) cfg
+        $ validateBuilds Amd64 (Linux AlpineWasm) cfg
 
     wasm_build_config =
       (crossConfig "wasm32-wasi" NoEmulatorNeeded Nothing)
@@ -917,28 +1073,54 @@ mkPlatform arch opsys = archName arch <> "-" <> opsysName opsys
 --  * Prefer jobs which have a corresponding release pipeline
 --  * Explicitly require tie-breaking for other cases.
 platform_mapping :: Map String (JobGroup BindistInfo)
-platform_mapping = Map.map go $
-  Map.fromListWith combine [ (uncurry mkPlatform (jobPlatform (jobInfo $ v j)), j) | j <- job_groups ]
+platform_mapping = Map.map go combined_result
   where
-    whitelist = [ "x86_64-linux-alpine3_12-int_native-validate+fully_static"
-                , "x86_64-linux-deb10-validate"
+    whitelist = [ "x86_64-linux-alpine3_12-validate"
+                , "x86_64-linux-deb11-validate"
+                , "x86_64-linux-deb12-validate"
+                , "x86_64-linux-deb10-validate+debug_info"
                 , "x86_64-linux-fedora33-release"
+                , "x86_64-linux-deb11-cross_aarch64-linux-gnu-validate"
                 , "x86_64-windows-validate"
+                , "nightly-x86_64-linux-alpine3_17-wasm-cross_wasm32-wasi-release+fully_static"
+                , "nightly-x86_64-linux-deb11-validate"
+                , "nightly-x86_64-linux-deb12-validate"
+                , "x86_64-linux-alpine3_17-wasm-cross_wasm32-wasi-release+fully_static"
+                , "nightly-aarch64-linux-deb10-validate"
+                , "nightly-x86_64-linux-alpine3_12-validate"
+                , "nightly-x86_64-linux-deb10-validate"
+                , "nightly-x86_64-linux-fedora33-release"
+                , "nightly-x86_64-windows-validate"
+                , "release-x86_64-linux-alpine3_12-release+no_split_sections"
+                , "release-x86_64-linux-deb10-release"
+                , "release-x86_64-linux-deb11-release"
+                , "release-x86_64-linux-deb12-release"
+                , "release-x86_64-linux-fedora33-release"
+                , "release-x86_64-windows-release"
                 ]
 
+    process sel = Map.fromListWith combine [ (uncurry mkPlatform (jobPlatform (jobInfo $ j)), j) | (sel -> Just j) <- job_groups  ]
+
+    vs = process v
+    ns = process n
+    rs = process r
+
+    all_platforms = Map.keysSet vs <> Map.keysSet ns <> Map.keysSet rs
+
+    combined_result = Map.fromList [ (p, StandardTriple { v = Map.lookup p vs
+                                                        , n = Map.lookup p ns
+                                                        , r = Map.lookup p rs })
+                                   | p <- S.toList all_platforms ]
+
     combine a b
-      | name (v a) `elem` whitelist = a -- Explicitly selected
-      | name (v b) `elem` whitelist = b
-      | hasReleaseBuild a, not (hasReleaseBuild b) = a -- Has release build, but other doesn't
-      | hasReleaseBuild b, not (hasReleaseBuild a) = b
-      | otherwise = error (show (name (v a)) ++ show (name (v b)))
+      | name a `elem` whitelist = a -- Explicitly selected
+      | name b `elem` whitelist = b
+      | otherwise = error (show (name a) ++ show (name b))
 
     go = fmap (BindistInfo . unwords . fromJust . mmlookup "BIN_DIST_NAME" . jobVariables)
 
-    hasReleaseBuild (StandardTriple{}) = True
-    hasReleaseBuild (ValidateOnly{}) = False
 
-data BindistInfo = BindistInfo { bindistName :: String }
+data BindistInfo = BindistInfo { bindistName :: String } deriving Show
 
 instance ToJSON BindistInfo where
   toJSON (BindistInfo n) = object [ "bindistName" A..= n ]
@@ -953,9 +1135,9 @@ main = do
     ("metadata":as) -> write_result as platform_mapping
     _ -> error "gen_ci.hs <gitlab|metadata> [file.json]"
 
+write_result :: ToJSON a => [FilePath] -> a -> IO ()
 write_result as obj =
   (case as of
     [] -> B.putStrLn
     (fp:_) -> B.writeFile fp)
     (A.encode obj)
-

.gitlab/generate-ci/generate-ci.cabal

+cabal-version:      3.0
+name:               generate-ci
+version:            0.1.0.0
+license:            BSD-3-Clause
+license-file:       LICENSE
+build-type:         Simple
+
+executable generate-ci
+    main-is:          gen_ci.hs
+    ghc-options:      -Wall
+    build-depends:    base,
+                      containers,
+                      bytestring,
+                      aeson >= 1.8.1
+    default-language: Haskell2010

.gitlab/generate-ci/generate-job-metadata

+#!/usr/bin/env bash
+
+set -e
+
+out_dir="$(git rev-parse --show-toplevel)/.gitlab"
+
+# Update job metadata for ghcup
+generate-ci metadata "$out_dir/jobs-metadata.json"
+echo "Updated $out_dir/jobs-metadata.json"

.gitlab/generate-ci/generate-jobs

+#!/usr/bin/env bash
+
+set -e
+
+out_dir="$(git rev-parse --show-toplevel)/.gitlab"
+
+tmp="$(mktemp)"
+generate-ci gitlab "$tmp"
+rm -f "$out_dir/jobs.yaml"
+echo "### THIS IS A GENERATED FILE, DO NOT MODIFY DIRECTLY" > "$out_dir/jobs.yaml"
+cat "$tmp" | jq >> "$out_dir/jobs.yaml"
+rm "$tmp"
+echo "Updated $out_dir/jobs.yaml"
+

.gitlab/generate_job_metadata

-#! /usr/bin/env nix-shell
-#!nix-shell -i bash -p cabal-install "haskell.packages.ghc924.ghcWithPackages (pkgs: with pkgs; [aeson])" git jq
-
-cd "$(dirname "${BASH_SOURCE[0]}")"
-cabal run gen_ci -- metadata jobs-metadata.json

.gitlab/generate_jobs

-#!/usr/bin/env nix-shell
-#!nix-shell -i bash -p cabal-install "haskell.packages.ghc924.ghcWithPackages (pkgs: with pkgs; [aeson])" git jq
-
-# shellcheck shell=bash
-
-set -euo pipefail
-
-cd "$(dirname "${BASH_SOURCE[0]}")"
-tmp=$(mktemp)
-cabal run gen_ci -- gitlab $tmp
-rm -f jobs.yaml
-echo "### THIS IS A GENERATED FILE, DO NOT MODIFY DIRECTLY" > jobs.yaml
-cat $tmp | jq | tee -a jobs.yaml

.gitlab/hello.hs

-{-# OPTIONS_GHC -Wno-missing-fields #-}
+{-# OPTIONS_GHC -Wall -Wno-missing-fields #-}
 
 import GHC hiding (parseModule)
 import GHC.Data.StringBuffer
@@ -9,6 +9,7 @@ import GHC.Platform
 import GHC.Plugins
 import GHC.Settings
 import GHC.Settings.Config
+import System.Mem.Weak
 
 fakeSettings :: Settings
 fakeSettings =
@@ -41,5 +42,6 @@ parse dflags src = do
 
 main :: IO ()
 main = do
+  _ <- mkWeak runGhc runGhc Nothing
   m <- parse fakeDynFlags "main = putStrLn \"hello world\""
   putStrLn $ showSDoc fakeDynFlags $ ppr m

.gitlab/issue_templates/bug.md → .gitlab/issue_templates/default.md

 ## Summary
 
-Write a brief description of the issue.
+Please read the guidance in [https://gitlab.haskell.org/ghc/ghc/-/wikis/report-a-bug] and write a brief description of the issue.
 
 ## Steps to reproduce