diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bf41e260a616ac50a82c04c96b53b7d02c5f8e41..4d609a09fd685f3581445d512eeb29aa5fc2a6be 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,9 @@ variables: GIT_SSL_NO_VERIFY: "1" # Commit of ghc/ci-images repository from which to pull Docker images - DOCKER_REV: efc1ab81236eb37e20cb287ec77aebb6c6341098 + DOCKER_REV: 66be29543a78357dfcdd90c112989d006dbed0ba + + GHC_WASM_META_BRANCH: master # Sequential version number of all cached things. # Bump to invalidate GitLab CI cache. @@ -97,6 +99,7 @@ workflow: - if: $CI_PROJECT_ID == "1" && $CI_COMMIT_BRANCH == "master" - if: $CI_PROJECT_ID == "1" && $CI_COMMIT_BRANCH =~ /ghc-[0-9]+\.[0-9]+/ - if: '$CI_PIPELINE_SOURCE == "web"' + - if: $CI_PIPELINE_SOURCE == "pipeline" # which versions of GHC to allow bootstrap with .bootstrap_matrix : &bootstrap_matrix @@ -228,30 +231,6 @@ ghc-linters: - if: $CI_MERGE_REQUEST_ID - *drafts-can-fail-lint -# Run mypy Python typechecker on linter scripts. -lint-linters: - image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV" - extends: .lint - script: - - mypy testsuite/tests/linters/regex-linters/*.py - dependencies: [] - -# Check that .T files all parse by listing broken tests. -lint-testsuite: - image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV" - extends: .lint - script: - - make -Ctestsuite list_broken TEST_HC=$GHC - dependencies: [] - -# Run mypy Python typechecker on testsuite driver -typecheck-testsuite: - image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV" - extends: .lint - script: - - mypy testsuite/driver/runtests.py - dependencies: [] - # We allow the submodule checker to fail when run on merge requests (to # accommodate, e.g., haddock changes not yet upstream) but not on `master` or # Marge jobs. @@ -288,55 +267,6 @@ lint-author: - if: $CI_MERGE_REQUEST_ID - *drafts-can-fail-lint -lint-ci-config: - image: nixos/nix:2.14.1 - extends: .lint - # We don't need history/submodules in this job - variables: - GIT_DEPTH: 1 - GIT_SUBMODULE_STRATEGY: none - before_script: - - echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf - script: - - nix run .gitlab/generate-ci#generate-jobs - # 1 if .gitlab/generate_jobs changed the output of the generated config - - nix shell nixpkgs#git -c git diff --exit-code - # And run this to generate the .gitlab/jobs-metadata.json - - nix run .gitlab/generate-ci#generate-job-metadata - artifacts: - when: always - paths: - - .gitlab/jobs-metadata.json - - .gitlab/jobs.yaml - dependencies: [] - -lint-submods: - extends: .lint-submods - # Allow failure on merge requests since any necessary submodule patches may - # not be upstreamed yet. - rules: - - if: '$CI_MERGE_REQUEST_LABELS =~ /.*marge_bot_batch_merge_job.*/' - allow_failure: false - # Don't run on nightly because the program needs a base commit to check. - - if: $NIGHTLY - when: never - - allow_failure: true - -lint-submods-branch: - extends: .lint-submods - variables: - BUILD_FLAVOUR: default - script: - - .gitlab/ci.sh configure - - .gitlab/ci.sh run_hadrian stage0:exe:lint-submodule-refs - - "echo Linting submodule changes between $CI_COMMIT_BEFORE_SHA..$CI_COMMIT_SHA" - - git submodule foreach git remote update - - _build/stageBoot/bin/lint-submodule-refs . $(git rev-list $CI_COMMIT_BEFORE_SHA..$CI_COMMIT_SHA) - rules: - - if: '$CI_COMMIT_BRANCH == "master"' - - if: '$CI_COMMIT_BRANCH =~ /ghc-[0.9]+\.[0-9]+/' - - *drafts-can-fail-lint - ############################################################ # GHC source code linting ############################################################ @@ -381,10 +311,8 @@ lint-submods-branch: hadrian-ghc-in-ghci: stage: quick-build needs: - - job: lint-linters - - job: lint-submods - optional: true - image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb10:$DOCKER_REV" + - only-wasm + image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb12:$DOCKER_REV" before_script: # workaround for docker permissions - sudo chown ghc:ghc -R . @@ -464,21 +392,6 @@ hadrian-multi: rules: - *full-ci -############################################################ -# stack-hadrian-build -############################################################ - -# Verify that Hadrian builds with stack. Note that we don't actually perform a -# build of GHC itself; we merely test that the Hadrian executable builds and -# works (by invoking `hadrian --version`). -stack-hadrian-build: - extends: hadrian-ghc-in-ghci - stage: quick-build - script: - - .gitlab/ci.sh setup - - .gitlab/ci.sh configure - - hadrian/build-stack --version - #################################### # Testing reinstallable ghc codepath #################################### @@ -1077,27 +990,6 @@ pages: # Generation of GHCUp metadata ############################################################# - -project-version: - stage: packaging - image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb10:$DOCKER_REV" - tags: - - x86_64-linux - variables: - BUILD_FLAVOUR: default - script: - # Calculate the project version - - sudo chown ghc:ghc -R . - - .gitlab/ci.sh setup - - .gitlab/ci.sh configure - - echo "ProjectVersion=$(cat VERSION)" > version.sh - - needs: [] - dependencies: [] - artifacts: - paths: - - version.sh - .ghcup-metadata: stage: deploy image: nixos/nix:2.14.1 @@ -1159,7 +1051,6 @@ ghcup-metadata-nightly: artifacts: false - job: source-tarball artifacts: false - - job: project-version script: - nix shell --extra-experimental-features nix-command -f .gitlab/rel_eng -c ghcup-metadata --metadata ghcup-0.0.7.yaml --date="$(date -d $CI_PIPELINE_CREATED_AT +%Y-%m-%d)" --pipeline-id="$CI_PIPELINE_ID" --version="$ProjectVersion" > "metadata_test.yaml" rules: @@ -1243,3 +1134,37 @@ ghcup-metadata-testing-release: rules: - if: '$RELEASE_JOB == "yes"' when: manual + +only-wasm: + stage: quick-build + image: nixos/nix:2.26.1 + tags: + - x86_64-linux + needs: [] + variables: + GIT_STRATEGY: none + KEEP_JOB_NAME: x86_64-linux-alpine3_20-wasm-cross_wasm32-wasi-release+host_fully_static + before_script: + - echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf + - echo "cores = $CPUS" >> /etc/nix/nix.conf + - echo "max-jobs = $CPUS" >> /etc/nix/nix.conf + - nix run nixpkgs#gnused -- -i -e 's/ nixbld//' /etc/nix/nix.conf + script: + - nix run nixpkgs#deno -- run --allow-env --allow-net https://gist.githubusercontent.com/TerrorJack/e0e886b87b9bfffb6c5fa5b3aeddcecc/raw/c927e9ef4a191d042087620965f6eb439749b9a0/cancel.js + +ghc-wasm-meta-ci: + stage: testing + needs: + - job: x86_64-linux-alpine3_20-wasm-cross_wasm32-wasi-release+host_fully_static + artifacts: false + variables: + UPSTREAM_GHC_FLAVOUR: "9.10" + UPSTREAM_GHC_PIPELINE_ID: $CI_PIPELINE_ID + rules: + - if: $UPSTREAM_WASI_SDK_PIPELINE_ID != null + variables: + UPSTREAM_WASI_SDK_PIPELINE_ID: $UPSTREAM_WASI_SDK_PIPELINE_ID + - when: always + trigger: + project: haskell-wasm/ghc-wasm-meta + branch: $GHC_WASM_META_BRANCH diff --git a/.gitlab/ci.sh b/.gitlab/ci.sh index beced53268e5f7158b026680231fc36614b971a5..a2e72574bfe0c93155c9b1570eec87f74fb3857e 100755 --- a/.gitlab/ci.sh +++ b/.gitlab/ci.sh @@ -225,7 +225,45 @@ function set_toolchain_paths() { export ALEX if [[ "${CROSS_TARGET:-}" == *"wasm"* ]]; then - source "/home/ghc/.ghc-wasm/env" + case "$(uname)" in + Linux) + if [[ ! -f /home/ghc/.ghc-wasm/.flag ]]; then + sudo sed -i -e 's/v3\.[0-9][0-9]/v3\.21/g' /etc/apk/repositories + sudo apk upgrade --available --update-cache + + curl -f -L --retry 5 https://github.com/tweag/rust-alpine-mimalloc/archive/refs/heads/master.tar.gz | tar xz -C /tmp + mv /tmp/rust-alpine-mimalloc-master/mimalloc.diff /tmp + sudo /tmp/rust-alpine-mimalloc-master/build.sh + + pushd "$(mktemp -d)" + curl -f -L --retry 5 https://gitlab.haskell.org/haskell-wasm/ghc-wasm-meta/-/archive/$GHC_WASM_META_BRANCH/ghc-wasm-meta-$GHC_WASM_META_BRANCH.tar.gz | tar xz --strip-components=1 + PREFIX=/home/ghc/.ghc-wasm SKIP_GHC=1 ./setup.sh + popd + + touch /home/ghc/.ghc-wasm/.flag + fi + + export LD_PRELOAD=/usr/lib/libmimalloc.so + source /home/ghc/.ghc-wasm/env + + if [[ "$(uname -m)" == "aarch64" ]]; then + export CONFIGURE_ARGS="--host=aarch64-alpine-linux --target=wasm32-wasi --with-intree-gmp --with-system-libffi" + fi + ;; + + Darwin) + pushd "$(mktemp -d)" + curl -f -L --retry 5 https://gitlab.haskell.org/haskell-wasm/ghc-wasm-meta/-/archive/$GHC_WASM_META_BRANCH/ghc-wasm-meta-$GHC_WASM_META_BRANCH.tar.gz | tar xz --strip-components=1 + PREFIX=/Users/$(whoami)/.ghc-wasm SKIP_GHC=1 ./setup.sh + popd + + source /Users/$(whoami)/.ghc-wasm/env + ;; + + *) + fail "wasm target only supported on linux/darwin hosts" + ;; + esac fi } @@ -496,6 +534,10 @@ function build_hadrian() { export XZ_OPT="${XZ_OPT:-} -T$cores" fi + if [[ "$(uname -m)-$(uname)" != "aarch64-Linux" ]]; then + export WASM_SO_OPT=1 + fi + if [[ -n "${REINSTALL_GHC:-}" ]]; then run_hadrian build-cabal -V else diff --git a/.gitlab/generate-ci/gen_ci.hs b/.gitlab/generate-ci/gen_ci.hs index ce03166575a4e63e441545b67adcdf931be21cb6..332b5da58f6885b3b46f90456556189110f87e5e 100644 --- a/.gitlab/generate-ci/gen_ci.hs +++ b/.gitlab/generate-ci/gen_ci.hs @@ -1025,6 +1025,8 @@ job_groups = make_wasm_jobs wasm_build_config {bignumBackend = Native} , modifyValidateJobs manual $ make_wasm_jobs wasm_build_config {unregisterised = True} + , wasm_darwin_jobs + , wasm_aarch64_jobs , onlyRule NonmovingGc (validateBuilds Amd64 (Linux Debian11) vanilla {validateNonmovingGc = True}) , onlyRule IpeData (validateBuilds Amd64 (Linux Debian10) zstdIpe) ] @@ -1049,6 +1051,22 @@ job_groups = . addVariable "HADRIAN_ARGS" "--docs=none") $ validateBuilds Amd64 (Linux Debian12) tsan + wasm_aarch64_jobs = + modifyJobs + ( delVariable "BROKEN_TESTS" + . setVariable "HADRIAN_ARGS" "--docs=none" + . delVariable "INSTALL_CONFIGURE_ARGS" + ) + $ validateBuilds AArch64 (Linux Alpine318) wasm_build_config + + wasm_darwin_jobs = + modifyJobs + ( setVariable "HADRIAN_ARGS" "--docs=none" + . delVariable "INSTALL_CONFIGURE_ARGS" + ) + $ validateBuilds AArch64 Darwin + $ wasm_build_config { hostFullyStatic = False } + make_wasm_jobs cfg = modifyJobs ( delVariable "BROKEN_TESTS" diff --git a/.gitlab/jobs.yaml b/.gitlab/jobs.yaml index ec1f4cd2919a83d823c5c26bad72f65e0c3996b0..2d5ea77b4120213f606eac42aabbf9b72e4e83ae 100644 --- a/.gitlab/jobs.yaml +++ b/.gitlab/jobs.yaml @@ -1,5 +1,71 @@ ### THIS IS A GENERATED FILE, DO NOT MODIFY DIRECTLY { + "aarch64-darwin-cross_wasm32-wasi-release": { + "after_script": [ + ".gitlab/ci.sh save_cache", + ".gitlab/ci.sh save_test_output", + ".gitlab/ci.sh clean", + "cat ci_timings" + ], + "allow_failure": false, + "artifacts": { + "expire_in": "2 weeks", + "paths": [ + "ghc-aarch64-darwin-cross_wasm32-wasi-release.tar.xz", + "junit.xml", + "unexpected-test-output.tar.gz" + ], + "reports": { + "junit": "junit.xml" + }, + "when": "always" + }, + "cache": { + "key": "aarch64-darwin-$CACHE_REV", + "paths": [ + "cabal-cache", + "toolchain" + ] + }, + "dependencies": [], + "image": null, + "needs": [ + { + "artifacts": false, + "job": "hadrian-ghc-in-ghci" + } + ], + "rules": [ + { + "if": "((($CI_MERGE_REQUEST_LABELS =~ /.*full-ci.*/) || ($CI_MERGE_REQUEST_LABELS =~ /.*marge_bot_batch_merge_job.*/) || ($CI_COMMIT_BRANCH == \"master\") || ($CI_COMMIT_BRANCH =~ /ghc-[0-9]+\\.[0-9]+/))) && ($RELEASE_JOB != \"yes\") && ($NIGHTLY == null)", + "when": "on_success" + } + ], + "script": [ + "find libraries -name config.sub -exec cp config.sub {} \\;", + ".gitlab/ci.sh setup", + ".gitlab/ci.sh configure", + ".gitlab/ci.sh build_hadrian", + ".gitlab/ci.sh test_hadrian" + ], + "stage": "full-build", + "tags": [ + "aarch64-darwin" + ], + "variables": { + "BIGNUM_BACKEND": "gmp", + "BIN_DIST_NAME": "ghc-aarch64-darwin-cross_wasm32-wasi-release", + "BUILD_FLAVOUR": "release", + "CONFIGURE_ARGS": "--with-intree-gmp --with-system-libffi --with-intree-gmp --with-system-libffi --enable-strict-ghc-toolchain-check", + "CROSS_TARGET": "wasm32-wasi", + "HADRIAN_ARGS": "--docs=none", + "LANG": "en_US.UTF-8", + "MACOSX_DEPLOYMENT_TARGET": "11.0", + "NIX_SYSTEM": "aarch64-darwin", + "RUNTEST_ARGS": "", + "TEST_ENV": "aarch64-darwin-cross_wasm32-wasi-release" + } + }, "aarch64-darwin-validate": { "after_script": [ ".gitlab/ci.sh save_cache", @@ -65,6 +131,69 @@ "TEST_ENV": "aarch64-darwin-validate" } }, + "aarch64-linux-alpine3_18-cross_wasm32-wasi-release+host_fully_static": { + "after_script": [ + ".gitlab/ci.sh save_cache", + ".gitlab/ci.sh save_test_output", + ".gitlab/ci.sh clean", + "cat ci_timings" + ], + "allow_failure": false, + "artifacts": { + "expire_in": "2 weeks", + "paths": [ + "ghc-aarch64-linux-alpine3_18-cross_wasm32-wasi-release+host_fully_static.tar.xz", + "junit.xml", + "unexpected-test-output.tar.gz" + ], + "reports": { + "junit": "junit.xml" + }, + "when": "always" + }, + "cache": { + "key": "aarch64-linux-alpine3_18-$CACHE_REV", + "paths": [ + "cabal-cache", + "toolchain" + ] + }, + "dependencies": [], + "image": "registry.gitlab.haskell.org/ghc/ci-images/aarch64-linux-alpine3_18:$DOCKER_REV", + "needs": [ + { + "artifacts": false, + "job": "hadrian-ghc-in-ghci" + } + ], + "rules": [ + { + "if": "((($CI_MERGE_REQUEST_LABELS =~ /.*full-ci.*/) || ($CI_MERGE_REQUEST_LABELS =~ /.*marge_bot_batch_merge_job.*/) || ($CI_COMMIT_BRANCH == \"master\") || ($CI_COMMIT_BRANCH =~ /ghc-[0-9]+\\.[0-9]+/))) && ($RELEASE_JOB != \"yes\") && ($NIGHTLY == null)", + "when": "on_success" + } + ], + "script": [ + "sudo chown ghc:ghc -R .", + ".gitlab/ci.sh setup", + ".gitlab/ci.sh configure", + ".gitlab/ci.sh build_hadrian", + ".gitlab/ci.sh test_hadrian" + ], + "stage": "full-build", + "tags": [ + "aarch64-linux" + ], + "variables": { + "BIGNUM_BACKEND": "gmp", + "BIN_DIST_NAME": "ghc-aarch64-linux-alpine3_18-cross_wasm32-wasi-release+host_fully_static", + "BUILD_FLAVOUR": "release+host_fully_static", + "CONFIGURE_ARGS": "--disable-ld-override --with-intree-gmp --with-system-libffi --enable-strict-ghc-toolchain-check", + "CROSS_TARGET": "wasm32-wasi", + "HADRIAN_ARGS": "--docs=none", + "RUNTEST_ARGS": "", + "TEST_ENV": "aarch64-linux-alpine3_18-cross_wasm32-wasi-release+host_fully_static" + } + }, "aarch64-linux-deb10-validate": { "after_script": [ ".gitlab/ci.sh save_cache", @@ -248,6 +377,73 @@ "TEST_ENV": "i386-linux-deb10-validate" } }, + "nightly-aarch64-darwin-cross_wasm32-wasi-release": { + "after_script": [ + ".gitlab/ci.sh save_cache", + ".gitlab/ci.sh save_test_output", + ".gitlab/ci.sh clean", + "cat ci_timings" + ], + "allow_failure": false, + "artifacts": { + "expire_in": "8 weeks", + "paths": [ + "ghc-aarch64-darwin-cross_wasm32-wasi-release.tar.xz", + "junit.xml", + "unexpected-test-output.tar.gz" + ], + "reports": { + "junit": "junit.xml" + }, + "when": "always" + }, + "cache": { + "key": "aarch64-darwin-$CACHE_REV", + "paths": [ + "cabal-cache", + "toolchain" + ] + }, + "dependencies": [], + "image": null, + "needs": [ + { + "artifacts": false, + "job": "hadrian-ghc-in-ghci" + } + ], + "rules": [ + { + "if": "(\"true\" == \"true\") && ($RELEASE_JOB != \"yes\") && ($NIGHTLY)", + "when": "on_success" + } + ], + "script": [ + "find libraries -name config.sub -exec cp config.sub {} \\;", + ".gitlab/ci.sh setup", + ".gitlab/ci.sh configure", + ".gitlab/ci.sh build_hadrian", + ".gitlab/ci.sh test_hadrian" + ], + "stage": "full-build", + "tags": [ + "aarch64-darwin" + ], + "variables": { + "BIGNUM_BACKEND": "gmp", + "BIN_DIST_NAME": "ghc-aarch64-darwin-cross_wasm32-wasi-release", + "BUILD_FLAVOUR": "release", + "CONFIGURE_ARGS": "--with-intree-gmp --with-system-libffi --with-intree-gmp --with-system-libffi --enable-strict-ghc-toolchain-check", + "CROSS_TARGET": "wasm32-wasi", + "HADRIAN_ARGS": "--docs=none", + "LANG": "en_US.UTF-8", + "MACOSX_DEPLOYMENT_TARGET": "11.0", + "NIX_SYSTEM": "aarch64-darwin", + "RUNTEST_ARGS": "", + "TEST_ENV": "aarch64-darwin-cross_wasm32-wasi-release", + "XZ_OPT": "-9" + } + }, "nightly-aarch64-darwin-validate": { "after_script": [ ".gitlab/ci.sh save_cache", @@ -314,6 +510,70 @@ "XZ_OPT": "-9" } }, + "nightly-aarch64-linux-alpine3_18-cross_wasm32-wasi-release+host_fully_static": { + "after_script": [ + ".gitlab/ci.sh save_cache", + ".gitlab/ci.sh save_test_output", + ".gitlab/ci.sh clean", + "cat ci_timings" + ], + "allow_failure": false, + "artifacts": { + "expire_in": "8 weeks", + "paths": [ + "ghc-aarch64-linux-alpine3_18-cross_wasm32-wasi-release+host_fully_static.tar.xz", + "junit.xml", + "unexpected-test-output.tar.gz" + ], + "reports": { + "junit": "junit.xml" + }, + "when": "always" + }, + "cache": { + "key": "aarch64-linux-alpine3_18-$CACHE_REV", + "paths": [ + "cabal-cache", + "toolchain" + ] + }, + "dependencies": [], + "image": "registry.gitlab.haskell.org/ghc/ci-images/aarch64-linux-alpine3_18:$DOCKER_REV", + "needs": [ + { + "artifacts": false, + "job": "hadrian-ghc-in-ghci" + } + ], + "rules": [ + { + "if": "(\"true\" == \"true\") && ($RELEASE_JOB != \"yes\") && ($NIGHTLY)", + "when": "on_success" + } + ], + "script": [ + "sudo chown ghc:ghc -R .", + ".gitlab/ci.sh setup", + ".gitlab/ci.sh configure", + ".gitlab/ci.sh build_hadrian", + ".gitlab/ci.sh test_hadrian" + ], + "stage": "full-build", + "tags": [ + "aarch64-linux" + ], + "variables": { + "BIGNUM_BACKEND": "gmp", + "BIN_DIST_NAME": "ghc-aarch64-linux-alpine3_18-cross_wasm32-wasi-release+host_fully_static", + "BUILD_FLAVOUR": "release+host_fully_static", + "CONFIGURE_ARGS": "--disable-ld-override --with-intree-gmp --with-system-libffi --enable-strict-ghc-toolchain-check", + "CROSS_TARGET": "wasm32-wasi", + "HADRIAN_ARGS": "--docs=none", + "RUNTEST_ARGS": "", + "TEST_ENV": "aarch64-linux-alpine3_18-cross_wasm32-wasi-release+host_fully_static", + "XZ_OPT": "-9" + } + }, "nightly-aarch64-linux-alpine3_18-validate": { "after_script": [ ".gitlab/ci.sh save_cache",