diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 704f6fd9ff0a85970fbc941e26471b2c3e18eecd..fcd21949bc1056226a1f7747d2215a9d0993133d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,9 @@ variables: GIT_SSL_NO_VERIFY: "1" # Commit of ghc/ci-images repository from which to pull Docker images - DOCKER_REV: efc1ab81236eb37e20cb287ec77aebb6c6341098 + DOCKER_REV: 66be29543a78357dfcdd90c112989d006dbed0ba + + GHC_WASM_META_BRANCH: master # Sequential version number of all cached things. # Bump to invalidate GitLab CI cache. @@ -41,7 +43,7 @@ variables: # -1 37.72s 142 MB # -0 34.40s 156 MB # - XZ_OPT: "-1" + XZ_OPT: "-9e" default: interruptible: true @@ -97,6 +99,7 @@ workflow: - if: $CI_PROJECT_ID == "1" && $CI_COMMIT_BRANCH == "master" - if: $CI_PROJECT_ID == "1" && $CI_COMMIT_BRANCH =~ /ghc-[0-9]+\.[0-9]+/ - if: '$CI_PIPELINE_SOURCE == "web"' + - if: $CI_PIPELINE_SOURCE == "pipeline" # which versions of GHC to allow bootstrap with .bootstrap_matrix : &bootstrap_matrix @@ -222,30 +225,6 @@ ghc-linters: - if: $CI_MERGE_REQUEST_ID - *drafts-can-fail-lint -# Run mypy Python typechecker on linter scripts. -lint-linters: - image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV" - extends: .lint - script: - - mypy testsuite/tests/linters/regex-linters/*.py - dependencies: [] - -# Check that .T files all parse by listing broken tests. -lint-testsuite: - image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV" - extends: .lint - script: - - make -Ctestsuite list_broken TEST_HC=$GHC - dependencies: [] - -# Run mypy Python typechecker on testsuite driver -typecheck-testsuite: - image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV" - extends: .lint - script: - - mypy testsuite/driver/runtests.py - dependencies: [] - # We allow the submodule checker to fail when run on merge requests (to # accommodate, e.g., haddock changes not yet upstream) but not on `master` or # Marge jobs. @@ -282,49 +261,6 @@ lint-author: - if: $CI_MERGE_REQUEST_ID - *drafts-can-fail-lint -lint-ci-config: - image: nixos/nix:2.12.0 - extends: .lint - # We don't need history/submodules in this job - variables: - GIT_DEPTH: 1 - GIT_SUBMODULE_STRATEGY: none - before_script: - - echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf - - nix-channel --update - script: - - .gitlab/generate_jobs - # 1 if .gitlab/generate_jobs changed the output of the generated config - - nix shell nixpkgs#git -c git diff --exit-code - dependencies: [] - -lint-submods: - extends: .lint-submods - # Allow failure on merge requests since any necessary submodule patches may - # not be upstreamed yet. - rules: - - if: '$CI_MERGE_REQUEST_LABELS =~ /.*marge_bot_batch_merge_job.*/' - allow_failure: false - # Don't run on nightly because the program needs a base commit to check. - - if: $NIGHTLY - when: never - - allow_failure: true - -lint-submods-branch: - extends: .lint-submods - variables: - BUILD_FLAVOUR: default - script: - - .gitlab/ci.sh configure - - .gitlab/ci.sh run_hadrian stage0:exe:lint-submodule-refs - - "echo Linting submodule changes between $CI_COMMIT_BEFORE_SHA..$CI_COMMIT_SHA" - - git submodule foreach git remote update - - _build/stageBoot/bin/lint-submodule-refs . $(git rev-list $CI_COMMIT_BEFORE_SHA..$CI_COMMIT_SHA) - rules: - - if: '$CI_COMMIT_BRANCH == "master"' - - if: '$CI_COMMIT_BRANCH =~ /ghc-[0.9]+\.[0-9]+/' - - *drafts-can-fail-lint - ############################################################ # GHC source code linting ############################################################ @@ -348,17 +284,6 @@ lint-submods-branch: paths: - cabal-cache -hlint-ghc-and-base: - extends: .lint-params - image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV" - variables: - BUILD_FLAVOUR: default - script: - - .gitlab/ci.sh setup - - .gitlab/ci.sh configure - - .gitlab/ci.sh run_hadrian lint:base - - .gitlab/ci.sh run_hadrian lint:compiler - ############################################################ # GHC-in-GHCi (Hadrian) ############################################################ @@ -366,10 +291,8 @@ hlint-ghc-and-base: hadrian-ghc-in-ghci: stage: quick-build needs: - - job: lint-linters - - job: lint-submods - optional: true - image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb10:$DOCKER_REV" + - only-wasm + image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb12:$DOCKER_REV" before_script: # workaround for docker permissions - sudo chown ghc:ghc -R . @@ -444,21 +367,6 @@ hadrian-multi: paths: - cabal-cache -############################################################ -# stack-hadrian-build -############################################################ - -# Verify that Hadrian builds with stack. Note that we don't actually perform a -# build of GHC itself; we merely test that the Hadrian executable builds and -# works (by invoking `hadrian --version`). -stack-hadrian-build: - extends: hadrian-ghc-in-ghci - stage: quick-build - script: - - .gitlab/ci.sh setup - - .gitlab/ci.sh configure - - hadrian/build-stack --version - #################################### # Testing reinstallable ghc codepath #################################### @@ -1111,3 +1019,37 @@ ghcup-metadata-testing-release: rules: - if: '$RELEASE_JOB == "yes"' when: manual + +only-wasm: + stage: quick-build + image: nixos/nix:2.26.1 + tags: + - x86_64-linux + needs: [] + variables: + GIT_STRATEGY: none + KEEP_JOB_NAME: x86_64-linux-alpine3_20-wasm-cross_wasm32-wasi-release+host_fully_static + before_script: + - echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf + - echo "cores = $CPUS" >> /etc/nix/nix.conf + - echo "max-jobs = $CPUS" >> /etc/nix/nix.conf + - nix run nixpkgs#gnused -- -i -e 's/ nixbld//' /etc/nix/nix.conf + script: + - nix run nixpkgs#deno -- run --allow-env --allow-net https://gist.githubusercontent.com/TerrorJack/e0e886b87b9bfffb6c5fa5b3aeddcecc/raw/c927e9ef4a191d042087620965f6eb439749b9a0/cancel.js + +ghc-wasm-meta-ci: + stage: testing + needs: + - job: x86_64-linux-alpine3_20-wasm-cross_wasm32-wasi-release+host_fully_static + artifacts: false + variables: + UPSTREAM_GHC_FLAVOUR: "9.6" + UPSTREAM_GHC_PIPELINE_ID: $CI_PIPELINE_ID + rules: + - if: $UPSTREAM_WASI_SDK_PIPELINE_ID != null + variables: + UPSTREAM_WASI_SDK_PIPELINE_ID: $UPSTREAM_WASI_SDK_PIPELINE_ID + - when: always + trigger: + project: haskell-wasm/ghc-wasm-meta + branch: $GHC_WASM_META_BRANCH diff --git a/.gitlab/ci.sh b/.gitlab/ci.sh index 320bea9c72140bd8a41488a7fd9067543eb390af..a6235af7bf4fae3ef2fd25ab7029424a0ecceeb0 100755 --- a/.gitlab/ci.sh +++ b/.gitlab/ci.sh @@ -240,7 +240,45 @@ function set_toolchain_paths() { export ALEX if [[ "${CROSS_TARGET:-}" == *"wasm"* ]]; then - source "/home/ghc/.ghc-wasm/env" + case "$(uname)" in + Linux) + if [[ ! -f /home/ghc/.ghc-wasm/.flag ]]; then + sudo sed -i -e 's/v3\.[0-9][0-9]/v3\.21/g' /etc/apk/repositories + sudo apk upgrade --available --update-cache + + curl -f -L --retry 5 https://github.com/tweag/rust-alpine-mimalloc/archive/refs/heads/master.tar.gz | tar xz -C /tmp + mv /tmp/rust-alpine-mimalloc-master/mimalloc.diff /tmp + sudo /tmp/rust-alpine-mimalloc-master/build.sh + + pushd "$(mktemp -d)" + curl -f -L --retry 5 https://gitlab.haskell.org/haskell-wasm/ghc-wasm-meta/-/archive/$GHC_WASM_META_BRANCH/ghc-wasm-meta-$GHC_WASM_META_BRANCH.tar.gz | tar xz --strip-components=1 + PREFIX=/home/ghc/.ghc-wasm SKIP_GHC=1 ./setup.sh + popd + + touch /home/ghc/.ghc-wasm/.flag + fi + + export LD_PRELOAD=/usr/lib/libmimalloc.so + source /home/ghc/.ghc-wasm/env + + if [[ "$(uname -m)" == "aarch64" ]]; then + export CONFIGURE_ARGS="--host=aarch64-alpine-linux --target=wasm32-wasi --with-intree-gmp --with-system-libffi" + fi + ;; + + Darwin) + pushd "$(mktemp -d)" + curl -f -L --retry 5 https://gitlab.haskell.org/haskell-wasm/ghc-wasm-meta/-/archive/$GHC_WASM_META_BRANCH/ghc-wasm-meta-$GHC_WASM_META_BRANCH.tar.gz | tar xz --strip-components=1 + PREFIX=/Users/$(whoami)/.ghc-wasm SKIP_GHC=1 ./setup.sh + popd + + source /Users/$(whoami)/.ghc-wasm/env + ;; + + *) + fail "wasm target only supported on linux/darwin hosts" + ;; + esac fi } diff --git a/.gitlab/gen_ci.hs b/.gitlab/gen_ci.hs index 702fb5d9c1882c9d32033f2c74f8e3de14543fc2..53e791e042d2ee98297bfc115f7439eca69520c3 100755 --- a/.gitlab/gen_ci.hs +++ b/.gitlab/gen_ci.hs @@ -151,6 +151,7 @@ data BuildConfig , threadSanitiser :: Bool , noSplitSections :: Bool , validateNonmovingGc :: Bool + , textWithSIMDUTF :: Bool } -- Extra arguments to pass to ./configure due to the BuildConfig @@ -171,13 +172,16 @@ mkJobFlavour BuildConfig{..} = Flavour buildFlavour opts [HostFullyStatic | hostFullyStatic] ++ [ThreadSanitiser | threadSanitiser] ++ [NoSplitSections | noSplitSections, buildFlavour == Release ] ++ - [BootNonmovingGc | validateNonmovingGc ] + [BootNonmovingGc | validateNonmovingGc ] ++ + [TextWithSIMDUTF | textWithSIMDUTF] + data Flavour = Flavour BaseFlavour [FlavourTrans] data FlavourTrans = Llvm | Dwarf | FullyStatic | HostFullyStatic | ThreadSanitiser | NoSplitSections | BootNonmovingGc + | TextWithSIMDUTF data BaseFlavour = Release | Validate | SlowValidate deriving Eq @@ -204,6 +208,7 @@ vanilla = BuildConfig , threadSanitiser = False , noSplitSections = False , validateNonmovingGc = False + , textWithSIMDUTF = False } splitSectionsBroken :: BuildConfig -> BuildConfig @@ -331,6 +336,7 @@ flavourString (Flavour base trans) = baseString base ++ concatMap (("+" ++) . fl flavourString ThreadSanitiser = "thread_sanitizer" flavourString NoSplitSections = "no_split_sections" flavourString BootNonmovingGc = "boot_nonmoving_gc" + flavourString TextWithSIMDUTF = "text_simdutf" -- The path to the docker image (just for linux builders) dockerImage :: Arch -> Opsys -> Maybe String @@ -946,6 +952,7 @@ job_groups = { hostFullyStatic = True , buildFlavour = Release -- TODO: This needs to be validate but wasm backend doesn't pass yet + , textWithSIMDUTF = True } diff --git a/.gitlab/jobs.yaml b/.gitlab/jobs.yaml index a3dcaa6f3a09fc6c5a354c5f9bded3f74d9705d7..d0b580ca02f875670b4beb29badc699805855cf3 100644 --- a/.gitlab/jobs.yaml +++ b/.gitlab/jobs.yaml @@ -671,7 +671,7 @@ "XZ_OPT": "-9" } }, - "nightly-x86_64-linux-alpine3_20-wasm-cross_wasm32-wasi-release+host_fully_static": { + "nightly-x86_64-linux-alpine3_20-wasm-cross_wasm32-wasi-release+host_fully_static+text_simdutf": { "after_script": [ ".gitlab/ci.sh save_cache", ".gitlab/ci.sh clean", @@ -681,7 +681,7 @@ "artifacts": { "expire_in": "8 weeks", "paths": [ - "ghc-x86_64-linux-alpine3_20-wasm-cross_wasm32-wasi-release+host_fully_static.tar.xz", + "ghc-x86_64-linux-alpine3_20-wasm-cross_wasm32-wasi-release+host_fully_static+text_simdutf.tar.xz", "junit.xml" ], "reports": { @@ -723,17 +723,17 @@ ], "variables": { "BIGNUM_BACKEND": "gmp", - "BIN_DIST_NAME": "ghc-x86_64-linux-alpine3_20-wasm-cross_wasm32-wasi-release+host_fully_static", + "BIN_DIST_NAME": "ghc-x86_64-linux-alpine3_20-wasm-cross_wasm32-wasi-release+host_fully_static+text_simdutf", "BROKEN_TESTS": "hpc001 hpc_fork tough tough2", - "BUILD_FLAVOUR": "release+host_fully_static", + "BUILD_FLAVOUR": "release+host_fully_static+text_simdutf", "CONFIGURE_ARGS": "--with-intree-gmp --with-system-libffi", "CROSS_TARGET": "wasm32-wasi", "HADRIAN_ARGS": "--docs=none", - "TEST_ENV": "x86_64-linux-alpine3_20-wasm-cross_wasm32-wasi-release+host_fully_static", + "TEST_ENV": "x86_64-linux-alpine3_20-wasm-cross_wasm32-wasi-release+host_fully_static+text_simdutf", "XZ_OPT": "-9" } }, - "nightly-x86_64-linux-alpine3_20-wasm-int_native-cross_wasm32-wasi-release+host_fully_static": { + "nightly-x86_64-linux-alpine3_20-wasm-int_native-cross_wasm32-wasi-release+host_fully_static+text_simdutf": { "after_script": [ ".gitlab/ci.sh save_cache", ".gitlab/ci.sh clean", @@ -743,7 +743,7 @@ "artifacts": { "expire_in": "8 weeks", "paths": [ - "ghc-x86_64-linux-alpine3_20-wasm-int_native-cross_wasm32-wasi-release+host_fully_static.tar.xz", + "ghc-x86_64-linux-alpine3_20-wasm-int_native-cross_wasm32-wasi-release+host_fully_static+text_simdutf.tar.xz", "junit.xml" ], "reports": { @@ -785,17 +785,17 @@ ], "variables": { "BIGNUM_BACKEND": "native", - "BIN_DIST_NAME": "ghc-x86_64-linux-alpine3_20-wasm-int_native-cross_wasm32-wasi-release+host_fully_static", + "BIN_DIST_NAME": "ghc-x86_64-linux-alpine3_20-wasm-int_native-cross_wasm32-wasi-release+host_fully_static+text_simdutf", "BROKEN_TESTS": "hpc001 hpc_fork tough tough2", - "BUILD_FLAVOUR": "release+host_fully_static", + "BUILD_FLAVOUR": "release+host_fully_static+text_simdutf", "CONFIGURE_ARGS": "--with-intree-gmp --with-system-libffi", "CROSS_TARGET": "wasm32-wasi", "HADRIAN_ARGS": "--docs=none", - "TEST_ENV": "x86_64-linux-alpine3_20-wasm-int_native-cross_wasm32-wasi-release+host_fully_static", + "TEST_ENV": "x86_64-linux-alpine3_20-wasm-int_native-cross_wasm32-wasi-release+host_fully_static+text_simdutf", "XZ_OPT": "-9" } }, - "nightly-x86_64-linux-alpine3_20-wasm-unreg-cross_wasm32-wasi-release+host_fully_static": { + "nightly-x86_64-linux-alpine3_20-wasm-unreg-cross_wasm32-wasi-release+host_fully_static+text_simdutf": { "after_script": [ ".gitlab/ci.sh save_cache", ".gitlab/ci.sh clean", @@ -805,7 +805,7 @@ "artifacts": { "expire_in": "8 weeks", "paths": [ - "ghc-x86_64-linux-alpine3_20-wasm-unreg-cross_wasm32-wasi-release+host_fully_static.tar.xz", + "ghc-x86_64-linux-alpine3_20-wasm-unreg-cross_wasm32-wasi-release+host_fully_static+text_simdutf.tar.xz", "junit.xml" ], "reports": { @@ -847,13 +847,13 @@ ], "variables": { "BIGNUM_BACKEND": "gmp", - "BIN_DIST_NAME": "ghc-x86_64-linux-alpine3_20-wasm-unreg-cross_wasm32-wasi-release+host_fully_static", + "BIN_DIST_NAME": "ghc-x86_64-linux-alpine3_20-wasm-unreg-cross_wasm32-wasi-release+host_fully_static+text_simdutf", "BROKEN_TESTS": "hpc001 hpc_fork tough tough2", - "BUILD_FLAVOUR": "release+host_fully_static", + "BUILD_FLAVOUR": "release+host_fully_static+text_simdutf", "CONFIGURE_ARGS": "--enable-unregisterised --with-intree-gmp --with-system-libffi", "CROSS_TARGET": "wasm32-wasi", "HADRIAN_ARGS": "--docs=none", - "TEST_ENV": "x86_64-linux-alpine3_20-wasm-unreg-cross_wasm32-wasi-release+host_fully_static", + "TEST_ENV": "x86_64-linux-alpine3_20-wasm-unreg-cross_wasm32-wasi-release+host_fully_static+text_simdutf", "XZ_OPT": "-9" } }, @@ -3706,7 +3706,7 @@ "TEST_ENV": "x86_64-linux-alpine3_12-validate+fully_static" } }, - "x86_64-linux-alpine3_20-wasm-cross_wasm32-wasi-release+host_fully_static": { + "x86_64-linux-alpine3_20-wasm-cross_wasm32-wasi-release+host_fully_static+text_simdutf": { "after_script": [ ".gitlab/ci.sh save_cache", ".gitlab/ci.sh clean", @@ -3716,7 +3716,7 @@ "artifacts": { "expire_in": "2 weeks", "paths": [ - "ghc-x86_64-linux-alpine3_20-wasm-cross_wasm32-wasi-release+host_fully_static.tar.xz", + "ghc-x86_64-linux-alpine3_20-wasm-cross_wasm32-wasi-release+host_fully_static+text_simdutf.tar.xz", "junit.xml" ], "reports": { @@ -3758,16 +3758,16 @@ ], "variables": { "BIGNUM_BACKEND": "gmp", - "BIN_DIST_NAME": "ghc-x86_64-linux-alpine3_20-wasm-cross_wasm32-wasi-release+host_fully_static", + "BIN_DIST_NAME": "ghc-x86_64-linux-alpine3_20-wasm-cross_wasm32-wasi-release+host_fully_static+text_simdutf", "BROKEN_TESTS": "hpc001 hpc_fork tough tough2", - "BUILD_FLAVOUR": "release+host_fully_static", + "BUILD_FLAVOUR": "release+host_fully_static+text_simdutf", "CONFIGURE_ARGS": "--with-intree-gmp --with-system-libffi", "CROSS_TARGET": "wasm32-wasi", "HADRIAN_ARGS": "--docs=none", - "TEST_ENV": "x86_64-linux-alpine3_20-wasm-cross_wasm32-wasi-release+host_fully_static" + "TEST_ENV": "x86_64-linux-alpine3_20-wasm-cross_wasm32-wasi-release+host_fully_static+text_simdutf" } }, - "x86_64-linux-alpine3_20-wasm-int_native-cross_wasm32-wasi-release+host_fully_static": { + "x86_64-linux-alpine3_20-wasm-int_native-cross_wasm32-wasi-release+host_fully_static+text_simdutf": { "after_script": [ ".gitlab/ci.sh save_cache", ".gitlab/ci.sh clean", @@ -3777,7 +3777,7 @@ "artifacts": { "expire_in": "2 weeks", "paths": [ - "ghc-x86_64-linux-alpine3_20-wasm-int_native-cross_wasm32-wasi-release+host_fully_static.tar.xz", + "ghc-x86_64-linux-alpine3_20-wasm-int_native-cross_wasm32-wasi-release+host_fully_static+text_simdutf.tar.xz", "junit.xml" ], "reports": { @@ -3820,16 +3820,16 @@ ], "variables": { "BIGNUM_BACKEND": "native", - "BIN_DIST_NAME": "ghc-x86_64-linux-alpine3_20-wasm-int_native-cross_wasm32-wasi-release+host_fully_static", + "BIN_DIST_NAME": "ghc-x86_64-linux-alpine3_20-wasm-int_native-cross_wasm32-wasi-release+host_fully_static+text_simdutf", "BROKEN_TESTS": "hpc001 hpc_fork tough tough2", - "BUILD_FLAVOUR": "release+host_fully_static", + "BUILD_FLAVOUR": "release+host_fully_static+text_simdutf", "CONFIGURE_ARGS": "--with-intree-gmp --with-system-libffi", "CROSS_TARGET": "wasm32-wasi", "HADRIAN_ARGS": "--docs=none", - "TEST_ENV": "x86_64-linux-alpine3_20-wasm-int_native-cross_wasm32-wasi-release+host_fully_static" + "TEST_ENV": "x86_64-linux-alpine3_20-wasm-int_native-cross_wasm32-wasi-release+host_fully_static+text_simdutf" } }, - "x86_64-linux-alpine3_20-wasm-unreg-cross_wasm32-wasi-release+host_fully_static": { + "x86_64-linux-alpine3_20-wasm-unreg-cross_wasm32-wasi-release+host_fully_static+text_simdutf": { "after_script": [ ".gitlab/ci.sh save_cache", ".gitlab/ci.sh clean", @@ -3839,7 +3839,7 @@ "artifacts": { "expire_in": "2 weeks", "paths": [ - "ghc-x86_64-linux-alpine3_20-wasm-unreg-cross_wasm32-wasi-release+host_fully_static.tar.xz", + "ghc-x86_64-linux-alpine3_20-wasm-unreg-cross_wasm32-wasi-release+host_fully_static+text_simdutf.tar.xz", "junit.xml" ], "reports": { @@ -3882,13 +3882,13 @@ ], "variables": { "BIGNUM_BACKEND": "gmp", - "BIN_DIST_NAME": "ghc-x86_64-linux-alpine3_20-wasm-unreg-cross_wasm32-wasi-release+host_fully_static", + "BIN_DIST_NAME": "ghc-x86_64-linux-alpine3_20-wasm-unreg-cross_wasm32-wasi-release+host_fully_static+text_simdutf", "BROKEN_TESTS": "hpc001 hpc_fork tough tough2", - "BUILD_FLAVOUR": "release+host_fully_static", + "BUILD_FLAVOUR": "release+host_fully_static+text_simdutf", "CONFIGURE_ARGS": "--enable-unregisterised --with-intree-gmp --with-system-libffi", "CROSS_TARGET": "wasm32-wasi", "HADRIAN_ARGS": "--docs=none", - "TEST_ENV": "x86_64-linux-alpine3_20-wasm-unreg-cross_wasm32-wasi-release+host_fully_static" + "TEST_ENV": "x86_64-linux-alpine3_20-wasm-unreg-cross_wasm32-wasi-release+host_fully_static+text_simdutf" } }, "x86_64-linux-deb10-int_native-validate": {