.gitlab-ci.yml

variables:
  GIT_SSL_NO_VERIFY: "1"

  # Commit of ghc/ci-images repository from which to pull Docker images
  DOCKER_REV: 69abb3614435999fd0895bc8a231b7494063e20c

  # Sequential version number capturing the versions of all tools fetched by
  # .gitlab/win32-init.sh.
  WINDOWS_TOOLCHAIN_VERSION: 1

  # Disable shallow clones; they break our linting rules
  GIT_DEPTH: 0

before_script:
  - git submodule sync --recursive
  - git submodule update --init --recursive
  - git checkout .gitmodules
  - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"

stages:
  - lint        # Source linting
  - quick-build # A very quick smoke-test to weed out broken commits
  - build       # A quick smoke-test to weed out broken commits
  - full-build  # Build all the things
  - cleanup     # See Note [Cleanup after the shell executor]
  - packaging   # Source distribution, etc.
  - testing     # head.hackage correctness and compiler performance testing
  - deploy      # push documentation

# N.B.Don't run on wip/ branches, instead on run on merge requests.
.only-default: &only-default
  only:
    - master
    - /ghc-[0-9]+\.[0-9]+/
    - merge_requests
    - tags
    - web

############################################################
# Runner Tags
############################################################
#
# * x86_64-linux: Any Docker-capable x86_64 Linux machine
# * aarch64-linux: Any Docker-capable AArch64 Linux machine
# * x86_64-windows: A x86_64 Windows machine
# * lint: Any Docker-capable x86_64 Linux machine; distinct from
#         x86_64-linux to ensure low-latency availability.
#


############################################################
# Linting
############################################################

ghc-linters:
  stage: lint
  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
  script:
    - git fetch "$CI_MERGE_REQUEST_PROJECT_URL" $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
    - base="$(git merge-base FETCH_HEAD $CI_COMMIT_SHA)"
    - "echo Linting changes between $base..$CI_COMMIT_SHA"
    #    - validate-commit-msg .git $(git rev-list $base..$CI_COMMIT_SHA)
    - validate-whitespace .git $(git rev-list $base..$CI_COMMIT_SHA)
    - .gitlab/linters/check-makefiles.py commits $base $CI_COMMIT_SHA
    - .gitlab/linters/check-cpp.py commits $base $CI_COMMIT_SHA
    - .gitlab/linters/check-version-number.sh
    - python3 utils/checkUniques/check-uniques.py .
  dependencies: []
  tags:
    - lint
  only:
    refs:
      - merge_requests

# Run mypy Python typechecker on linter scripts.
lint-linters:
  <<: *only-default
  stage: lint
  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
  script:
    - mypy .gitlab/linters/*.py
  dependencies: []
  tags:
    - lint

# Check that .T files all parse by listing broken tests.
lint-testsuite:
  <<: *only-default
  stage: lint
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
  script:
    - make -Ctestsuite list_broken TEST_HC=ghc
  dependencies: []
  tags:
    - lint

# Run mypy Python typechecker on testsuite driver
typecheck-testsuite:
  <<: *only-default
  stage: lint
  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
  script:
    - mypy testsuite/driver/runtests.py
  dependencies: []
  tags:
    - lint

# We allow the submodule checker to fail when run on merge requests (to
# accomodate, e.g., haddock changes not yet upstream) but not on `master` or
# Marge jobs.
.lint-submods:
  <<: *only-default
  stage: lint
  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
  script:
    - git fetch "$CI_MERGE_REQUEST_PROJECT_URL" $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
    - base="$(git merge-base FETCH_HEAD $CI_COMMIT_SHA)"
    - "echo Linting submodule changes between $base..$CI_COMMIT_SHA"
    - git submodule foreach git remote update
      # TODO: Fix submodule linter
    - submodchecker .git $(git rev-list $base..$CI_COMMIT_SHA) || true
  dependencies: []
  tags:
    - lint

lint-submods-marge:
  extends: .lint-submods
  only:
    refs:
      - merge_requests
    variables:
      - "$CI_MERGE_REQUEST_LABELS =~ /.*marge_bot_batch_merge_job.*/"

lint-submods-mr:
  extends: .lint-submods
  # Allow failure since any necessary submodule patches may not be upstreamed
  # yet.
  allow_failure: true
  only:
    refs:
      - merge_requests
  except:
    variables:
      - "$CI_MERGE_REQUEST_LABELS =~ /.*marge_bot_batch_merge_job.*/"

lint-submods-branch:
  extends: .lint-submods
  script:
    - "echo Linting submodule changes between $CI_COMMIT_BEFORE_SHA..$CI_COMMIT_SHA"
    - git submodule foreach git remote update
    - submodchecker .git $(git rev-list $CI_COMMIT_BEFORE_SHA..$CI_COMMIT_SHA)
  only:
    refs:
      - master
      - /ghc-[0-9]+\.[0-9]+/

.lint-changelogs:
  <<: *only-default
  stage: lint
  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
  dependencies: []
  tags:
    - lint
  script:
    - |
      grep TBA libraries/*/changelog.md && (
          echo "Error: Found \"TBA\"s in changelogs."
          exit 1
      )

lint-changelogs:
  extends: .lint-changelogs
  # Allow failure since this isn't a final release.
  allow_failure: true
  only:
    refs:
      - /ghc-[0-9]+\.[0-9]+/

lint-release-changelogs:
  extends: .lint-changelogs
  only:
    refs:
      - /ghc-[0-9]+\.[0-9]+\.[0-9]+-.*/


############################################################
# Validation via Pipelines (hadrian)
############################################################

.validate-hadrian:
  <<: *only-default
  variables:
    FLAVOUR: "validate"
  script:
    - cabal update
    - git clean -xdf && git submodule foreach git clean -xdf
    - .gitlab/prepare-system.sh
    - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
    - ./boot
    - ./configure $CONFIGURE_ARGS
    - hadrian/build.cabal.sh --flavour=$FLAVOUR -j`mk/detect-cpu-count.sh` --docs=no-sphinx binary-dist
    - mv _build/bindist/ghc*.tar.xz ghc.tar.xz
    - export TOP=$(pwd)
    - cd _build/bindist/ghc-*/ && ./configure --prefix=$TOP/_build/install && make install && cd ../../../
    - |
      # Prepare to push git notes.
      METRICS_FILE=$CI_PROJECT_DIR/performance-metrics.tsv
      git config user.email "ben+ghc-ci@smart-cactus.org"
      git config user.name "GHC GitLab CI"
    - hadrian/build.cabal.sh --flavour=$FLAVOUR -j`mk/detect-cpu-count.sh` --docs=no-sphinx test --summary-junit=./junit.xml --test-compiler=$TOP/_build/install/bin/ghc || (METRICS_FILE=$METRICS_FILE .gitlab/push-test-metrics.sh && false)
    - |
      # Push git notes.
      METRICS_FILE=$METRICS_FILE .gitlab/push-test-metrics.sh
  cache:
    key: hadrian
    paths:
      - cabal-cache
  artifacts:
    reports:
      junit: junit.xml
    expire_in: 2 week
    paths:
      - ghc.tar.xz
      - junit.xml

.validate-linux-hadrian:
  extends: .validate-hadrian
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
  variables:
    TEST_ENV: "x86_64-linux-deb9-hadrian"
  before_script:
    # workaround for docker permissions
    - sudo chown ghc:ghc -R .
    - git submodule sync --recursive
    - git submodule update --init --recursive
    - git checkout .gitmodules
    - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
  tags:
    - x86_64-linux

validate-x86_64-linux-deb9-hadrian:
  extends: .validate-linux-hadrian
  stage: build

validate-x86_64-linux-deb9-unreg-hadrian:
  extends: .validate-linux-hadrian
  stage: full-build
  variables:
    CONFIGURE_ARGS: --enable-unregisterised
    TEST_ENV: "x86_64-linux-deb9-unreg-hadrian"

hadrian-ghc-in-ghci:
  <<: *only-default
  stage: quick-build
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
  before_script:
    # workaround for docker permissions
    - sudo chown ghc:ghc -R .
    - git submodule sync --recursive
    - git submodule update --init --recursive
    - git checkout .gitmodules
  variables:
    GHC_FLAGS: -Werror
  tags:
    - x86_64-linux
  script:
    - cabal update
    - cd hadrian; cabal new-build --project-file=ci.project; cd ..
    - git clean -xdf && git submodule foreach git clean -xdf
    - .gitlab/prepare-system.sh
    - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
    - ./boot
    - ./configure $CONFIGURE_ARGS
    # Load ghc-in-ghci then immediately exit and check the modules loaded
    - echo ":q" | hadrian/ghci.sh -j`mk/detect-cpu-count.sh`| tail -n2 | grep "Ok,"
  cache:
    key: hadrian-ghci
    paths:
      - cabal-cache

############################################################
# Validation via Pipelines (make)
############################################################

.validate:
  <<: *only-default
  variables:
    TEST_TYPE: test
  before_script:
    - git clean -xdf && git submodule foreach git clean -xdf
  script:
    - ./boot
    - ./configure $CONFIGURE_ARGS
    - |
      THREADS=`mk/detect-cpu-count.sh`
      make V=0 -j$THREADS WERROR=-Werror
    - make binary-dist-prep TAR_COMP_OPTS="-1"
    - make test_bindist TEST_PREP=YES
    - |
      # Prepare to push git notes.
      METRICS_FILE=$CI_PROJECT_DIR/performance-metrics.tsv
      git config user.email "ben+ghc-ci@smart-cactus.org"
      git config user.name "GHC GitLab CI"
    - |
      THREADS=`mk/detect-cpu-count.sh`
      make $TEST_TYPE THREADS=$THREADS JUNIT_FILE=../../junit.xml METRICS_FILE=$METRICS_FILE
    - |
      # Push git notes.
      METRICS_FILE=$METRICS_FILE .gitlab/push-test-metrics.sh
  dependencies: []
  artifacts:
    reports:
      junit: junit.xml
    expire_in: 2 week
    paths:
      - $BIN_DIST_PREP_TAR_COMP
      - junit.xml
      - performance-metrics.tsv

#################################
# x86_64-darwin
#################################

validate-x86_64-darwin:
  extends: .validate
  stage: full-build
  tags:
    - x86_64-darwin
  variables:
    GHC_VERSION: 8.6.5
    CABAL_INSTALL_VERSION: 2.4.1.0
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-apple-darwin.tar.xz"
    MACOSX_DEPLOYMENT_TARGET: "10.7"
    # Only Sierra and onwards supports clock_gettime. See #12858
    ac_cv_func_clock_gettime: "no"
    LANG: "en_US.UTF-8"
    CONFIGURE_ARGS: --with-intree-gmp
    TEST_ENV: "x86_64-darwin"
  before_script:
    - git clean -xdf && git submodule foreach git clean -xdf
    - git submodule sync --recursive
    - git submodule update --init --recursive
    - git checkout .gitmodules
    - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"

    - bash .gitlab/darwin-init.sh
    - PATH="`pwd`/toolchain/bin:$PATH"
  after_script:
    - cp -Rf $HOME/.cabal cabal-cache
  artifacts:
    when: always
    expire_in: 2 week
  cache:
    key: "darwin-$GHC_VERSION"
    paths:
      - cabal-cache
      - toolchain

# Disabled because of OS X CI capacity
.validate-x86_64-darwin-hadrian:
  <<: *only-default
  stage: full-build
  tags:
    - x86_64-darwin
  variables:
    GHC_VERSION: 8.6.3
    MACOSX_DEPLOYMENT_TARGET: "10.7"
    ac_cv_func_clock_gettime: "no"
    LANG: "en_US.UTF-8"
    CONFIGURE_ARGS: --with-intree-gmp
    TEST_ENV: "x86_64-darwin-hadrian"
    FLAVOUR: "validate"
  before_script:
    - git clean -xdf && git submodule foreach git clean -xdf
    - git submodule sync --recursive
    - git submodule update --init --recursive
    - git checkout .gitmodules
    - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"

    - bash .gitlab/darwin-init.sh
    - PATH="`pwd`/toolchain/bin:$PATH"
  script:
    - cabal update
    - ./boot
    - ./configure $CONFIGURE_ARGS
    - hadrian/build.cabal.sh --flavour=$FLAVOUR -j`mk/detect-cpu-count.sh` --docs=no-sphinx binary-dist
    - mv _build/bindist/ghc*.tar.xz ghc.tar.xz
    - export TOP=$(pwd)
    - cd _build/bindist/ghc-*/ && ./configure --prefix=$TOP/_build/install && make install && cd ../../../
    - hadrian/build.cabal.sh --flavour=$FLAVOUR -j`mk/detect-cpu-count.sh` --docs=no-sphinx test --summary-junit=./junit.xml --test-compiler=$TOP/_build/install/bin/ghc
  after_script:
    - cp -Rf $HOME/.cabal cabal-cache
  artifacts:
    when: always
    expire_in: 2 week
    reports:
      junit: junit.xml
    paths:
      - ghc.tar.xz
      - junit.xml

.validate-linux:
  extends: .validate
  tags:
    - x86_64-linux
  before_script:
    - git clean -xdf && git submodule foreach git clean -xdf
    - git submodule sync --recursive
    - git submodule update --init --recursive
    - git checkout .gitmodules
    - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
      # Build hyperlinked sources for documentation when building releases
    - |
      if [[ -n "$CI_COMMIT_TAG" ]]; then
        echo "EXTRA_HADDOCK_OPTS += --hyperlinked-source --quickjump" >> mk/build.mk
      fi

    - .gitlab/prepare-system.sh
    # workaround for docker permissions
    - sudo chown ghc:ghc -R .
  after_script:
    - cp -Rf $HOME/.cabal cabal-cache
  cache:
    key: linux
    paths:
      - cabal-cache
      - toolchain

#################################
# aarch64-linux-deb9
#################################

.build-aarch64-linux-deb9:
  extends: .validate-linux
  stage: full-build
  image: "registry.gitlab.haskell.org/ghc/ci-images/aarch64-linux-deb9:$DOCKER_REV"
  allow_failure: true
  variables:
    TEST_ENV: "aarch64-linux-deb9"
    BIN_DIST_PREP_TAR_COMP: "ghc-aarch64-linux-deb9.tar.xz"
  cache:
    key: linux-aarch64-deb9
  tags:
    - aarch64-linux

validate-aarch64-linux-deb9:
  extends: .build-aarch64-linux-deb9
  artifacts:
    when: always
    expire_in: 2 week

nightly-aarch64-linux-deb9:
  extends: .build-aarch64-linux-deb9
  artifacts:
    expire_in: 2 year
  variables:
    TEST_TYPE: slowtest
  only:
    variables:
      - $NIGHTLY

#################################
# armv7-linux-deb9
#################################

.build-armv7-linux-deb9:
  extends: .validate-linux
  stage: full-build
  image: "registry.gitlab.haskell.org/ghc/ci-images/armv7-linux-deb9:$DOCKER_REV"
  allow_failure: true
  variables:
    TEST_ENV: "armv7-linux-deb9"
    BIN_DIST_PREP_TAR_COMP: "ghc-armv7-linux-deb9.tar.xz"
    CONFIGURE_ARGS: "--host=armv7-linux-gnueabihf --build=armv7-linux-gnueabihf --target=armv7-linux-gnueabihf"
  cache:
    key: linux-armv7-deb9
  tags:
    - armv7-linux

validate-armv7-linux-deb9:
  extends: .build-armv7-linux-deb9
  artifacts:
    when: always
    expire_in: 2 week

nightly-armv7-linux-deb9:
  extends: .build-armv7-linux-deb9
  artifacts:
    expire_in: 2 year
  variables:
    TEST_TYPE: slowtest
  only:
    variables:
      - $NIGHTLY

#################################
# i386-linux-deb9
#################################

.build-i386-linux-deb9:
  extends: .validate-linux
  stage: full-build
  image: "registry.gitlab.haskell.org/ghc/ci-images/i386-linux-deb9:$DOCKER_REV"
  variables:
    TEST_ENV: "i386-linux-deb9"
    BIN_DIST_PREP_TAR_COMP: "ghc-i386-deb9-linux.tar.xz"
  cache:
    key: linux-i386-deb9

validate-i386-linux-deb9:
  extends: .build-i386-linux-deb9
  artifacts:
    when: always
    expire_in: 2 week

nightly-i386-linux-deb9:
  extends: .build-i386-linux-deb9
  variables:
    TEST_TYPE: slowtest
  artifacts:
    when: always
    expire_in: 2 week
  only:
    variables:
      - $NIGHTLY

#################################
# x86_64-linux-deb9
#################################

.build-x86_64-linux-deb9:
  extends: .validate-linux
  stage: full-build
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
  variables:
    TEST_ENV: "x86_64-linux-deb9"
    BIN_DIST_PREP_TAR_COMP: "./ghc-x86_64-deb9-linux.tar.xz"
  cache:
    key: linux-x86_64-deb9

# Disabled to reduce CI load
.validate-x86_64-linux-deb9:
  extends: .build-x86_64-linux-deb9
  artifacts:
    when: always
    expire_in: 2 week

nightly-x86_64-linux-deb9:
  extends: .build-x86_64-linux-deb9
  artifacts:
    expire_in: 2 year
  variables:
    TEST_TYPE: slowtest
  only:
    variables:
      - $NIGHTLY

# N.B. Has DEBUG assertions enabled in stage2
validate-x86_64-linux-deb9-debug:
  extends: .build-x86_64-linux-deb9
  stage: build
  variables:
    BUILD_FLAVOUR: validate
    # Ensure that stage2 also has DEBUG enabled
    ValidateSpeed: SLOW
    # Override validate flavour default; see #16890.
    BUILD_SPHINX_PDF: "YES"
    TEST_TYPE: slowtest
    TEST_ENV: "x86_64-linux-deb9-debug"
    BIN_DIST_PREP_COMP: "ghc-x86_64-deb9-linux-debug.tar.xz"
  artifacts:
    when: always
    expire_in: 2 week

# Disabled to alleviate CI load
.validate-x86_64-linux-deb9-llvm:
  extends: .build-x86_64-linux-deb9
  stage: full-build
  variables:
    BUILD_FLAVOUR: perf-llvm
    TEST_ENV: "x86_64-linux-deb9-llvm"

nightly-x86_64-linux-deb9-llvm:
  extends: .build-x86_64-linux-deb9
  stage: full-build
  variables:
    BUILD_FLAVOUR: perf-llvm
    TEST_ENV: "x86_64-linux-deb9-llvm"
  only:
    variables:
      - $NIGHTLY

validate-x86_64-linux-deb9-integer-simple:
  extends: .build-x86_64-linux-deb9
  stage: full-build
  variables:
    INTEGER_LIBRARY: integer-simple
    TEST_ENV: "x86_64-linux-deb9-integer-simple"
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-deb9-linux-integer-simple.tar.xz"

nightly-x86_64-linux-deb9-integer-simple:
  extends: .build-x86_64-linux-deb9
  stage: full-build
  variables:
    INTEGER_LIBRARY: integer-simple
    TEST_ENV: "x86_64-linux-deb9-integer-simple"
    TEST_TYPE: slowtest
  artifacts:
    expire_in: 2 year
  only:
    variables:
      - $NIGHTLY

release-x86_64-linux-deb9-dwarf:
  extends: .validate-linux
  stage: build
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
  variables:
    CONFIGURE_ARGS: "--enable-dwarf-unwind"
    BUILD_FLAVOUR: dwarf
    TEST_ENV: "x86_64-linux-deb9-dwarf"
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-deb9-linux-dwarf.tar.xz"
  artifacts:
    when: always
    expire_in: 2 week
  cache:
    key: linux-x86_64-deb9


#################################
# x86_64-linux-deb8
#################################

release-x86_64-linux-deb8:
  extends: .validate-linux
  stage: full-build
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
  variables:
    TEST_ENV: "x86_64-linux-deb8"
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-deb8-linux.tar.xz"
  only:
    - tags
  cache:
    key: linux-x86_64-deb8
  artifacts:
    when: always
    expire_in: 2 week

#################################
# x86_64-linux-alpine
#################################

.build-x86_64-linux-alpine:
  extends: .validate-linux
  stage: full-build
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-alpine:$DOCKER_REV"
  # There are currently a few failing tests
  allow_failure: true
  variables:
    BUILD_SPHINX_PDF: "NO"
    TEST_ENV: "x86_64-linux-alpine"
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-alpine-linux.tar.xz"
    # Can't use ld.gold due to #13958.
    CONFIGURE_ARGS: "--disable-ld-override"
  cache:
    key: linux-x86_64-alpine
  artifacts:
    when: always
    expire_in: 2 week

release-x86_64-linux-alpine:
  extends: .build-x86_64-linux-alpine
  only:
    - tags

nightly-x86_64-linux-alpine:
  extends: .build-x86_64-linux-alpine
  only:
    variables:
      - $NIGHTLY

#################################
# x86_64-linux-centos7
#################################

release-x86_64-linux-centos7:
  extends: .validate-linux
  stage: full-build
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-centos7:$DOCKER_REV"
  variables:
    # The sphinx release shipped with Centos 7 fails to build out documentation
    BUILD_SPHINX_HTML: "NO"
    BUILD_SPHINX_PDF: "NO"
    TEST_ENV: "x86_64-linux-centos7"
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-centos7-linux.tar.xz"
  only:
    - tags
  cache:
    key: linux-x86_64-centos7
  artifacts:
    when: always
    expire_in: 2 week

#################################
# x86_64-linux-fedora27
#################################

validate-x86_64-linux-fedora27:
  extends: .validate-linux
  stage: full-build
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-fedora27:$DOCKER_REV"
  variables:
    TEST_ENV: "x86_64-linux-fedora27"
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-fedora27-linux.tar.xz"
  cache:
    key: linux-x86_64-fedora27
  artifacts:
    when: always
    expire_in: 2 week

############################################################
# Validation via Pipelines (Windows)
############################################################

.build-windows:
  <<: *only-default
  before_script:
    - git clean -xdf
    - git submodule foreach git clean -xdf

    # Use a local temporary directory to ensure that concurrent builds don't
    # interfere with one another
    - |
      mkdir tmp
      set TMP=%cd%\tmp
      set TEMP=%cd%\tmp

    - set PATH=C:\msys64\usr\bin;%PATH%
    - git submodule sync --recursive
    - git submodule update --init --recursive
    - git checkout .gitmodules
    - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
    - bash .gitlab/win32-init.sh
  after_script:
    - rd /s /q tmp
    - robocopy /np /nfl /ndl /e "%APPDATA%\cabal" cabal-cache
    - bash -c 'make clean || true'
  dependencies: []
  variables:
    FORCE_SYMLINKS: 1
    LANG: "en_US.UTF-8"
    SPHINXBUILD: "/mingw64/bin/sphinx-build.exe"
  cache:
    paths:
      - cabal-cache
      - ghc-8.6.5
      - ghc-tarballs

.build-windows-hadrian:
  extends: .build-windows
  stage: full-build
  variables:
    GHC_VERSION: "8.6.5"
    FLAVOUR: "validate"
  # due to #16574 this currently fails
  allow_failure: true
  script:
    - |
      python boot
      bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
    - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh --flavour=$FLAVOUR -j`mk/detect-cpu-count.sh` --flavour=Quick --docs=no-sphinx binary-dist"
    - mv _build/bindist/ghc*.tar.xz ghc.tar.xz
    - export TOP=$(pwd)
    - bash -c "cd _build/bindist/ghc-*/ && PATH=`pwd`/toolchain/bin:$PATH ./configure --prefix=$TOP/_build/install && make install && cd ../../../"
    - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh --flavour=$FLAVOUR -j`mk/detect-cpu-count.sh` --flavour=quick test --summary-junit=./junit.xml --skip-perf --test-compiler=$TOP/_build/install/bin/ghc"
    # skipping perf tests for now since we build a quick-flavoured GHC,
    # which might result in some broken perf tests?
  tags:
    - x86_64-windows
  artifacts:
    reports:
      junit: junit.xml
    expire_in: 2 week
    when: always
    paths:
      - ghc.tar.xz
      - junit.xml

validate-x86_64-windows-hadrian:
  extends: .build-windows-hadrian
  variables:
    MSYSTEM: MINGW64
    TEST_ENV: "x86_64-windows-hadrian"
  cache:
    key: "x86_64-windows-hadrian-$WINDOWS_TOOLCHAIN_VERSION"

nightly-i386-windows-hadrian:
  extends: .build-windows-hadrian
  variables:
    MSYSTEM: MINGW32
    TEST_ENV: "i386-windows-hadrian"
  only:
    variables:
      - $NIGHTLY
  cache:
    key: "i386-windows-hadrian-$WINDOWS_TOOLCHAIN_VERSION"

.build-windows-make:
  extends: .build-windows
  stage: full-build
  # due to #16084
  allow_failure: true
  variables:
    BUILD_FLAVOUR: "quick"
    GHC_VERSION: "8.6.5"
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-mingw32.tar.xz"
  script:
    - |
      python boot
      bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex $CONFIGURE_ARGS'
    - bash -c "echo include mk/flavours/${BUILD_FLAVOUR}.mk > mk/build.mk"
    - bash -c "echo 'GhcLibHcOpts+=-haddock' >> mk/build.mk"
    - bash -c "PATH=`pwd`/toolchain/bin:$PATH make -j`mk/detect-cpu-count.sh`"
    - bash -c "PATH=`pwd`/toolchain/bin:$PATH make binary-dist-prep TAR_COMP_OPTS=-1"
    - bash -c "PATH=`pwd`/toolchain/bin:$PATH make test_bindist TEST_PREP=YES"
    - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
  tags:
    - x86_64-windows
  artifacts:
    when: always
    expire_in: 2 week
    reports:
      junit: junit.xml
    paths:
      - $BIN_DIST_PREP_TAR_COMP
      - junit.xml

validate-x86_64-windows:
  extends: .build-windows-make
  variables:
    MSYSTEM: MINGW64
    CONFIGURE_ARGS: "--target=x86_64-unknown-mingw32"
    TEST_ENV: "x86_64-windows"
  cache:
    key: "x86_64-windows-$WINDOWS_TOOLCHAIN_VERSION"

nightly-x86_64-windows:
  extends: .build-windows-make
  stage: full-build
  variables:
    BUILD_FLAVOUR: "validate"
    MSYSTEM: MINGW64
    CONFIGURE_ARGS: "--target=x86_64-unknown-mingw32"
  only:
    variables:
      - $NIGHTLY
  cache:
    key: "x86_64-windows-$WINDOWS_TOOLCHAIN_VERSION"

# Normal Windows validate builds are profiled; that won't do for releases.
release-x86_64-windows:
  extends: validate-x86_64-windows
  variables:
    MSYSTEM: MINGW64
    BUILD_FLAVOUR: "perf"
    CONFIGURE_ARGS: "--target=x86_64-unknown-mingw32"
    TEST_ENV: "x86_64-windows"
  only:
    - tags

release-i386-windows:
  extends: .build-windows-make
  only:
    - tags
  variables:
    MSYSTEM: MINGW32
    BUILD_FLAVOUR: "perf"
    CONFIGURE_ARGS: "--target=i386-unknown-mingw32"
    # Due to #15934
    BUILD_PROF_LIBS: "NO"
    TEST_ENV: "i386-windows"
  cache:
    key: "i386-windows-$WINDOWS_TOOLCHAIN_VERSION"

nightly-i386-windows:
  extends: .build-windows-make
  only:
    variables:
      - $NIGHTLY
  variables:
    MSYSTEM: MINGW32
    CONFIGURE_ARGS: "--target=i386-unknown-mingw32"
    # Due to #15934
    BUILD_PROF_LIBS: "NO"
    TEST_ENV: "i386-windows"
  cache:
    key: "i386-windows-$WINDOWS_TOOLCHAIN_VERSION"

############################################################
# Cleanup
############################################################

# Note [Cleaning up after shell executor]
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# As noted in [1], gitlab-runner's shell executor doesn't clean up its working
# directory after builds. Unfortunately, we are forced to use the shell executor
# on Darwin. To avoid running out of disk space we add a stage at the end of
# the build to remove the /.../GitLabRunner/builds directory. Since we only run a
# single build at a time on Darwin this should be safe.
#
# We used to have a similar cleanup job on Windows as well however it ended up
# being quite fragile as we have multiple Windows builders yet there is no
# guarantee that the cleanup job is run on the same machine as the build itself
# was run. Consequently we were forced to instead handle cleanup with a separate
# cleanup cron job on Windows.
#
# [1] https://gitlab.com/gitlab-org/gitlab-runner/issues/3856

# See Note [Cleanup after shell executor]
cleanup-darwin:
  <<: *only-default
  stage: cleanup
  tags:
    - x86_64-darwin
  when: always
  dependencies: []
  before_script:
    - echo "Time to clean up"
  script:
    - echo "Let's go"
  after_script:
    - BUILD_DIR=$CI_PROJECT_DIR
    - echo "Cleaning $BUILD_DIR"
    - cd $HOME
    - rm -Rf $BUILD_DIR/*
    - exit 0

############################################################
# Packaging
############################################################

doc-tarball:
  <<: *only-default
  stage: packaging
  tags:
    - x86_64-linux
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
  dependencies:
    - validate-x86_64-linux-deb9-debug
    - validate-x86_64-windows
  variables:
    LINUX_BINDIST: "ghc-x86_64-deb9-linux.tar.xz"
    WINDOWS_BINDIST: "ghc-x86_64-mingw32.tar.xz"
  # Due to Windows allow_failure
  allow_failure: true
  artifacts:
    paths:
      - haddock.html.tar.xz
      - libraries.html.tar.xz
      - users_guide.html.tar.xz
      - index.html
      - "*.pdf"
  script:
    - |
      if [ ! -f "$LINUX_BINDIST" ]; then
        echo "Error: $LINUX_BINDIST does not exist. Did the Debian 9 job fail?"
        exit 1
      fi
      if [ ! -f "$WINDOWS_BINDIST" ]; then
        echo "Error: $WINDOWS_BINDIST does not exist. Did the 64-bit Windows job fail?"
        exit 1
      fi
    - rm -Rf docs
    - bash -ex distrib/mkDocs/mkDocs $LINUX_BINDIST $WINDOWS_BINDIST
    - ls -lh
    - mv docs/*.tar.xz docs/index.html .

source-tarball:
  stage: packaging
  tags:
    - x86_64-linux
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
  dependencies: []
  only:
    - tags
  artifacts:
    paths:
      - ghc-*.tar.xz
      - version
  script:
    - mk/get-win32-tarballs.sh download all
    - ./boot
    - ./configure
    - make sdist
    - mv sdistprep/*.xz  .
    - make show! --quiet VALUE=ProjectVersion > version
    - source version
    - echo "$ProjectVersion" > version


############################################################
# Testing via head.hackage
############################################################

# Triggering jobs in the ghc/head.hackage project requires that we have a job
# token for that repository. Furthermore the head.hackage CI job must have
# access to an unprivileged access token with the ability to query the ghc/ghc
# project such that it can find the job ID of the fedora27 job for the current
# pipeline.

.hackage:
  <<: *only-default
  stage: testing
  image: ghcci/x86_64-linux-deb9:0.2
  tags:
    - x86_64-linux
  dependencies: []
  variables:
    HEAD_HACKAGE_PROJECT_ID: "78"
  script:
    - bash .gitlab/start-head.hackage.sh

hackage:
  extends: .hackage
  when: manual

hackage-label:
  extends: .hackage
  only:
    variables:
      - $CI_MERGE_REQUEST_LABELS =~ /.*user-facing.*/

nightly-hackage:
  extends: .hackage
  only:
    variables:
      - $NIGHTLY

############################################################
# Nofib testing
############################################################

perf-nofib:
  stage: testing
  dependencies:
    - release-x86_64-linux-deb9-dwarf
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
  only:
    refs:
      - merge_requests
      - master
      - /ghc-[0-9]+\.[0-9]+/
  tags:
    - x86_64-linux
  script:
    - root=$(pwd)/ghc
    - |
      mkdir tmp
      tar -xf ghc-x86_64-deb9-linux-dwarf.tar.xz -C tmp
      pushd tmp/ghc-*/
      ./configure --prefix=$root
      make install
      popd
      rm -Rf tmp
    - export BOOT_HC=$(which ghc)
    - cabal update; cabal install -w $BOOT_HC regex-compat
    - export PATH=$root/bin:$PATH
    - make -C nofib boot mode=fast -j$CPUS
    - "make -C nofib EXTRA_RUNTEST_OPTS='-cachegrind +RTS -V0 -RTS' NoFibRuns=1 mode=fast -j$CPUS 2>&1 | tee nofib.log"
  artifacts:
    expire_in: 12 week
    when: always
    paths:
      - nofib.log

############################################################
# Documentation deployment via GitLab Pages
############################################################

pages:
  stage: deploy
  dependencies:
    - doc-tarball
  image: ghcci/x86_64-linux-deb9:0.2
  # Due to Windows allow_failure
  allow_failure: true
  tags:
    - x86_64-linux
  script:
    - mkdir -p public/doc
    - tar -xf haddock.html.tar.xz -C public/doc
    - tar -xf libraries.html.tar.xz -C public/doc
    - tar -xf users_guide.html.tar.xz -C public/doc
    - |
      cat >public/index.html <<EOF
      <!DOCTYPE HTML>
      <meta charset="UTF-8">
      <meta http-equiv="refresh" content="1; url=doc/">
      EOF
    - cp -f index.html public/doc
  only:
    - master
  artifacts:
    paths:
      - public