.gitlab-ci.yml 29.5 KB
Newer Older
1 2
variables:
  GIT_SSL_NO_VERIFY: "1"
3

4
  # Commit of ghc/ci-images repository from which to pull Docker images
5
  DOCKER_REV: 3f731f5d37a156e7ebe10cd32656946083baaf4a
6

7
  # Sequential version number capturing the versions of all tools fetched by
Ben Gamari's avatar
Ben Gamari committed
8
  # .gitlab/ci.sh.
9 10
  WINDOWS_TOOLCHAIN_VERSION: 1

11 12 13
  # Disable shallow clones; they break our linting rules
  GIT_DEPTH: 0

Ben Gamari's avatar
Ben Gamari committed
14 15 16 17
  # Overridden by individual jobs
  CONFIGURE_ARGS: ""

  GIT_SUBMODULE_STRATEGY: "recursive"
18

Ben Gamari's avatar
Ben Gamari committed
19
stages:
20 21 22 23 24 25 26 27
  - lint        # Source linting
  - quick-build # A very quick smoke-test to weed out broken commits
  - build       # A quick smoke-test to weed out broken commits
  - full-build  # Build all the things
  - cleanup     # See Note [Cleanup after the shell executor]
  - packaging   # Source distribution, etc.
  - testing     # head.hackage correctness and compiler performance testing
  - deploy      # push documentation
Ben Gamari's avatar
Ben Gamari committed
28

Ben Gamari's avatar
Ben Gamari committed
29
# N.B.Don't run on wip/ branches, instead on run on merge requests.
30 31 32 33 34 35
.only-default: &only-default
  only:
    - master
    - /ghc-[0-9]+\.[0-9]+/
    - merge_requests
    - tags
36
    - web
37

38 39 40 41 42 43 44 45
.nightly: &nightly
  only:
    variables:
      - $NIGHTLY
  artifacts:
    when: always
    expire_in: 8 weeks

46
.release: &release
Ben Gamari's avatar
Ben Gamari committed
47 48 49
  variables:
    BUILD_FLAVOUR: "perf"
    FLAVOUR: "perf"
50 51 52 53
  artifacts:
    when: always
    expire_in: 1 year
  only:
54 55
    variables:
      - $RELEASE == "yes"
56

Ben Gamari's avatar
Ben Gamari committed
57
############################################################
58
# Runner Tags
Ben Gamari's avatar
Ben Gamari committed
59 60 61 62 63 64 65 66 67 68
############################################################
#
# * x86_64-linux: Any Docker-capable x86_64 Linux machine
# * aarch64-linux: Any Docker-capable AArch64 Linux machine
# * x86_64-windows: A x86_64 Windows machine
# * lint: Any Docker-capable x86_64 Linux machine; distinct from
#         x86_64-linux to ensure low-latency availability.
#


Ben Gamari's avatar
Ben Gamari committed
69 70 71 72 73 74
############################################################
# Linting
############################################################

ghc-linters:
  stage: lint
75
  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
Ben Gamari's avatar
Ben Gamari committed
76
  script:
77
    - git fetch "$CI_MERGE_REQUEST_PROJECT_URL" $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
78
    - base="$(git merge-base FETCH_HEAD $CI_COMMIT_SHA)"
79
    - "echo Linting changes between $base..$CI_COMMIT_SHA"
80 81
    #    - validate-commit-msg .git $(git rev-list $base..$CI_COMMIT_SHA)
    - validate-whitespace .git $(git rev-list $base..$CI_COMMIT_SHA)
82 83
    - .gitlab/linters/check-makefiles.py commits $base $CI_COMMIT_SHA
    - .gitlab/linters/check-cpp.py commits $base $CI_COMMIT_SHA
84
    - .gitlab/linters/check-version-number.sh
85
    - python3 utils/checkUniques/check-uniques.py .
86
  dependencies: []
Ben Gamari's avatar
Ben Gamari committed
87 88
  tags:
    - lint
89 90 91 92
  only:
    refs:
      - merge_requests

93
# Run mypy Python typechecker on linter scripts.
Ben Gamari's avatar
Ben Gamari committed
94 95 96 97 98 99 100 101 102 103
lint-linters:
  <<: *only-default
  stage: lint
  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
  script:
    - mypy .gitlab/linters/*.py
  dependencies: []
  tags:
    - lint

104
# Check that .T files all parse by listing broken tests.
105
lint-testsuite:
Ben Gamari's avatar
Ben Gamari committed
106
  <<: *only-default
107 108 109 110 111 112 113 114
  stage: lint
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
  script:
    - make -Ctestsuite list_broken TEST_HC=ghc
  dependencies: []
  tags:
    - lint

115 116 117 118 119 120 121 122 123 124 125
# Run mypy Python typechecker on testsuite driver
typecheck-testsuite:
  <<: *only-default
  stage: lint
  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
  script:
    - mypy testsuite/driver/runtests.py
  dependencies: []
  tags:
    - lint

126
# We allow the submodule checker to fail when run on merge requests (to
Brian Wignall's avatar
Brian Wignall committed
127
# accommodate, e.g., haddock changes not yet upstream) but not on `master` or
128 129
# Marge jobs.
.lint-submods:
Ben Gamari's avatar
Ben Gamari committed
130
  <<: *only-default
131 132 133
  stage: lint
  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
  script:
134
    - git fetch "$CI_MERGE_REQUEST_PROJECT_URL" $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
Ben Gamari's avatar
Ben Gamari committed
135
    - base="$(git merge-base FETCH_HEAD $CI_COMMIT_SHA)"
136
    - "echo Linting submodule changes between $base..$CI_COMMIT_SHA"
137
    - git submodule foreach git remote update
138
    - submodchecker . $(git rev-list $base..$CI_COMMIT_SHA)
139 140 141 142
  dependencies: []
  tags:
    - lint

143 144 145 146 147 148
lint-submods-marge:
  extends: .lint-submods
  only:
    refs:
      - merge_requests
    variables:
Ben Gamari's avatar
Ben Gamari committed
149
      - "$CI_MERGE_REQUEST_LABELS =~ /.*marge_bot_batch_merge_job.*/"
150 151 152

lint-submods-mr:
  extends: .lint-submods
153 154
  # Allow failure since any necessary submodule patches may not be upstreamed
  # yet.
155 156 157 158
  allow_failure: true
  only:
    refs:
      - merge_requests
159 160
  except:
    variables:
Ben Gamari's avatar
Ben Gamari committed
161
      - "$CI_MERGE_REQUEST_LABELS =~ /.*marge_bot_batch_merge_job.*/"
Ben Gamari's avatar
Ben Gamari committed
162

163 164 165
lint-submods-branch:
  extends: .lint-submods
  script:
166
    - "echo Linting submodule changes between $CI_COMMIT_BEFORE_SHA..$CI_COMMIT_SHA"
167
    - git submodule foreach git remote update
Ben Gamari's avatar
Ben Gamari committed
168
    - submodchecker . $(git rev-list $CI_COMMIT_BEFORE_SHA..$CI_COMMIT_SHA)
169 170 171 172 173
  only:
    refs:
      - master
      - /ghc-[0-9]+\.[0-9]+/

174
.lint-changelogs:
Ben Gamari's avatar
Ben Gamari committed
175
  <<: *only-default
176 177 178 179 180 181
  stage: lint
  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
  dependencies: []
  tags:
    - lint
  script:
182
    - bash .gitlab/linters/check-changelogs.sh
183 184 185

lint-changelogs:
  extends: .lint-changelogs
186
  # Allow failure since this isn't a final release.
187 188 189 190 191 192 193 194
  allow_failure: true
  only:
    refs:
      - /ghc-[0-9]+\.[0-9]+/

lint-release-changelogs:
  extends: .lint-changelogs
  only:
195 196
    refs:
      - /ghc-[0-9]+\.[0-9]+\.[0-9]+-.*/
197 198


Ben Gamari's avatar
Ben Gamari committed
199 200 201
############################################################
# Validation via Pipelines (hadrian)
############################################################
202

Ben Gamari's avatar
Ben Gamari committed
203
.validate-hadrian:
204
  <<: *only-default
205 206
  variables:
    FLAVOUR: "validate"
Ben Gamari's avatar
Ben Gamari committed
207
  script:
Ben Gamari's avatar
Ben Gamari committed
208 209 210 211
    - .gitlab/ci.sh setup
    - .gitlab/ci.sh configure
    - .gitlab/ci.sh build_hadrian
    - .gitlab/ci.sh test_hadrian
Ben Gamari's avatar
Ben Gamari committed
212 213
  cache:
    key: hadrian
214
    paths:
Ben Gamari's avatar
Ben Gamari committed
215
      - cabal-cache
216
  artifacts:
217 218 219
    reports:
      junit: junit.xml
    expire_in: 2 week
220 221
    paths:
      - ghc.tar.xz
222
      - junit.xml
223

224
.validate-linux-hadrian:
Ben Gamari's avatar
Ben Gamari committed
225
  extends: .validate-hadrian
226
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
227 228
  variables:
    TEST_ENV: "x86_64-linux-deb9-hadrian"
Ben Gamari's avatar
Ben Gamari committed
229 230 231
  before_script:
    # workaround for docker permissions
    - sudo chown ghc:ghc -R .
232 233 234
    - git submodule sync --recursive
    - git submodule update --init --recursive
    - git checkout .gitmodules
235
    - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
Ben Gamari's avatar
Ben Gamari committed
236 237
  after_script:
    - .gitlab/ci.sh clean
Ben Gamari's avatar
Ben Gamari committed
238 239 240
  tags:
    - x86_64-linux

241 242 243 244
validate-x86_64-linux-deb9-hadrian:
  extends: .validate-linux-hadrian
  stage: build

245
validate-x86_64-linux-deb9-unreg-hadrian:
246
  extends: .validate-linux-hadrian
247 248 249
  stage: full-build
  variables:
    CONFIGURE_ARGS: --enable-unregisterised
250
    TEST_ENV: "x86_64-linux-deb9-unreg-hadrian"
251

252
hadrian-ghc-in-ghci:
253
  <<: *only-default
254
  stage: quick-build
255
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
256 257 258 259 260 261
  before_script:
    # workaround for docker permissions
    - sudo chown ghc:ghc -R .
    - git submodule sync --recursive
    - git submodule update --init --recursive
    - git checkout .gitmodules
262 263
  variables:
    GHC_FLAGS: -Werror
264 265 266 267
  tags:
    - x86_64-linux
  script:
    - cabal update
268
    - cd hadrian; cabal new-build --project-file=ci.project; cd ..
269
    - git clean -xdf && git submodule foreach git clean -xdf
Ben Gamari's avatar
Ben Gamari committed
270
    - .gitlab/ci.sh setup
271 272 273
    - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
    - ./boot
    - ./configure $CONFIGURE_ARGS
274
    # Load ghc-in-ghci then immediately exit and check the modules loaded
275
    - echo ":q" | hadrian/ghci -j`mk/detect-cpu-count.sh`| tail -n2 | grep "Ok,"
276 277 278 279 280
  cache:
    key: hadrian-ghci
    paths:
      - cabal-cache

Ben Gamari's avatar
Ben Gamari committed
281 282 283 284 285
############################################################
# Validation via Pipelines (make)
############################################################

.validate:
286
  <<: *only-default
287 288
  variables:
    TEST_TYPE: test
Ben Gamari's avatar
Ben Gamari committed
289
    MAKE_ARGS: "-Werror"
Ben Gamari's avatar
Ben Gamari committed
290
  script:
Ben Gamari's avatar
Ben Gamari committed
291 292 293 294
    - .gitlab/ci.sh setup
    - .gitlab/ci.sh configure
    - .gitlab/ci.sh build_make
    - .gitlab/ci.sh test_make
295
  dependencies: []
296 297 298
  artifacts:
    reports:
      junit: junit.xml
299
    expire_in: 2 week
300
    paths:
301
      - $BIN_DIST_PREP_TAR_COMP
302
      - junit.xml
303
      - performance-metrics.tsv
304

Ben Gamari's avatar
Ben Gamari committed
305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346
#################################
# x86_64-freebsd
#################################

.build-x86_64-freebsd:
  extends: .validate
  tags:
    - x86_64-freebsd
  allow_failure: true
  variables:
    # N.B. we use iconv from ports as I see linker errors when we attempt
    # to use the "native" iconv embedded in libc as suggested by the
    # porting guide [1].
    # [1] https://www.freebsd.org/doc/en/books/porters-handbook/using-iconv.html)
    CONFIGURE_ARGS: "--with-gmp-includes=/usr/local/include --with-gmp-libraries=/usr/local/lib --with-iconv-includes=/usr/local/include --with-iconv-libraries=/usr/local/lib"
    GHC_VERSION: 8.6.3
    CABAL_INSTALL_VERSION: 3.0.0.0
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-portbld-freebsd.tar.xz"
    TEST_ENV: "x86_64-freebsd"
    BUILD_FLAVOUR: "validate"
  after_script:
    - cp -Rf $HOME/.cabal cabal-cache
    - .gitlab/ci.sh clean
  artifacts:
    when: always
    expire_in: 2 week
  cache:
    key: "freebsd-$GHC_VERSION"
    paths:
      - cabal-cache
      - toolchain

# Disabled due to lack of builder capacity
.validate-x86_64-freebsd:
  extends: .build-x86_64-freebsd
  stage: full-build

nightly-x86_64-freebsd:
  <<: *nightly
  extends: .build-x86_64-freebsd
  stage: full-build

347 348 349 350 351
release-x86_64-freebsd:
  <<: *release
  extends: .build-x86_64-freebsd
  stage: full-build

Ben Gamari's avatar
Ben Gamari committed
352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382
.build-x86_64-freebsd-hadrian:
  extends: .validate-hadrian
  stage: full-build
  tags:
    - x86_64-freebsd
  allow_failure: true
  variables:
    CONFIGURE_ARGS: "--with-gmp-includes=/usr/local/include --with-gmp-libraries=/usr/local/lib --with-iconv-includes=/usr/local/include --with-iconv-libraries=/usr/local/lib"
    HADRIAN_ARGS: "--docs=no-sphinx"
    GHC_VERSION: 8.6.3
    CABAL_INSTALL_VERSION: 3.0.0.0
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-portbld-freebsd.tar.xz"
    TEST_ENV: "x86_64-freebsd-hadrian"
    FLAVOUR: "validate"
  after_script:
    - cp -Rf $HOME/.cabal cabal-cache
    - .gitlab/ci.sh clean
  artifacts:
    when: always
    expire_in: 2 week
  cache:
    key: "freebsd-$GHC_VERSION"
    paths:
      - cabal-cache
      - toolchain

# Disabled due to lack of builder capacity
.validate-x86_64-freebsd-hadrian:
  extends: .build-x86_64-freebsd-hadrian
  stage: full-build

383 384 385 386
#################################
# x86_64-darwin
#################################

Ben Gamari's avatar
Ben Gamari committed
387
validate-x86_64-darwin:
Ben Gamari's avatar
Ben Gamari committed
388
  extends: .validate
389
  stage: full-build
Ben Gamari's avatar
Ben Gamari committed
390 391 392
  tags:
    - x86_64-darwin
  variables:
393
    GHC_VERSION: 8.8.3
Ben Gamari's avatar
Ben Gamari committed
394
    CABAL_INSTALL_VERSION: 3.0.0.0
395
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-apple-darwin.tar.xz"
Ben Gamari's avatar
Ben Gamari committed
396 397 398
    MACOSX_DEPLOYMENT_TARGET: "10.7"
    # Only Sierra and onwards supports clock_gettime. See #12858
    ac_cv_func_clock_gettime: "no"
399
    LANG: "en_US.UTF-8"
Ben Gamari's avatar
Ben Gamari committed
400
    CONFIGURE_ARGS: "--with-intree-gmp"
401
    TEST_ENV: "x86_64-darwin"
Ben Gamari's avatar
Ben Gamari committed
402
    BUILD_FLAVOUR: "perf"
Ben Gamari's avatar
Ben Gamari committed
403 404
  after_script:
    - cp -Rf $HOME/.cabal cabal-cache
Ben Gamari's avatar
Ben Gamari committed
405
    - .gitlab/ci.sh clean
406 407
  artifacts:
    when: always
408
    expire_in: 2 week
Ben Gamari's avatar
Ben Gamari committed
409
  cache:
410
    key: "darwin-$GHC_VERSION"
Ben Gamari's avatar
Ben Gamari committed
411 412 413 414
    paths:
      - cabal-cache
      - toolchain

415 416
# Disabled because of OS X CI capacity
.validate-x86_64-darwin-hadrian:
417 418 419 420 421
  <<: *only-default
  stage: full-build
  tags:
    - x86_64-darwin
  variables:
422
    GHC_VERSION: 8.8.3
423 424 425 426
    MACOSX_DEPLOYMENT_TARGET: "10.7"
    ac_cv_func_clock_gettime: "no"
    LANG: "en_US.UTF-8"
    CONFIGURE_ARGS: --with-intree-gmp
427
    TEST_ENV: "x86_64-darwin-hadrian"
428
    FLAVOUR: "validate"
429
  script:
Ben Gamari's avatar
Ben Gamari committed
430 431 432 433
    - .gitlab/ci.sh setup
    - .gitlab/ci.sh configure
    - .gitlab/ci.sh build_hadrian
    - .gitlab/ci.sh test_hadrian
434 435
  after_script:
    - cp -Rf $HOME/.cabal cabal-cache
Ben Gamari's avatar
Ben Gamari committed
436
    - .gitlab/ci.sh clean
437 438 439 440 441 442 443 444 445
  artifacts:
    when: always
    expire_in: 2 week
    reports:
      junit: junit.xml
    paths:
      - ghc.tar.xz
      - junit.xml

Ben Gamari's avatar
Ben Gamari committed
446
.validate-linux:
Ben Gamari's avatar
Ben Gamari committed
447 448 449
  extends: .validate
  tags:
    - x86_64-linux
Ben Gamari's avatar
Ben Gamari committed
450 451
  variables:
    BUILD_FLAVOUR: "perf"
Ben Gamari's avatar
Ben Gamari committed
452
  before_script:
453 454 455
      # Build hyperlinked sources for documentation when building releases
    - |
      if [[ -n "$CI_COMMIT_TAG" ]]; then
Ben Gamari's avatar
Ben Gamari committed
456
        HADDOCK_HYPERLINKED_SOURCES=1
457
      fi
458

Ben Gamari's avatar
Ben Gamari committed
459 460 461 462 463
    # workaround for docker permissions
    - sudo chown ghc:ghc -R .
  after_script:
    - cp -Rf $HOME/.cabal cabal-cache
  cache:
Ben Gamari's avatar
Ben Gamari committed
464
    key: linux
Ben Gamari's avatar
Ben Gamari committed
465 466 467 468
    paths:
      - cabal-cache
      - toolchain

469 470 471 472 473
#################################
# aarch64-linux-deb9
#################################

.build-aarch64-linux-deb9:
Ben Gamari's avatar
Ben Gamari committed
474
  extends: .validate-linux
475
  stage: full-build
476
  image: "registry.gitlab.haskell.org/ghc/ci-images/aarch64-linux-deb9:$DOCKER_REV"
Ben Gamari's avatar
Ben Gamari committed
477
  allow_failure: true
478 479
  variables:
    TEST_ENV: "aarch64-linux-deb9"
480
    BIN_DIST_PREP_TAR_COMP: "ghc-aarch64-linux-deb9.tar.xz"
Ben Gamari's avatar
Ben Gamari committed
481 482 483 484 485
  cache:
    key: linux-aarch64-deb9
  tags:
    - aarch64-linux

486 487 488 489 490 491
validate-aarch64-linux-deb9:
  extends: .build-aarch64-linux-deb9
  artifacts:
    when: always
    expire_in: 2 week

492
nightly-aarch64-linux-deb9:
493
  <<: *nightly
494
  extends: .build-aarch64-linux-deb9
495 496 497
  variables:
    TEST_TYPE: slowtest

498 499 500 501 502 503 504 505 506 507
#################################
# armv7-linux-deb9
#################################

.build-armv7-linux-deb9:
  extends: .validate-linux
  stage: full-build
  image: "registry.gitlab.haskell.org/ghc/ci-images/armv7-linux-deb9:$DOCKER_REV"
  variables:
    TEST_ENV: "armv7-linux-deb9"
508
    BIN_DIST_PREP_TAR_COMP: "ghc-armv7-linux-deb9.tar.xz"
Ben Gamari's avatar
Ben Gamari committed
509
    CONFIGURE_ARGS: "--host=armv7-linux-gnueabihf --build=armv7-linux-gnueabihf --target=armv7-linux-gnueabihf"
510 511 512 513 514 515 516 517 518 519 520 521
  cache:
    key: linux-armv7-deb9
  tags:
    - armv7-linux

validate-armv7-linux-deb9:
  extends: .build-armv7-linux-deb9
  artifacts:
    when: always
    expire_in: 2 week

nightly-armv7-linux-deb9:
522
  <<: *nightly
523 524 525 526
  extends: .build-armv7-linux-deb9
  variables:
    TEST_TYPE: slowtest

527 528 529 530 531
#################################
# i386-linux-deb9
#################################

.build-i386-linux-deb9:
532 533
  extends: .validate-linux
  stage: full-build
534
  image: "registry.gitlab.haskell.org/ghc/ci-images/i386-linux-deb9:$DOCKER_REV"
535 536
  variables:
    TEST_ENV: "i386-linux-deb9"
537
    BIN_DIST_PREP_TAR_COMP: "ghc-i386-deb9-linux.tar.xz"
538 539 540 541 542
  cache:
    key: linux-i386-deb9

validate-i386-linux-deb9:
  extends: .build-i386-linux-deb9
543 544 545
  artifacts:
    when: always
    expire_in: 2 week
546 547

nightly-i386-linux-deb9:
548
  <<: *nightly
549
  extends: .build-i386-linux-deb9
550 551 552
  variables:
    TEST_TYPE: slowtest

553 554 555
#################################
# x86_64-linux-deb9
#################################
Ben Gamari's avatar
Ben Gamari committed
556

557
.build-x86_64-linux-deb9:
Ben Gamari's avatar
Ben Gamari committed
558
  extends: .validate-linux
559
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
560 561
  variables:
    TEST_ENV: "x86_64-linux-deb9"
562
    BIN_DIST_PREP_TAR_COMP: "./ghc-x86_64-deb9-linux.tar.xz"
Ben Gamari's avatar
Ben Gamari committed
563 564 565
  cache:
    key: linux-x86_64-deb9

566 567
# Disabled to reduce CI load
.validate-x86_64-linux-deb9:
568
  extends: .build-x86_64-linux-deb9
569
  stage: full-build
570 571 572 573
  artifacts:
    when: always
    expire_in: 2 week

574 575 576 577 578
release-x86_64-linux-deb9:
  <<: *release
  extends: .build-x86_64-linux-deb9
  stage: full-build

579
nightly-x86_64-linux-deb9:
580
  <<: *nightly
581
  extends: .build-x86_64-linux-deb9
582
  stage: full-build
583 584 585
  variables:
    TEST_TYPE: slowtest

586 587 588
# N.B. Has DEBUG assertions enabled in stage2
validate-x86_64-linux-deb9-debug:
  extends: .build-x86_64-linux-deb9
589
  stage: full-build
590 591
  variables:
    BUILD_FLAVOUR: validate
592 593
    # Ensure that stage2 also has DEBUG enabled
    ValidateSpeed: SLOW
Ben Gamari's avatar
Ben Gamari committed
594 595
    # Override validate flavour default; see #16890.
    BUILD_SPHINX_PDF: "YES"
596
    TEST_TYPE: slowtest
597
    TEST_ENV: "x86_64-linux-deb9-debug"
598
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-deb9-linux-debug.tar.xz"
599 600 601
  artifacts:
    when: always
    expire_in: 2 week
602

603 604
# Disabled to alleviate CI load
.validate-x86_64-linux-deb9-llvm:
605
  extends: .build-x86_64-linux-deb9
606
  stage: full-build
Ben Gamari's avatar
Ben Gamari committed
607 608
  variables:
    BUILD_FLAVOUR: perf-llvm
609
    TEST_ENV: "x86_64-linux-deb9-llvm"
610

611
nightly-x86_64-linux-deb9-llvm:
612
  <<: *nightly
613 614 615 616 617 618
  extends: .build-x86_64-linux-deb9
  stage: full-build
  variables:
    BUILD_FLAVOUR: perf-llvm
    TEST_ENV: "x86_64-linux-deb9-llvm"

619 620 621 622
validate-x86_64-linux-deb9-integer-simple:
  extends: .build-x86_64-linux-deb9
  stage: full-build
  variables:
623
    BUILD_FLAVOUR: validate
624
    INTEGER_LIBRARY: integer-simple
625
    TEST_ENV: "x86_64-linux-deb9-integer-simple-validate"
626
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-deb9-linux-integer-simple.tar.xz"
627 628

nightly-x86_64-linux-deb9-integer-simple:
629
  <<: *nightly
630 631 632 633 634 635 636
  extends: .build-x86_64-linux-deb9
  stage: full-build
  variables:
    INTEGER_LIBRARY: integer-simple
    TEST_ENV: "x86_64-linux-deb9-integer-simple"
    TEST_TYPE: slowtest

637 638
validate-x86_64-linux-deb9-dwarf:
  extends: .build-x86_64-linux-deb9
639
  stage: full-build
640 641 642
  variables:
    CONFIGURE_ARGS: "--enable-dwarf-unwind"
    BUILD_FLAVOUR: dwarf
643
    TEST_ENV: "x86_64-linux-deb9-dwarf"
644
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-deb9-linux-dwarf.tar.xz"
645

Ben Gamari's avatar
Ben Gamari committed
646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665
#################################
# x86_64-linux-deb10
#################################

.build-x86_64-linux-deb10:
  extends: .validate-linux
  stage: full-build
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb10:$DOCKER_REV"
  variables:
    TEST_ENV: "x86_64-linux-deb10"
    BIN_DIST_PREP_TAR_COMP: "./ghc-x86_64-deb10-linux.tar.xz"
  cache:
    key: linux-x86_64-deb10

# Disabled to alleviate CI load
.validate-x86_64-linux-deb10:
  extends: .build-x86_64-linux-deb10
  stage: full-build

nightly-x86_64-linux-deb10:
666
  <<: *nightly
Ben Gamari's avatar
Ben Gamari committed
667 668 669 670 671 672 673 674
  extends: .build-x86_64-linux-deb10
  variables:
    TEST_TYPE: slowtest

release-x86_64-linux-deb10:
  <<: *release
  extends: .build-x86_64-linux-deb10

675 676 677
#################################
# x86_64-linux-deb8
#################################
Ben Gamari's avatar
Ben Gamari committed
678

679
.build-x86_64-linux-deb8:
Ben Gamari's avatar
Ben Gamari committed
680
  extends: .validate-linux
681
  stage: full-build
682
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
683
  allow_failure: true
684 685
  variables:
    TEST_ENV: "x86_64-linux-deb8"
686
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-deb8-linux.tar.xz"
687 688 689 690 691 692
    # Debian 8's Sphinx is too old to support the table directive's :widths:
    # option: https://sourceforge.net/p/docutils/patches/120/
    BUILD_SPHINX_HTML: "NO"
    BUILD_SPHINX_INFO: "NO"
    BUILD_SPHINX_PDF: "NO"
    BUILD_SPHINX_MAN: "NO"
Ben Gamari's avatar
Ben Gamari committed
693 694
  cache:
    key: linux-x86_64-deb8
695 696 697 698

release-x86_64-linux-deb8:
  <<: *release
  extends: .build-x86_64-linux-deb8
Ben Gamari's avatar
Ben Gamari committed
699

700 701 702 703
#################################
# x86_64-linux-alpine
#################################

704 705
.build-x86_64-linux-alpine-hadrian:
  extends: .validate-linux-hadrian
706 707 708 709 710 711
  stage: full-build
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-alpine:$DOCKER_REV"
  # There are currently a few failing tests
  allow_failure: true
  variables:
    TEST_ENV: "x86_64-linux-alpine"
712
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-alpine-linux.tar.xz"
713 714
    # Can't use ld.gold due to #13958.
    CONFIGURE_ARGS: "--disable-ld-override"
715
    HADRIAN_ARGS: "--docs=no-sphinx"
716 717 718
    # encoding004 due to lack of locale support
    # T10458 due to fact that dynamic linker tries to reload libAS
    BROKEN_TESTS: "encoding004 T10458"
719 720 721 722 723 724
  cache:
    key: linux-x86_64-alpine
  artifacts:
    when: always
    expire_in: 2 week

725
release-x86_64-linux-alpine:
726
  <<: *release
727
  extends: .build-x86_64-linux-alpine-hadrian
728 729

nightly-x86_64-linux-alpine:
730
  <<: *nightly
731
  extends: .build-x86_64-linux-alpine-hadrian
732

733 734 735 736
#################################
# x86_64-linux-centos7
#################################

737
.build-x86_64-linux-centos7:
738 739 740 741 742 743 744 745
  extends: .validate-linux
  stage: full-build
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-centos7:$DOCKER_REV"
  variables:
    # The sphinx release shipped with Centos 7 fails to build out documentation
    BUILD_SPHINX_HTML: "NO"
    BUILD_SPHINX_PDF: "NO"
    TEST_ENV: "x86_64-linux-centos7"
746
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-centos7-linux.tar.xz"
Ben Gamari's avatar
Ben Gamari committed
747 748
    # CentOS seems to default to ascii
    LANG: "en_US.UTF-8"
749 750
  cache:
    key: linux-x86_64-centos7
751 752 753 754

release-x86_64-linux-centos7:
  <<: *release
  extends: .build-x86_64-linux-centos7
755 756 757 758 759

#################################
# x86_64-linux-fedora27
#################################

760
validate-x86_64-linux-fedora27:
Ben Gamari's avatar
Ben Gamari committed
761
  extends: .validate-linux
762
  stage: full-build
763
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-fedora27:$DOCKER_REV"
764 765
  variables:
    TEST_ENV: "x86_64-linux-fedora27"
766
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-fedora27-linux.tar.xz"
Ben Gamari's avatar
Ben Gamari committed
767 768
  cache:
    key: linux-x86_64-fedora27
769 770
  artifacts:
    when: always
771 772 773
    # These are used for head.hackage jobs therefore we keep them around for
    # longer.
    expire_in: 8 week
Ben Gamari's avatar
Ben Gamari committed
774

Ben Gamari's avatar
Ben Gamari committed
775
############################################################
776
# Validation via Pipelines (Windows)
Ben Gamari's avatar
Ben Gamari committed
777 778
############################################################

779
.build-windows:
780
  <<: *only-default
781
  # For the reasons given in #17777 this build isn't reliable.
782
  allow_failure: true
Ben Gamari's avatar
Ben Gamari committed
783
  before_script:
784
    - git clean -xdf
785

Ben Gamari's avatar
Ben Gamari committed
786 787
    # Setup toolchain
    - bash .gitlab/ci.sh setup
788
  after_script:
Ben Gamari's avatar
Ben Gamari committed
789 790 791
    - |
      Copy-Item -Recurse -Path $Env:APPDATA\cabal -Destination cabal-cache
    - bash .gitlab/ci.sh clean
792
  dependencies: []
Tamar Christina's avatar
Tamar Christina committed
793
  variables:
794
    #FORCE_SYMLINKS: 1
795
    LANG: "en_US.UTF-8"
796
    SPHINXBUILD: "/mingw64/bin/sphinx-build.exe"
Ben Gamari's avatar
Ben Gamari committed
797
    CABAL_INSTALL_VERSION: 3.0.0.0
798
    GHC_VERSION: "8.8.3"
Ben Gamari's avatar
Ben Gamari committed
799 800 801
  cache:
    paths:
      - cabal-cache
Ben Gamari's avatar
Ben Gamari committed
802
      - toolchain
Ben Gamari's avatar
Ben Gamari committed
803 804
      - ghc-tarballs

805 806
.build-windows-hadrian:
  extends: .build-windows
807
  stage: full-build
Ben Gamari's avatar
Ben Gamari committed
808
  variables:
809
    FLAVOUR: "validate"
Ben Gamari's avatar
Ben Gamari committed
810 811 812 813
    # skipping perf tests for now since we build a quick-flavoured GHC,
    # which might result in some broken perf tests?
    HADRIAN_ARGS: "--docs=no-sphinx --skip-perf"

Ben Gamari's avatar
Ben Gamari committed
814
  script:
Ben Gamari's avatar
Ben Gamari committed
815 816 817
    - bash .gitlab/ci.sh configure
    - bash .gitlab/ci.sh build_hadrian
    - bash .gitlab/ci.sh test_hadrian
Ben Gamari's avatar
Ben Gamari committed
818
  tags:
Ben Gamari's avatar
Ben Gamari committed
819 820
    - new-x86_64-windows
    - test
821
  artifacts:
822 823 824
    reports:
      junit: junit.xml
    expire_in: 2 week
825 826 827
    when: always
    paths:
      - ghc.tar.xz
828
      - junit.xml
Ben Gamari's avatar
Ben Gamari committed
829

830
validate-x86_64-windows-hadrian:
831 832 833
  extends: .build-windows-hadrian
  variables:
    MSYSTEM: MINGW64
834
    TEST_ENV: "x86_64-windows-hadrian"
835
  cache:
836
    key: "x86_64-windows-hadrian-$WINDOWS_TOOLCHAIN_VERSION"
837 838

nightly-i386-windows-hadrian:
839
  <<: *nightly
840 841 842
  extends: .build-windows-hadrian
  variables:
    MSYSTEM: MINGW32
843
    TEST_ENV: "i386-windows-hadrian"
844
  cache:
845
    key: "i386-windows-hadrian-$WINDOWS_TOOLCHAIN_VERSION"
846 847 848

.build-windows-make:
  extends: .build-windows
849
  stage: full-build
850
  variables:
851
    BUILD_FLAVOUR: "quick"
852
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-mingw32.tar.xz"
853
  script:
Ben Gamari's avatar
Ben Gamari committed
854 855 856
    - bash .gitlab/ci.sh configure
    - bash .gitlab/ci.sh build_make
    - bash .gitlab/ci.sh test_make
857
  tags:
Ben Gamari's avatar
Ben Gamari committed
858 859
    - new-x86_64-windows
    - test
860 861
  artifacts:
    when: always
862
    expire_in: 2 week
863 864 865
    reports:
      junit: junit.xml
    paths:
866
      # N.B. variable interpolation apparently doesn't work on Windows so
867
      # this can't be $BIN_DIST_PREP_TAR_COMP
868
      - "ghc-x86_64-mingw32.tar.xz"
869
      - junit.xml
Ben Gamari's avatar
Ben Gamari committed
870

Ben Gamari's avatar
Ben Gamari committed
871
.build-x86_64-windows-make:
872 873 874
  extends: .build-windows-make
  variables:
    MSYSTEM: MINGW64
875
    TEST_ENV: "x86_64-windows"
876
  cache:
877
    key: "x86_64-windows-$WINDOWS_TOOLCHAIN_VERSION"
878

Ben Gamari's avatar
Ben Gamari committed
879 880 881
validate-x86_64-windows:
  extends: .build-x86_64-windows-make

882
nightly-x86_64-windows:
883
  <<: *nightly
Ben Gamari's avatar
Ben Gamari committed
884
  extends: .build-x86_64-windows-make
885 886 887 888
  stage: full-build
  variables:
    BUILD_FLAVOUR: "validate"

889 890
# Normal Windows validate builds are profiled; that won't do for releases.
release-x86_64-windows:
891
  <<: *release
892 893 894
  extends: validate-x86_64-windows
  variables:
    BUILD_FLAVOUR: "perf"
895 896 897 898 899 900 901
    #
release-x86_64-windows-integer-simple:
  <<: *release
  extends: validate-x86_64-windows
  variables:
    INTEGER_LIBRARY: integer-simple
    BUILD_FLAVOUR: "perf"
902

Ben Gamari's avatar
Ben Gamari committed
903 904

.build-i386-windows-make:
905 906 907
  extends: .build-windows-make
  variables:
    MSYSTEM: MINGW32
908 909
    # Due to #15934
    BUILD_PROF_LIBS: "NO"
910
    TEST_ENV: "i386-windows"
911 912
  # Due to #17736
  allow_failure: true
913
  cache:
914
    key: "i386-windows-$WINDOWS_TOOLCHAIN_VERSION"
915

Ben Gamari's avatar
Ben Gamari committed
916 917 918 919 920 921 922 923 924 925 926
validate-i386-windows:
  extends: .build-i386-windows-make
  variables:
    BUILD_FLAVOUR: "perf"

release-i386-windows:
  <<: *release
  extends: .build-i386-windows-make
  variables:
    BUILD_FLAVOUR: "perf"

927
nightly-i386-windows:
928
  <<: *nightly
Ben Gamari's avatar
Ben Gamari committed
929
  extends: .build-i386-windows-make
930

931 932 933 934
############################################################
# Cleanup
############################################################

935 936
# Note [Cleaning up after shell executor]
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
937 938 939
#
# As noted in [1], gitlab-runner's shell executor doesn't clean up its working
# directory after builds. Unfortunately, we are forced to use the shell executor
940 941 942 943 944 945 946 947 948
# on Darwin. To avoid running out of disk space we add a stage at the end of
# the build to remove the /.../GitLabRunner/builds directory. Since we only run a
# single build at a time on Darwin this should be safe.
#
# We used to have a similar cleanup job on Windows as well however it ended up
# being quite fragile as we have multiple Windows builders yet there is no
# guarantee that the cleanup job is run on the same machine as the build itself
# was run. Consequently we were forced to instead handle cleanup with a separate
# cleanup cron job on Windows.
949 950 951
#
# [1] https://gitlab.com/gitlab-org/gitlab-runner/issues/3856

952 953
# See Note [Cleanup after shell executor]
cleanup-darwin:
954
  <<: *only-default
955 956 957 958
  stage: cleanup
  tags:
    - x86_64-darwin
  when: always
959
  dependencies: []
960 961 962 963 964 965 966 967 968 969 970
  before_script:
    - echo "Time to clean up"
  script:
    - echo "Let's go"
  after_script:
    - BUILD_DIR=$CI_PROJECT_DIR
    - echo "Cleaning $BUILD_DIR"
    - cd $HOME
    - rm -Rf $BUILD_DIR/*
    - exit 0

971 972 973 974 975
############################################################
# Packaging
############################################################

doc-tarball:
976
  <<: *only-default
977 978 979
  stage: packaging
  tags:
    - x86_64-linux
980
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
981
  dependencies:
982
    - validate-x86_64-linux-deb9-debug
983
    - validate-x86_64-windows
984
  variables:
985
    LINUX_BINDIST: "ghc-x86_64-deb9-linux-debug.tar.xz"
986
    WINDOWS_BINDIST: "ghc-x86_64-mingw32.tar.xz"
987 988
  # Due to Windows allow_failure
  allow_failure: true
989 990 991 992 993 994 995 996
  artifacts:
    paths:
      - haddock.html.tar.xz
      - libraries.html.tar.xz
      - users_guide.html.tar.xz
      - index.html
      - "*.pdf"
  script:
997 998 999 1000 1001 1002 1003 1004 1005
    - |
      if [ ! -f "$LINUX_BINDIST" ]; then
        echo "Error: $LINUX_BINDIST does not exist. Did the Debian 9 job fail?"
        exit 1
      fi
      if [ ! -f "$WINDOWS_BINDIST" ]; then
        echo "Error: $WINDOWS_BINDIST does not exist. Did the 64-bit Windows job fail?"
        exit 1
      fi
1006
    - rm -Rf docs
1007
    - bash -ex distrib/mkDocs/mkDocs $LINUX_BINDIST $WINDOWS_BINDIST
1008
    - ls -lh
1009
    - mv docs/*.tar.xz docs/index.html .
1010

1011 1012 1013 1014
source-tarball:
  stage: packaging
  tags:
    - x86_64-linux
1015
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
1016
  when: always
1017
  dependencies: []
1018 1019 1020 1021 1022 1023 1024
  only:
    - tags
  artifacts:
    paths:
      - ghc-*.tar.xz
      - version
  script:
1025
    - python3 mk/get-win32-tarballs.py download all
1026 1027 1028 1029
    - ./boot
    - ./configure
    - make sdist
    - mv sdistprep/*.xz  .
1030 1031 1032
    - make show! --quiet VALUE=ProjectVersion > version
    - source version
    - echo "$ProjectVersion" > version
1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045


############################################################
# Testing via head.hackage
############################################################

# Triggering jobs in the ghc/head.hackage project requires that we have a job
# token for that repository. Furthermore the head.hackage CI job must have
# access to an unprivileged access token with the ability to query the ghc/ghc
# project such that it can find the job ID of the fedora27 job for the current
# pipeline.

.hackage:
1046
  <<: *only-default
1047
  stage: testing
1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067
  image: ghcci/x86_64-linux-deb9:0.2
  tags:
    - x86_64-linux
  dependencies: []
  variables:
    HEAD_HACKAGE_PROJECT_ID: "78"
  script:
    - bash .gitlab/start-head.hackage.sh

hackage:
  extends: .hackage
  when: manual

hackage-label:
  extends: .hackage
  only:
    variables:
      - $CI_MERGE_REQUEST_LABELS =~ /.*user-facing.*/

nightly-hackage:
1068
  <<: *nightly
1069
  extends: .hackage
1070

1071 1072 1073 1074 1075 1076 1077
############################################################
# Nofib testing
############################################################

perf-nofib:
  stage: testing
  dependencies:
1078
    - validate-x86_64-linux-deb9-dwarf
1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
  only:
    refs:
      - merge_requests
      - master
      - /ghc-[0-9]+\.[0-9]+/
  tags:
    - x86_64-linux
  script:
    - root=$(pwd)/ghc
    - |
      mkdir tmp
1091
      tar -xf ghc-x86_64-deb9-linux-dwarf.tar.xz -C tmp
1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111
      pushd tmp/ghc-*/
      ./configure --prefix=$root
      make install
      popd
      rm -Rf tmp
    - export BOOT_HC=$(which ghc)
    - cabal update; cabal install -w $BOOT_HC regex-compat
    - export PATH=$root/bin:$PATH
    - make -C nofib boot mode=fast -j$CPUS
    - "make -C nofib EXTRA_RUNTEST_OPTS='-cachegrind +RTS -V0 -RTS' NoFibRuns=1 mode=fast -j$CPUS 2>&1 | tee nofib.log"
  artifacts:
    expire_in: 12 week
    when: always
    paths:
      - nofib.log

############################################################
# Documentation deployment via GitLab Pages
############################################################

1112 1113 1114 1115 1116
pages:
  stage: deploy
  dependencies:
    - doc-tarball
  image: ghcci/x86_64-linux-deb9:0.2
1117 1118
  # Due to Windows allow_failure
  allow_failure: true
1119 1120 1121 1122 1123 1124 1125
  tags:
    - x86_64-linux
  script:
    - mkdir -p public/doc
    - tar -xf haddock.html.tar.xz -C public/doc
    - tar -xf libraries.html.tar.xz -C public/doc
    - tar -xf users_guide.html.tar.xz -C public/doc
1126 1127 1128 1129 1130 1131
    - |
      cat >public/index.html <<EOF
      <!DOCTYPE HTML>
      <meta charset="UTF-8">
      <meta http-equiv="refresh" content="1; url=doc/">
      EOF
1132 1133 1134 1135 1136 1137
    - cp -f index.html public/doc
  only:
    - master
  artifacts:
    paths:
      - public