.gitlab-ci.yml 29.6 KB
Newer Older
1 2
variables:
  GIT_SSL_NO_VERIFY: "1"
3

4
  # Commit of ghc/ci-images repository from which to pull Docker images
5
  DOCKER_REV: 6223fe0b5942f4fa35bdec92c74566cf195bfb42
6

7
  # Sequential version number capturing the versions of all tools fetched by
Ben Gamari's avatar
Ben Gamari committed
8
  # .gitlab/ci.sh.
9 10
  WINDOWS_TOOLCHAIN_VERSION: 1

11 12 13
  # Disable shallow clones; they break our linting rules
  GIT_DEPTH: 0

Ben Gamari's avatar
Ben Gamari committed
14 15 16 17
  # Overridden by individual jobs
  CONFIGURE_ARGS: ""

  GIT_SUBMODULE_STRATEGY: "recursive"
18

Ben Gamari's avatar
Ben Gamari committed
19
stages:
20 21 22 23 24 25 26 27
  - lint        # Source linting
  - quick-build # A very quick smoke-test to weed out broken commits
  - build       # A quick smoke-test to weed out broken commits
  - full-build  # Build all the things
  - cleanup     # See Note [Cleanup after the shell executor]
  - packaging   # Source distribution, etc.
  - testing     # head.hackage correctness and compiler performance testing
  - deploy      # push documentation
Ben Gamari's avatar
Ben Gamari committed
28

29 30 31 32 33 34 35 36
workflow:
  # N.B.Don't run on wip/ branches, instead on run on merge requests.
  rules:
    - if: $CI_MERGE_REQUEST_ID
    - if: $CI_COMMIT_TAG
    - if: '$CI_COMMIT_BRANCH == "master"'
    - if: '$CI_COMMIT_BRANCH =~ /ghc-[0.9]+\.[0-9]+/'
    - if: '$CI_PIPELINE_SOURCE == "web"'
37

38
.nightly: &nightly
39 40
  rules:
    - if: $NIGHTLY
41 42 43 44
  artifacts:
    when: always
    expire_in: 8 weeks

45
.release: &release
Ben Gamari's avatar
Ben Gamari committed
46 47 48
  variables:
    BUILD_FLAVOUR: "perf"
    FLAVOUR: "perf"
49 50 51
  artifacts:
    when: always
    expire_in: 1 year
52 53
  rules:
    - if: '$RELEASE == "yes"'
54

Ben Gamari's avatar
Ben Gamari committed
55
############################################################
56
# Runner Tags
Ben Gamari's avatar
Ben Gamari committed
57 58 59 60 61 62 63 64 65 66
############################################################
#
# * x86_64-linux: Any Docker-capable x86_64 Linux machine
# * aarch64-linux: Any Docker-capable AArch64 Linux machine
# * x86_64-windows: A x86_64 Windows machine
# * lint: Any Docker-capable x86_64 Linux machine; distinct from
#         x86_64-linux to ensure low-latency availability.
#


Ben Gamari's avatar
Ben Gamari committed
67 68 69 70 71 72
############################################################
# Linting
############################################################

ghc-linters:
  stage: lint
73
  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
Ben Gamari's avatar
Ben Gamari committed
74
  script:
75
    - git fetch "$CI_MERGE_REQUEST_PROJECT_URL" $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
76
    - base="$(git merge-base FETCH_HEAD $CI_COMMIT_SHA)"
77
    - "echo Linting changes between $base..$CI_COMMIT_SHA"
78 79
    #    - validate-commit-msg .git $(git rev-list $base..$CI_COMMIT_SHA)
    - validate-whitespace .git $(git rev-list $base..$CI_COMMIT_SHA)
80 81
    - .gitlab/linters/check-makefiles.py commits $base $CI_COMMIT_SHA
    - .gitlab/linters/check-cpp.py commits $base $CI_COMMIT_SHA
82
    - .gitlab/linters/check-version-number.sh
83
    - python3 utils/checkUniques/check-uniques.py .
84
  dependencies: []
Ben Gamari's avatar
Ben Gamari committed
85 86
  tags:
    - lint
87 88
  rules:
    - if: $CI_MERGE_REQUEST_ID
89

90
# Run mypy Python typechecker on linter scripts.
Ben Gamari's avatar
Ben Gamari committed
91 92 93 94 95 96 97 98 99
lint-linters:
  stage: lint
  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
  script:
    - mypy .gitlab/linters/*.py
  dependencies: []
  tags:
    - lint

100
# Check that .T files all parse by listing broken tests.
101 102 103 104 105 106 107 108 109
lint-testsuite:
  stage: lint
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
  script:
    - make -Ctestsuite list_broken TEST_HC=ghc
  dependencies: []
  tags:
    - lint

110 111 112 113 114 115 116 117 118 119
# Run mypy Python typechecker on testsuite driver
typecheck-testsuite:
  stage: lint
  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
  script:
    - mypy testsuite/driver/runtests.py
  dependencies: []
  tags:
    - lint

120
# We allow the submodule checker to fail when run on merge requests (to
Brian Wignall's avatar
Brian Wignall committed
121
# accommodate, e.g., haddock changes not yet upstream) but not on `master` or
122 123 124 125 126
# Marge jobs.
.lint-submods:
  stage: lint
  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
  script:
127
    - git fetch "$CI_MERGE_REQUEST_PROJECT_URL" $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
Ben Gamari's avatar
Ben Gamari committed
128
    - base="$(git merge-base FETCH_HEAD $CI_COMMIT_SHA)"
129
    - "echo Linting submodule changes between $base..$CI_COMMIT_SHA"
130
    - git submodule foreach git remote update
131
    - submodchecker . $(git rev-list $base..$CI_COMMIT_SHA)
132 133 134 135
  dependencies: []
  tags:
    - lint

136
lint-submods:
137
  extends: .lint-submods
138 139 140 141 142 143
  # Allow failure on merge requests since any necessary submodule patches may
  # not be upstreamed yet.
  rules:
    - if: '$CI_MERGE_REQUEST_LABELS =~ /.*marge_bot_batch_merge_job.*/'
      allow_failure: false
    - allow_failure: true
Ben Gamari's avatar
Ben Gamari committed
144

145 146 147
lint-submods-branch:
  extends: .lint-submods
  script:
148
    - "echo Linting submodule changes between $CI_COMMIT_BEFORE_SHA..$CI_COMMIT_SHA"
149
    - git submodule foreach git remote update
Ben Gamari's avatar
Ben Gamari committed
150
    - submodchecker . $(git rev-list $CI_COMMIT_BEFORE_SHA..$CI_COMMIT_SHA)
151 152 153
  rules:
    - if: '$CI_COMMIT_BRANCH == "master"'
    - if: '$CI_COMMIT_BRANCH =~ /ghc-[0.9]+\.[0-9]+/'
154

155 156 157 158 159 160 161
.lint-changelogs:
  stage: lint
  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
  dependencies: []
  tags:
    - lint
  script:
162
    - bash .gitlab/linters/check-changelogs.sh
163 164 165

lint-changelogs:
  extends: .lint-changelogs
166
  # Allow failure since this isn't a final release.
167
  allow_failure: true
168 169
  rules:
    - if: '$CI_COMMIT_BRANCH =~ /ghc-[0.9]+\.[0-9]+/'
170 171 172

lint-release-changelogs:
  extends: .lint-changelogs
173 174
  rules:
    - if: '$CI_COMMIT_BRANCH =~ /ghc-[0.9]+\.[0-9]+/'
175 176


Ben Gamari's avatar
Ben Gamari committed
177 178 179
############################################################
# Validation via Pipelines (hadrian)
############################################################
180

Ben Gamari's avatar
Ben Gamari committed
181
.validate-hadrian:
182 183
  variables:
    FLAVOUR: "validate"
Ben Gamari's avatar
Ben Gamari committed
184
  script:
Ben Gamari's avatar
Ben Gamari committed
185 186 187 188
    - .gitlab/ci.sh setup
    - .gitlab/ci.sh configure
    - .gitlab/ci.sh build_hadrian
    - .gitlab/ci.sh test_hadrian
Ben Gamari's avatar
Ben Gamari committed
189 190
  cache:
    key: hadrian
191
    paths:
Ben Gamari's avatar
Ben Gamari committed
192
      - cabal-cache
193
  artifacts:
194 195 196
    reports:
      junit: junit.xml
    expire_in: 2 week
197 198
    paths:
      - ghc.tar.xz
199
      - junit.xml
200

201
.validate-linux-hadrian:
Ben Gamari's avatar
Ben Gamari committed
202
  extends: .validate-hadrian
203
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
204 205
  variables:
    TEST_ENV: "x86_64-linux-deb9-hadrian"
Ben Gamari's avatar
Ben Gamari committed
206 207 208
  before_script:
    # workaround for docker permissions
    - sudo chown ghc:ghc -R .
209 210 211
    - git submodule sync --recursive
    - git submodule update --init --recursive
    - git checkout .gitmodules
212
    - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
Ben Gamari's avatar
Ben Gamari committed
213 214
  after_script:
    - .gitlab/ci.sh clean
Ben Gamari's avatar
Ben Gamari committed
215 216 217
  tags:
    - x86_64-linux

218 219 220 221
validate-x86_64-linux-deb9-hadrian:
  extends: .validate-linux-hadrian
  stage: build

222
validate-x86_64-linux-deb9-unreg-hadrian:
223
  extends: .validate-linux-hadrian
224 225 226
  stage: full-build
  variables:
    CONFIGURE_ARGS: --enable-unregisterised
227
    TEST_ENV: "x86_64-linux-deb9-unreg-hadrian"
228

229
hadrian-ghc-in-ghci:
230
  stage: quick-build
231
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
232 233 234 235 236 237
  before_script:
    # workaround for docker permissions
    - sudo chown ghc:ghc -R .
    - git submodule sync --recursive
    - git submodule update --init --recursive
    - git checkout .gitmodules
238 239
  variables:
    GHC_FLAGS: -Werror
240 241 242 243
  tags:
    - x86_64-linux
  script:
    - cabal update
244
    - cd hadrian; cabal new-build --project-file=ci.project; cd ..
245
    - git clean -xdf && git submodule foreach git clean -xdf
Ben Gamari's avatar
Ben Gamari committed
246
    - .gitlab/ci.sh setup
247 248 249
    - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
    - ./boot
    - ./configure $CONFIGURE_ARGS
250
    # Load ghc-in-ghci then immediately exit and check the modules loaded
251
    - echo ":q" | hadrian/ghci -j`mk/detect-cpu-count.sh`| tail -n2 | grep "Ok,"
252 253 254 255 256
  cache:
    key: hadrian-ghci
    paths:
      - cabal-cache

Ben Gamari's avatar
Ben Gamari committed
257 258 259 260 261
############################################################
# Validation via Pipelines (make)
############################################################

.validate:
262 263
  variables:
    TEST_TYPE: test
Ben Gamari's avatar
Ben Gamari committed
264
    MAKE_ARGS: "-Werror"
Ben Gamari's avatar
Ben Gamari committed
265
  script:
Ben Gamari's avatar
Ben Gamari committed
266 267 268 269
    - .gitlab/ci.sh setup
    - .gitlab/ci.sh configure
    - .gitlab/ci.sh build_make
    - .gitlab/ci.sh test_make
270
  dependencies: []
271 272 273
  artifacts:
    reports:
      junit: junit.xml
274
    expire_in: 2 week
275
    paths:
276
      - $BIN_DIST_PREP_TAR_COMP
277
      - junit.xml
278
      - performance-metrics.tsv
279

Ben Gamari's avatar
Ben Gamari committed
280 281 282 283 284 285 286 287 288 289 290 291 292 293 294
#################################
# x86_64-freebsd
#################################

.build-x86_64-freebsd:
  extends: .validate
  tags:
    - x86_64-freebsd
  allow_failure: true
  variables:
    # N.B. we use iconv from ports as I see linker errors when we attempt
    # to use the "native" iconv embedded in libc as suggested by the
    # porting guide [1].
    # [1] https://www.freebsd.org/doc/en/books/porters-handbook/using-iconv.html)
    CONFIGURE_ARGS: "--with-gmp-includes=/usr/local/include --with-gmp-libraries=/usr/local/lib --with-iconv-includes=/usr/local/include --with-iconv-libraries=/usr/local/lib"
295 296
    GHC_VERSION: 8.10.1
    CABAL_INSTALL_VERSION: 3.2.0.0
Ben Gamari's avatar
Ben Gamari committed
297 298 299 300 301 302 303 304 305 306 307 308 309 310 311
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-portbld-freebsd.tar.xz"
    TEST_ENV: "x86_64-freebsd"
    BUILD_FLAVOUR: "validate"
  after_script:
    - cp -Rf $HOME/.cabal cabal-cache
    - .gitlab/ci.sh clean
  artifacts:
    when: always
    expire_in: 2 week
  cache:
    key: "freebsd-$GHC_VERSION"
    paths:
      - cabal-cache
      - toolchain

312 313
# Conditional due to lack of builder capacity
validate-x86_64-freebsd:
Ben Gamari's avatar
Ben Gamari committed
314 315
  extends: .build-x86_64-freebsd
  stage: full-build
316 317
  rules:
    - if: '$CI_MERGE_REQUEST_LABELS =~ /.*FreeBSD.*/'
Ben Gamari's avatar
Ben Gamari committed
318 319 320 321 322 323

nightly-x86_64-freebsd:
  <<: *nightly
  extends: .build-x86_64-freebsd
  stage: full-build

324 325 326 327 328
release-x86_64-freebsd:
  <<: *release
  extends: .build-x86_64-freebsd
  stage: full-build

Ben Gamari's avatar
Ben Gamari committed
329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359
.build-x86_64-freebsd-hadrian:
  extends: .validate-hadrian
  stage: full-build
  tags:
    - x86_64-freebsd
  allow_failure: true
  variables:
    CONFIGURE_ARGS: "--with-gmp-includes=/usr/local/include --with-gmp-libraries=/usr/local/lib --with-iconv-includes=/usr/local/include --with-iconv-libraries=/usr/local/lib"
    HADRIAN_ARGS: "--docs=no-sphinx"
    GHC_VERSION: 8.6.3
    CABAL_INSTALL_VERSION: 3.0.0.0
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-portbld-freebsd.tar.xz"
    TEST_ENV: "x86_64-freebsd-hadrian"
    FLAVOUR: "validate"
  after_script:
    - cp -Rf $HOME/.cabal cabal-cache
    - .gitlab/ci.sh clean
  artifacts:
    when: always
    expire_in: 2 week
  cache:
    key: "freebsd-$GHC_VERSION"
    paths:
      - cabal-cache
      - toolchain

# Disabled due to lack of builder capacity
.validate-x86_64-freebsd-hadrian:
  extends: .build-x86_64-freebsd-hadrian
  stage: full-build

360 361 362 363
#################################
# x86_64-darwin
#################################

Ben Gamari's avatar
Ben Gamari committed
364
validate-x86_64-darwin:
Ben Gamari's avatar
Ben Gamari committed
365
  extends: .validate
366
  stage: full-build
Ben Gamari's avatar
Ben Gamari committed
367 368 369
  tags:
    - x86_64-darwin
  variables:
370
    GHC_VERSION: 8.8.3
Ben Gamari's avatar
Ben Gamari committed
371
    CABAL_INSTALL_VERSION: 3.0.0.0
372
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-apple-darwin.tar.xz"
Ben Gamari's avatar
Ben Gamari committed
373 374 375
    MACOSX_DEPLOYMENT_TARGET: "10.7"
    # Only Sierra and onwards supports clock_gettime. See #12858
    ac_cv_func_clock_gettime: "no"
376
    LANG: "en_US.UTF-8"
Ben Gamari's avatar
Ben Gamari committed
377
    CONFIGURE_ARGS: "--with-intree-gmp"
378
    TEST_ENV: "x86_64-darwin"
Ben Gamari's avatar
Ben Gamari committed
379
    BUILD_FLAVOUR: "perf"
Ben Gamari's avatar
Ben Gamari committed
380 381
  after_script:
    - cp -Rf $HOME/.cabal cabal-cache
Ben Gamari's avatar
Ben Gamari committed
382
    - .gitlab/ci.sh clean
383 384
  artifacts:
    when: always
385
    expire_in: 2 week
Ben Gamari's avatar
Ben Gamari committed
386
  cache:
387
    key: "darwin-$GHC_VERSION"
Ben Gamari's avatar
Ben Gamari committed
388 389 390 391
    paths:
      - cabal-cache
      - toolchain

392 393
# Disabled because of OS X CI capacity
.validate-x86_64-darwin-hadrian:
394 395 396 397
  stage: full-build
  tags:
    - x86_64-darwin
  variables:
398
    GHC_VERSION: 8.8.3
399 400 401 402
    MACOSX_DEPLOYMENT_TARGET: "10.7"
    ac_cv_func_clock_gettime: "no"
    LANG: "en_US.UTF-8"
    CONFIGURE_ARGS: --with-intree-gmp
403
    TEST_ENV: "x86_64-darwin-hadrian"
404
    FLAVOUR: "validate"
405
  script:
Ben Gamari's avatar
Ben Gamari committed
406 407 408 409
    - .gitlab/ci.sh setup
    - .gitlab/ci.sh configure
    - .gitlab/ci.sh build_hadrian
    - .gitlab/ci.sh test_hadrian
410 411
  after_script:
    - cp -Rf $HOME/.cabal cabal-cache
Ben Gamari's avatar
Ben Gamari committed
412
    - .gitlab/ci.sh clean
413 414 415 416 417 418 419 420 421
  artifacts:
    when: always
    expire_in: 2 week
    reports:
      junit: junit.xml
    paths:
      - ghc.tar.xz
      - junit.xml

Ben Gamari's avatar
Ben Gamari committed
422
.validate-linux:
Ben Gamari's avatar
Ben Gamari committed
423 424 425
  extends: .validate
  tags:
    - x86_64-linux
Ben Gamari's avatar
Ben Gamari committed
426 427
  variables:
    BUILD_FLAVOUR: "perf"
Ben Gamari's avatar
Ben Gamari committed
428
  before_script:
429 430 431
      # Build hyperlinked sources for documentation when building releases
    - |
      if [[ -n "$CI_COMMIT_TAG" ]]; then
Ben Gamari's avatar
Ben Gamari committed
432
        HADDOCK_HYPERLINKED_SOURCES=1
433
      fi
434

Ben Gamari's avatar
Ben Gamari committed
435 436 437 438 439
    # workaround for docker permissions
    - sudo chown ghc:ghc -R .
  after_script:
    - cp -Rf $HOME/.cabal cabal-cache
  cache:
Ben Gamari's avatar
Ben Gamari committed
440
    key: linux
Ben Gamari's avatar
Ben Gamari committed
441 442 443 444
    paths:
      - cabal-cache
      - toolchain

445 446 447 448 449
#################################
# aarch64-linux-deb9
#################################

.build-aarch64-linux-deb9:
Ben Gamari's avatar
Ben Gamari committed
450
  extends: .validate-linux
451
  stage: full-build
452
  image: "registry.gitlab.haskell.org/ghc/ci-images/aarch64-linux-deb9:$DOCKER_REV"
Ben Gamari's avatar
Ben Gamari committed
453
  allow_failure: true
454 455
  variables:
    TEST_ENV: "aarch64-linux-deb9"
456
    BIN_DIST_PREP_TAR_COMP: "ghc-aarch64-linux-deb9.tar.xz"
Ben Gamari's avatar
Ben Gamari committed
457 458 459 460 461
  cache:
    key: linux-aarch64-deb9
  tags:
    - aarch64-linux

462 463 464 465 466 467
validate-aarch64-linux-deb9:
  extends: .build-aarch64-linux-deb9
  artifacts:
    when: always
    expire_in: 2 week

468
nightly-aarch64-linux-deb9:
469
  <<: *nightly
470
  extends: .build-aarch64-linux-deb9
471 472 473
  variables:
    TEST_TYPE: slowtest

474 475 476 477 478 479 480 481
#################################
# armv7-linux-deb9
#################################

.build-armv7-linux-deb9:
  extends: .validate-linux
  stage: full-build
  image: "registry.gitlab.haskell.org/ghc/ci-images/armv7-linux-deb9:$DOCKER_REV"
482 483
  # Due to linker issues
  allow_failure: true
484 485
  variables:
    TEST_ENV: "armv7-linux-deb9"
486
    BIN_DIST_PREP_TAR_COMP: "ghc-armv7-linux-deb9.tar.xz"
487
    CONFIGURE_ARGS: "--host=armv7-linux-gnueabihf --build=armv7-linux-gnueabihf --target=armv7-linux-gnueabihf"
488 489
    # N.B. We disable ld.lld explicitly here because it appears to fail
    # non-deterministically on ARMv7. See #18280.
490 491
    LD: "ld.gold"
    GccUseLdOpt: "-fuse-ld=gold"
492 493 494 495 496 497 498 499 500 501 502 503
  cache:
    key: linux-armv7-deb9
  tags:
    - armv7-linux

validate-armv7-linux-deb9:
  extends: .build-armv7-linux-deb9
  artifacts:
    when: always
    expire_in: 2 week

nightly-armv7-linux-deb9:
504
  <<: *nightly
505 506 507 508
  extends: .build-armv7-linux-deb9
  variables:
    TEST_TYPE: slowtest

509 510 511 512 513
#################################
# i386-linux-deb9
#################################

.build-i386-linux-deb9:
514 515
  extends: .validate-linux
  stage: full-build
516
  image: "registry.gitlab.haskell.org/ghc/ci-images/i386-linux-deb9:$DOCKER_REV"
517 518
  variables:
    TEST_ENV: "i386-linux-deb9"
519
    BIN_DIST_PREP_TAR_COMP: "ghc-i386-deb9-linux.tar.xz"
520 521 522 523 524
  cache:
    key: linux-i386-deb9

validate-i386-linux-deb9:
  extends: .build-i386-linux-deb9
525 526 527
  artifacts:
    when: always
    expire_in: 2 week
528 529

nightly-i386-linux-deb9:
530
  <<: *nightly
531
  extends: .build-i386-linux-deb9
532 533 534
  variables:
    TEST_TYPE: slowtest

535 536 537
#################################
# x86_64-linux-deb9
#################################
Ben Gamari's avatar
Ben Gamari committed
538

539
.build-x86_64-linux-deb9:
Ben Gamari's avatar
Ben Gamari committed
540
  extends: .validate-linux
541
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
542 543
  variables:
    TEST_ENV: "x86_64-linux-deb9"
544
    BIN_DIST_PREP_TAR_COMP: "./ghc-x86_64-deb9-linux.tar.xz"
Ben Gamari's avatar
Ben Gamari committed
545 546 547
  cache:
    key: linux-x86_64-deb9

548 549
# Disabled to reduce CI load
.validate-x86_64-linux-deb9:
550
  extends: .build-x86_64-linux-deb9
551
  stage: full-build
552 553 554 555
  artifacts:
    when: always
    expire_in: 2 week

556 557 558 559 560
release-x86_64-linux-deb9:
  <<: *release
  extends: .build-x86_64-linux-deb9
  stage: full-build

561
nightly-x86_64-linux-deb9:
562
  <<: *nightly
563
  extends: .build-x86_64-linux-deb9
564
  stage: full-build
565 566 567
  variables:
    TEST_TYPE: slowtest

568 569 570
# N.B. Has DEBUG assertions enabled in stage2
validate-x86_64-linux-deb9-debug:
  extends: .build-x86_64-linux-deb9
571
  stage: full-build
572 573
  variables:
    BUILD_FLAVOUR: validate
574 575
    # Ensure that stage2 also has DEBUG enabled
    ValidateSpeed: SLOW
Ben Gamari's avatar
Ben Gamari committed
576 577
    # Override validate flavour default; see #16890.
    BUILD_SPHINX_PDF: "YES"
578
    TEST_TYPE: slowtest
579
    TEST_ENV: "x86_64-linux-deb9-debug"
580
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-deb9-linux-debug.tar.xz"
581 582 583
  artifacts:
    when: always
    expire_in: 2 week
584

585 586
# Disabled to alleviate CI load
.validate-x86_64-linux-deb9-llvm:
587
  extends: .build-x86_64-linux-deb9
588
  stage: full-build
Ben Gamari's avatar
Ben Gamari committed
589 590
  variables:
    BUILD_FLAVOUR: perf-llvm
591
    TEST_ENV: "x86_64-linux-deb9-llvm"
592

593
nightly-x86_64-linux-deb9-llvm:
594
  <<: *nightly
595 596 597 598 599 600
  extends: .build-x86_64-linux-deb9
  stage: full-build
  variables:
    BUILD_FLAVOUR: perf-llvm
    TEST_ENV: "x86_64-linux-deb9-llvm"

601 602 603 604
validate-x86_64-linux-deb9-integer-simple:
  extends: .build-x86_64-linux-deb9
  stage: full-build
  variables:
605
    BUILD_FLAVOUR: validate
606
    INTEGER_LIBRARY: integer-simple
607
    TEST_ENV: "x86_64-linux-deb9-integer-simple-validate"
608
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-deb9-linux-integer-simple.tar.xz"
609 610

nightly-x86_64-linux-deb9-integer-simple:
611
  <<: *nightly
612 613 614 615 616 617 618
  extends: .build-x86_64-linux-deb9
  stage: full-build
  variables:
    INTEGER_LIBRARY: integer-simple
    TEST_ENV: "x86_64-linux-deb9-integer-simple"
    TEST_TYPE: slowtest

619 620
validate-x86_64-linux-deb9-dwarf:
  extends: .build-x86_64-linux-deb9
621
  stage: full-build
622 623 624
  variables:
    CONFIGURE_ARGS: "--enable-dwarf-unwind"
    BUILD_FLAVOUR: dwarf
625
    TEST_ENV: "x86_64-linux-deb9-dwarf"
626
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-deb9-linux-dwarf.tar.xz"
627

Ben Gamari's avatar
Ben Gamari committed
628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647
#################################
# x86_64-linux-deb10
#################################

.build-x86_64-linux-deb10:
  extends: .validate-linux
  stage: full-build
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb10:$DOCKER_REV"
  variables:
    TEST_ENV: "x86_64-linux-deb10"
    BIN_DIST_PREP_TAR_COMP: "./ghc-x86_64-deb10-linux.tar.xz"
  cache:
    key: linux-x86_64-deb10

# Disabled to alleviate CI load
.validate-x86_64-linux-deb10:
  extends: .build-x86_64-linux-deb10
  stage: full-build

nightly-x86_64-linux-deb10:
648
  <<: *nightly
Ben Gamari's avatar
Ben Gamari committed
649 650 651 652 653 654 655 656
  extends: .build-x86_64-linux-deb10
  variables:
    TEST_TYPE: slowtest

release-x86_64-linux-deb10:
  <<: *release
  extends: .build-x86_64-linux-deb10

657 658 659
#################################
# x86_64-linux-deb8
#################################
Ben Gamari's avatar
Ben Gamari committed
660

661
.build-x86_64-linux-deb8:
Ben Gamari's avatar
Ben Gamari committed
662
  extends: .validate-linux
663
  stage: full-build
664
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
665
  # Due to #18298.
666
  allow_failure: true
667 668
  variables:
    TEST_ENV: "x86_64-linux-deb8"
669
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-deb8-linux.tar.xz"
670 671 672 673 674 675
    # Debian 8's Sphinx is too old to support the table directive's :widths:
    # option: https://sourceforge.net/p/docutils/patches/120/
    BUILD_SPHINX_HTML: "NO"
    BUILD_SPHINX_INFO: "NO"
    BUILD_SPHINX_PDF: "NO"
    BUILD_SPHINX_MAN: "NO"
Ben Gamari's avatar
Ben Gamari committed
676 677
  cache:
    key: linux-x86_64-deb8
678 679 680 681

release-x86_64-linux-deb8:
  <<: *release
  extends: .build-x86_64-linux-deb8
Ben Gamari's avatar
Ben Gamari committed
682

683 684 685 686
#################################
# x86_64-linux-alpine
#################################

687 688
.build-x86_64-linux-alpine-hadrian:
  extends: .validate-linux-hadrian
689 690 691 692 693 694
  stage: full-build
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-alpine:$DOCKER_REV"
  # There are currently a few failing tests
  allow_failure: true
  variables:
    TEST_ENV: "x86_64-linux-alpine"
695
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-alpine-linux.tar.xz"
696 697
    # Can't use ld.gold due to #13958.
    CONFIGURE_ARGS: "--disable-ld-override"
698
    HADRIAN_ARGS: "--docs=no-sphinx"
699 700 701
    # encoding004 due to lack of locale support
    # T10458 due to fact that dynamic linker tries to reload libAS
    BROKEN_TESTS: "encoding004 T10458"
702 703 704 705 706 707
  cache:
    key: linux-x86_64-alpine
  artifacts:
    when: always
    expire_in: 2 week

708
release-x86_64-linux-alpine:
709
  <<: *release
710
  extends: .build-x86_64-linux-alpine-hadrian
711 712

nightly-x86_64-linux-alpine:
713
  <<: *nightly
714
  extends: .build-x86_64-linux-alpine-hadrian
715

716 717 718 719
#################################
# x86_64-linux-centos7
#################################

720
.build-x86_64-linux-centos7:
721 722 723 724 725 726 727 728
  extends: .validate-linux
  stage: full-build
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-centos7:$DOCKER_REV"
  variables:
    # The sphinx release shipped with Centos 7 fails to build out documentation
    BUILD_SPHINX_HTML: "NO"
    BUILD_SPHINX_PDF: "NO"
    TEST_ENV: "x86_64-linux-centos7"
729
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-centos7-linux.tar.xz"
Ben Gamari's avatar
Ben Gamari committed
730 731
    # CentOS seems to default to ascii
    LANG: "en_US.UTF-8"
732 733
  cache:
    key: linux-x86_64-centos7
734 735 736 737

release-x86_64-linux-centos7:
  <<: *release
  extends: .build-x86_64-linux-centos7
738 739 740 741 742

#################################
# x86_64-linux-fedora27
#################################

743
validate-x86_64-linux-fedora27:
Ben Gamari's avatar
Ben Gamari committed
744
  extends: .validate-linux
745
  stage: full-build
746
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-fedora27:$DOCKER_REV"
747 748
  variables:
    TEST_ENV: "x86_64-linux-fedora27"
749
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-fedora27-linux.tar.xz"
Ben Gamari's avatar
Ben Gamari committed
750 751
  cache:
    key: linux-x86_64-fedora27
752 753
  artifacts:
    when: always
754 755 756
    # These are used for head.hackage jobs therefore we keep them around for
    # longer.
    expire_in: 8 week
Ben Gamari's avatar
Ben Gamari committed
757

Ben Gamari's avatar
Ben Gamari committed
758
############################################################
759
# Validation via Pipelines (Windows)
Ben Gamari's avatar
Ben Gamari committed
760 761
############################################################

762
.build-windows:
763
  # For the reasons given in #17777 this build isn't reliable.
764
  allow_failure: true
Ben Gamari's avatar
Ben Gamari committed
765
  before_script:
766
    - git clean -xdf
767

Ben Gamari's avatar
Ben Gamari committed
768 769
    # Setup toolchain
    - bash .gitlab/ci.sh setup
770
  after_script:
Ben Gamari's avatar
Ben Gamari committed
771 772 773
    - |
      Copy-Item -Recurse -Path $Env:APPDATA\cabal -Destination cabal-cache
    - bash .gitlab/ci.sh clean
774
  dependencies: []
Tamar Christina's avatar
Tamar Christina committed
775
  variables:
776
    #FORCE_SYMLINKS: 1
777
    LANG: "en_US.UTF-8"
778
    SPHINXBUILD: "/mingw64/bin/sphinx-build.exe"
Ben Gamari's avatar
Ben Gamari committed
779
    CABAL_INSTALL_VERSION: 3.0.0.0
780
    GHC_VERSION: "8.8.3"
Ben Gamari's avatar
Ben Gamari committed
781 782 783
  cache:
    paths:
      - cabal-cache
Ben Gamari's avatar
Ben Gamari committed
784
      - toolchain
Ben Gamari's avatar
Ben Gamari committed
785 786
      - ghc-tarballs

787 788
.build-windows-hadrian:
  extends: .build-windows
789
  stage: full-build
Ben Gamari's avatar
Ben Gamari committed
790
  variables:
791
    FLAVOUR: "validate"
Ben Gamari's avatar
Ben Gamari committed
792 793 794 795
    # skipping perf tests for now since we build a quick-flavoured GHC,
    # which might result in some broken perf tests?
    HADRIAN_ARGS: "--docs=no-sphinx --skip-perf"

Ben Gamari's avatar
Ben Gamari committed
796
  script:
Ben Gamari's avatar
Ben Gamari committed
797 798 799
    - bash .gitlab/ci.sh configure
    - bash .gitlab/ci.sh build_hadrian
    - bash .gitlab/ci.sh test_hadrian
Ben Gamari's avatar
Ben Gamari committed
800
  tags:
Ben Gamari's avatar
Ben Gamari committed
801 802
    - new-x86_64-windows
    - test
803
  artifacts:
804 805 806
    reports:
      junit: junit.xml
    expire_in: 2 week
807 808 809
    when: always
    paths:
      - ghc.tar.xz
810
      - junit.xml
Ben Gamari's avatar
Ben Gamari committed
811

812
validate-x86_64-windows-hadrian:
813 814 815
  extends: .build-windows-hadrian
  variables:
    MSYSTEM: MINGW64
816
    TEST_ENV: "x86_64-windows-hadrian"
817
  cache:
818
    key: "x86_64-windows-hadrian-$WINDOWS_TOOLCHAIN_VERSION"
819 820

nightly-i386-windows-hadrian:
821
  <<: *nightly
822 823 824
  extends: .build-windows-hadrian
  variables:
    MSYSTEM: MINGW32
825
    TEST_ENV: "i386-windows-hadrian"
826
  cache:
827
    key: "i386-windows-hadrian-$WINDOWS_TOOLCHAIN_VERSION"
828 829 830

.build-windows-make:
  extends: .build-windows
831
  stage: full-build
832
  variables:
833
    BUILD_FLAVOUR: "quick"
834
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-mingw32.tar.xz"
835
  script:
Ben Gamari's avatar
Ben Gamari committed
836 837 838
    - bash .gitlab/ci.sh configure
    - bash .gitlab/ci.sh build_make
    - bash .gitlab/ci.sh test_make
839
  tags:
Ben Gamari's avatar
Ben Gamari committed
840 841
    - new-x86_64-windows
    - test
842 843
  artifacts:
    when: always
844
    expire_in: 2 week
845 846 847
    reports:
      junit: junit.xml
    paths:
848
      # N.B. variable interpolation apparently doesn't work on Windows so
849
      # this can't be $BIN_DIST_PREP_TAR_COMP
850
      - "ghc-x86_64-mingw32.tar.xz"
851
      - junit.xml
Ben Gamari's avatar
Ben Gamari committed
852

Ben Gamari's avatar
Ben Gamari committed
853
.build-x86_64-windows-make:
854 855 856
  extends: .build-windows-make
  variables:
    MSYSTEM: MINGW64
857
    TEST_ENV: "x86_64-windows"
858
  cache:
859
    key: "x86_64-windows-$WINDOWS_TOOLCHAIN_VERSION"
860

Ben Gamari's avatar
Ben Gamari committed
861 862 863
validate-x86_64-windows:
  extends: .build-x86_64-windows-make

864
nightly-x86_64-windows:
865
  <<: *nightly
Ben Gamari's avatar
Ben Gamari committed
866
  extends: .build-x86_64-windows-make
867 868 869 870
  stage: full-build
  variables:
    BUILD_FLAVOUR: "validate"

871 872
# Normal Windows validate builds are profiled; that won't do for releases.
release-x86_64-windows:
873
  <<: *release
874 875 876
  extends: validate-x86_64-windows
  variables:
    BUILD_FLAVOUR: "perf"
877 878 879 880 881 882 883
    #
release-x86_64-windows-integer-simple:
  <<: *release
  extends: validate-x86_64-windows
  variables:
    INTEGER_LIBRARY: integer-simple
    BUILD_FLAVOUR: "perf"
884

Ben Gamari's avatar
Ben Gamari committed
885 886

.build-i386-windows-make:
887 888 889
  extends: .build-windows-make
  variables:
    MSYSTEM: MINGW32
890 891
    # Due to #15934
    BUILD_PROF_LIBS: "NO"
892
    TEST_ENV: "i386-windows"
893 894
  # Due to #17736
  allow_failure: true
895
  cache:
896
    key: "i386-windows-$WINDOWS_TOOLCHAIN_VERSION"
897

Ben Gamari's avatar
Ben Gamari committed
898 899 900 901 902 903 904 905 906 907 908
validate-i386-windows:
  extends: .build-i386-windows-make
  variables:
    BUILD_FLAVOUR: "perf"

release-i386-windows:
  <<: *release
  extends: .build-i386-windows-make
  variables:
    BUILD_FLAVOUR: "perf"

909
nightly-i386-windows:
910
  <<: *nightly
Ben Gamari's avatar
Ben Gamari committed
911
  extends: .build-i386-windows-make
912

913 914 915 916
############################################################
# Cleanup
############################################################

917 918
# Note [Cleaning up after shell executor]
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
919 920 921
#
# As noted in [1], gitlab-runner's shell executor doesn't clean up its working
# directory after builds. Unfortunately, we are forced to use the shell executor
922 923 924 925 926 927 928 929 930
# on Darwin. To avoid running out of disk space we add a stage at the end of
# the build to remove the /.../GitLabRunner/builds directory. Since we only run a
# single build at a time on Darwin this should be safe.
#
# We used to have a similar cleanup job on Windows as well however it ended up
# being quite fragile as we have multiple Windows builders yet there is no
# guarantee that the cleanup job is run on the same machine as the build itself
# was run. Consequently we were forced to instead handle cleanup with a separate
# cleanup cron job on Windows.
931 932 933
#
# [1] https://gitlab.com/gitlab-org/gitlab-runner/issues/3856

934 935 936 937 938 939
# See Note [Cleanup after shell executor]
cleanup-darwin:
  stage: cleanup
  tags:
    - x86_64-darwin
  when: always
940
  dependencies: []
941 942 943 944 945 946 947 948 949 950 951
  before_script:
    - echo "Time to clean up"
  script:
    - echo "Let's go"
  after_script:
    - BUILD_DIR=$CI_PROJECT_DIR
    - echo "Cleaning $BUILD_DIR"
    - cd $HOME
    - rm -Rf $BUILD_DIR/*
    - exit 0

952 953 954 955 956 957 958 959
############################################################
# Packaging
############################################################

doc-tarball:
  stage: packaging
  tags:
    - x86_64-linux
960
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
961
  dependencies:
962
    - validate-x86_64-linux-deb9-debug
963
    - validate-x86_64-windows
964
  variables:
965
    LINUX_BINDIST: "ghc-x86_64-deb9-linux-debug.tar.xz"
966
    WINDOWS_BINDIST: "ghc-x86_64-mingw32.tar.xz"
967 968
  # Due to Windows allow_failure
  allow_failure: true
969 970 971 972 973 974 975 976
  artifacts:
    paths:
      - haddock.html.tar.xz
      - libraries.html.tar.xz
      - users_guide.html.tar.xz
      - index.html
      - "*.pdf"
  script:
977 978 979 980 981 982 983 984 985
    - |
      if [ ! -f "$LINUX_BINDIST" ]; then
        echo "Error: $LINUX_BINDIST does not exist. Did the Debian 9 job fail?"
        exit 1
      fi
      if [ ! -f "$WINDOWS_BINDIST" ]; then
        echo "Error: $WINDOWS_BINDIST does not exist. Did the 64-bit Windows job fail?"
        exit 1
      fi
986
    - rm -Rf docs
987
    - bash -ex distrib/mkDocs/mkDocs $LINUX_BINDIST $WINDOWS_BINDIST
988
    - ls -lh
989
    - mv docs/*.tar.xz docs/index.html .
990

991 992 993 994
source-tarball:
  stage: packaging
  tags:
    - x86_64-linux
995
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
996
  dependencies: []
997 998 999
  rules:
    - if: $CI_COMMIT_TAG
      when: always
1000 1001 1002 1003 1004
  artifacts:
    paths:
      - ghc-*.tar.xz
      - version
  script:
1005
    - python3 mk/get-win32-tarballs.py download all
1006 1007 1008 1009
    - ./boot
    - ./configure
    - make sdist
    - mv sdistprep/*.xz  .
1010 1011 1012
    - make show! --quiet VALUE=ProjectVersion > version
    - source version
    - echo "$ProjectVersion" > version
1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025


############################################################
# Testing via head.hackage
############################################################

# Triggering jobs in the ghc/head.hackage project requires that we have a job
# token for that repository. Furthermore the head.hackage CI job must have
# access to an unprivileged access token with the ability to query the ghc/ghc
# project such that it can find the job ID of the fedora27 job for the current
# pipeline.

.hackage:
1026
  stage: testing
1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041
  image: ghcci/x86_64-linux-deb9:0.2
  tags:
    - x86_64-linux
  dependencies: []
  variables:
    HEAD_HACKAGE_PROJECT_ID: "78"
  script:
    - bash .gitlab/start-head.hackage.sh

hackage:
  extends: .hackage
  when: manual

hackage-label:
  extends: .hackage
1042 1043
  rules:
    - if: '$CI_MERGE_REQUEST_LABELS =~ /.*user-facing.*/'
1044 1045

nightly-hackage:
1046
  <<: *nightly
1047
  extends: .hackage
1048

1049 1050 1051 1052 1053 1054 1055
############################################################
# Nofib testing
############################################################

perf-nofib:
  stage: testing
  dependencies:
1056
    - validate-x86_64-linux-deb9-dwarf
1057
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
1058 1059 1060 1061
  rules:
    - if: $CI_MERGE_REQUEST_ID
    - if: '$CI_COMMIT_BRANCH == "master"'
    - if: '$CI_COMMIT_BRANCH =~ /ghc-[0.9]+\.[0-9]+/'
1062 1063 1064 1065 1066 1067
  tags:
    - x86_64-linux
  script:
    - root=$(pwd)/ghc
    - |
      mkdir tmp
1068
      tar -xf ghc-x86_64-deb9-linux-dwarf.tar.xz -C tmp
1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088
      pushd tmp/ghc-*/
      ./configure --prefix=$root
      make install
      popd
      rm -Rf tmp
    - export BOOT_HC=$(which ghc)
    - cabal update; cabal install -w $BOOT_HC regex-compat
    - export PATH=$root/bin:$PATH
    - make -C nofib boot mode=fast -j$CPUS
    - "make -C nofib EXTRA_RUNTEST_OPTS='-cachegrind +RTS -V0 -RTS' NoFibRuns=1 mode=fast -j$CPUS 2>&1 | tee nofib.log"
  artifacts:
    expire_in: 12 week
    when: always
    paths:
      - nofib.log

############################################################
# Documentation deployment via GitLab Pages
############################################################

1089 1090 1091 1092 1093
pages:
  stage: deploy
  dependencies:
    - doc-tarball
  image: ghcci/x86_64-linux-deb9:0.2
1094 1095
  # Due to Windows allow_failure
  allow_failure: true
1096 1097 1098 1099 1100 1101 1102
  tags:
    - x86_64-linux
  script:
    - mkdir -p public/doc
    - tar -xf haddock.html.tar.xz -C public/doc
    - tar -xf libraries.html.tar.xz -C public/doc
    - tar -xf users_guide.html.tar.xz -C public/doc
1103 1104 1105 1106 1107 1108
    - |
      cat >public/index.html <<EOF
      <!DOCTYPE HTML>
      <meta charset="UTF-8">
      <meta http-equiv="refresh" content="1; url=doc/">
      EOF
1109
    - cp -f index.html public/doc
1110 1111
  rules:
    - if: '$CI_COMMIT_BRANCH == "master"'
1112 1113 1114
  artifacts:
    paths:
      - public