.gitlab-ci.yml 30.9 KB
Newer Older
1
2
variables:
  GIT_SSL_NO_VERIFY: "1"
3

4
  # Commit of ghc/ci-images repository from which to pull Docker images
5
  DOCKER_REV: 8beb70e553e521796f4250000107c008b477040f
6

7
8
9
10
  # Sequential version number capturing the versions of all tools fetched by
  # .gitlab/win32-init.sh.
  WINDOWS_TOOLCHAIN_VERSION: 1

Ben Gamari's avatar
Ben Gamari committed
11
12
13
  # Disable shallow clones; they break our linting rules
  GIT_DEPTH: 0

14
15
16
17
before_script:
  - git submodule sync --recursive
  - git submodule update --init --recursive
  - git checkout .gitmodules
18
  - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
19

Ben Gamari's avatar
Ben Gamari committed
20
stages:
21
22
23
24
25
26
27
28
  - lint        # Source linting
  - quick-build # A very quick smoke-test to weed out broken commits
  - build       # A quick smoke-test to weed out broken commits
  - full-build  # Build all the things
  - cleanup     # See Note [Cleanup after the shell executor]
  - packaging   # Source distribution, etc.
  - testing     # head.hackage correctness and compiler performance testing
  - deploy      # push documentation
Ben Gamari's avatar
Ben Gamari committed
29

Ben Gamari's avatar
Ben Gamari committed
30
# N.B.Don't run on wip/ branches, instead on run on merge requests.
31
32
33
34
35
36
.only-default: &only-default
  only:
    - master
    - /ghc-[0-9]+\.[0-9]+/
    - merge_requests
    - tags
37
    - web
38

Ben Gamari's avatar
Ben Gamari committed
39
############################################################
40
# Runner Tags
Ben Gamari's avatar
Ben Gamari committed
41
42
43
44
45
46
47
48
49
50
############################################################
#
# * x86_64-linux: Any Docker-capable x86_64 Linux machine
# * aarch64-linux: Any Docker-capable AArch64 Linux machine
# * x86_64-windows: A x86_64 Windows machine
# * lint: Any Docker-capable x86_64 Linux machine; distinct from
#         x86_64-linux to ensure low-latency availability.
#


Ben Gamari's avatar
Ben Gamari committed
51
52
53
54
55
56
############################################################
# Linting
############################################################

ghc-linters:
  stage: lint
57
  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
Ben Gamari's avatar
Ben Gamari committed
58
  script:
Ben Gamari's avatar
Ben Gamari committed
59
    - git fetch "$CI_MERGE_REQUEST_PROJECT_URL" $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
60
    - base="$(git merge-base FETCH_HEAD $CI_COMMIT_SHA)"
61
    - "echo Linting changes between $base..$CI_COMMIT_SHA"
62
63
    #    - validate-commit-msg .git $(git rev-list $base..$CI_COMMIT_SHA)
    - validate-whitespace .git $(git rev-list $base..$CI_COMMIT_SHA)
64
65
    - .gitlab/linters/check-makefiles.py commits $base $CI_COMMIT_SHA
    - .gitlab/linters/check-cpp.py commits $base $CI_COMMIT_SHA
66
    - .gitlab/linters/check-version-number.sh
67
    - python3 utils/checkUniques/check-uniques.py .
68
  dependencies: []
Ben Gamari's avatar
Ben Gamari committed
69
70
  tags:
    - lint
71
72
73
74
  only:
    refs:
      - merge_requests

75
# Run mypy Python typechecker on linter scripts.
Ben Gamari's avatar
Ben Gamari committed
76
77
78
79
80
81
82
83
84
85
lint-linters:
  <<: *only-default
  stage: lint
  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
  script:
    - mypy .gitlab/linters/*.py
  dependencies: []
  tags:
    - lint

86
# Check that .T files all parse by listing broken tests.
87
lint-testsuite:
Ben Gamari's avatar
Ben Gamari committed
88
  <<: *only-default
89
90
91
92
93
94
95
96
  stage: lint
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
  script:
    - make -Ctestsuite list_broken TEST_HC=ghc
  dependencies: []
  tags:
    - lint

97
98
99
100
101
102
103
104
105
106
107
# Run mypy Python typechecker on testsuite driver
typecheck-testsuite:
  <<: *only-default
  stage: lint
  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
  script:
    - mypy testsuite/driver/runtests.py
  dependencies: []
  tags:
    - lint

108
109
110
111
# We allow the submodule checker to fail when run on merge requests (to
# accomodate, e.g., haddock changes not yet upstream) but not on `master` or
# Marge jobs.
.lint-submods:
Ben Gamari's avatar
Ben Gamari committed
112
  <<: *only-default
113
114
115
  stage: lint
  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
  script:
Ben Gamari's avatar
Ben Gamari committed
116
    - git fetch "$CI_MERGE_REQUEST_PROJECT_URL" $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
Ben Gamari's avatar
Ben Gamari committed
117
    - base="$(git merge-base FETCH_HEAD $CI_COMMIT_SHA)"
118
    - "echo Linting submodule changes between $base..$CI_COMMIT_SHA"
119
    - git submodule foreach git remote update
120
      # TODO: Fix submodule linter
Ben Gamari's avatar
Ben Gamari committed
121
    - submodchecker . $(git rev-list $base..$CI_COMMIT_SHA) || true
122
123
124
125
  dependencies: []
  tags:
    - lint

126
127
128
129
130
131
lint-submods-marge:
  extends: .lint-submods
  only:
    refs:
      - merge_requests
    variables:
Ben Gamari's avatar
Ben Gamari committed
132
      - "$CI_MERGE_REQUEST_LABELS =~ /.*marge_bot_batch_merge_job.*/"
133
134
135

lint-submods-mr:
  extends: .lint-submods
136
137
  # Allow failure since any necessary submodule patches may not be upstreamed
  # yet.
138
139
140
141
  allow_failure: true
  only:
    refs:
      - merge_requests
142
143
  except:
    variables:
Ben Gamari's avatar
Ben Gamari committed
144
      - "$CI_MERGE_REQUEST_LABELS =~ /.*marge_bot_batch_merge_job.*/"
Ben Gamari's avatar
Ben Gamari committed
145

146
147
148
lint-submods-branch:
  extends: .lint-submods
  script:
149
    - "echo Linting submodule changes between $CI_COMMIT_BEFORE_SHA..$CI_COMMIT_SHA"
150
    - git submodule foreach git remote update
Ben Gamari's avatar
Ben Gamari committed
151
    - submodchecker . $(git rev-list $CI_COMMIT_BEFORE_SHA..$CI_COMMIT_SHA)
152
153
154
155
156
  only:
    refs:
      - master
      - /ghc-[0-9]+\.[0-9]+/

157
.lint-changelogs:
Ben Gamari's avatar
Ben Gamari committed
158
  <<: *only-default
159
160
161
162
163
164
165
166
167
168
169
170
171
172
  stage: lint
  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
  dependencies: []
  tags:
    - lint
  script:
    - |
      grep TBA libraries/*/changelog.md && (
          echo "Error: Found \"TBA\"s in changelogs."
          exit 1
      )

lint-changelogs:
  extends: .lint-changelogs
173
  # Allow failure since this isn't a final release.
174
175
176
177
178
179
180
181
  allow_failure: true
  only:
    refs:
      - /ghc-[0-9]+\.[0-9]+/

lint-release-changelogs:
  extends: .lint-changelogs
  only:
182
183
    refs:
      - /ghc-[0-9]+\.[0-9]+\.[0-9]+-.*/
184
185


Ben Gamari's avatar
Ben Gamari committed
186
187
188
############################################################
# Validation via Pipelines (hadrian)
############################################################
189

Ben Gamari's avatar
Ben Gamari committed
190
.validate-hadrian:
191
  <<: *only-default
192
193
  variables:
    FLAVOUR: "validate"
Ben Gamari's avatar
Ben Gamari committed
194
  script:
195
    - cabal update
196
    - git clean -xdf && git submodule foreach git clean -xdf
Ben Gamari's avatar
Ben Gamari committed
197
    - .gitlab/prepare-system.sh
Ben Gamari's avatar
Ben Gamari committed
198
199
    - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
    - ./boot
Ben Gamari's avatar
Ben Gamari committed
200
    - ./configure $CONFIGURE_ARGS
201
    - hadrian/build.cabal.sh --flavour=$FLAVOUR -j`mk/detect-cpu-count.sh` --docs=no-sphinx binary-dist
202
    - mv _build/bindist/ghc*.tar.xz ghc.tar.xz
203
204
    - export TOP=$(pwd)
    - cd _build/bindist/ghc-*/ && ./configure --prefix=$TOP/_build/install && make install && cd ../../../
205
206
    - |
      # Prepare to push git notes.
207
      export METRICS_FILE=$CI_PROJECT_DIR/performance-metrics.tsv
208
209
      git config user.email "ben+ghc-ci@smart-cactus.org"
      git config user.name "GHC GitLab CI"
210
    - hadrian/build.cabal.sh --flavour=$FLAVOUR -j`mk/detect-cpu-count.sh` --docs=no-sphinx test --summary-junit=./junit.xml --test-compiler=$TOP/_build/install/bin/ghc || (.gitlab/push-test-metrics.sh && false)
211
212
    - |
      # Push git notes.
213
      .gitlab/push-test-metrics.sh
Ben Gamari's avatar
Ben Gamari committed
214
215
  cache:
    key: hadrian
216
    paths:
Ben Gamari's avatar
Ben Gamari committed
217
      - cabal-cache
218
  artifacts:
219
220
221
    reports:
      junit: junit.xml
    expire_in: 2 week
222
223
    paths:
      - ghc.tar.xz
224
      - junit.xml
225

226
.validate-linux-hadrian:
Ben Gamari's avatar
Ben Gamari committed
227
  extends: .validate-hadrian
228
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
229
230
  variables:
    TEST_ENV: "x86_64-linux-deb9-hadrian"
Ben Gamari's avatar
Ben Gamari committed
231
232
233
  before_script:
    # workaround for docker permissions
    - sudo chown ghc:ghc -R .
234
235
236
    - git submodule sync --recursive
    - git submodule update --init --recursive
    - git checkout .gitmodules
237
    - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
Ben Gamari's avatar
Ben Gamari committed
238
239
240
  tags:
    - x86_64-linux

241
242
243
244
validate-x86_64-linux-deb9-hadrian:
  extends: .validate-linux-hadrian
  stage: build

245
validate-x86_64-linux-deb9-unreg-hadrian:
246
  extends: .validate-linux-hadrian
247
248
249
  stage: full-build
  variables:
    CONFIGURE_ARGS: --enable-unregisterised
250
    TEST_ENV: "x86_64-linux-deb9-unreg-hadrian"
251

252
hadrian-ghc-in-ghci:
253
  <<: *only-default
254
  stage: quick-build
255
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
256
257
258
259
260
261
  before_script:
    # workaround for docker permissions
    - sudo chown ghc:ghc -R .
    - git submodule sync --recursive
    - git submodule update --init --recursive
    - git checkout .gitmodules
262
263
  variables:
    GHC_FLAGS: -Werror
264
265
266
267
  tags:
    - x86_64-linux
  script:
    - cabal update
268
    - cd hadrian; cabal new-build --project-file=ci.project; cd ..
269
    - git clean -xdf && git submodule foreach git clean -xdf
Ben Gamari's avatar
Ben Gamari committed
270
    - .gitlab/prepare-system.sh
271
272
273
    - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
    - ./boot
    - ./configure $CONFIGURE_ARGS
274
    # Load ghc-in-ghci then immediately exit and check the modules loaded
275
    - echo ":q" | hadrian/ghci.sh -j`mk/detect-cpu-count.sh`| tail -n2 | grep "Ok,"
276
277
278
279
280
  cache:
    key: hadrian-ghci
    paths:
      - cabal-cache

Ben Gamari's avatar
Ben Gamari committed
281
282
283
284
285
############################################################
# Validation via Pipelines (make)
############################################################

.validate:
286
  <<: *only-default
287
288
  variables:
    TEST_TYPE: test
Ben Gamari's avatar
Ben Gamari committed
289
  before_script:
290
    - git clean -xdf && git submodule foreach git clean -xdf
Ben Gamari's avatar
Ben Gamari committed
291
  script:
Ben Gamari's avatar
Ben Gamari committed
292
    - ./boot
Ben Gamari's avatar
Ben Gamari committed
293
    - ./configure $CONFIGURE_ARGS
Ben Gamari's avatar
Ben Gamari committed
294
295
    - |
      THREADS=`mk/detect-cpu-count.sh`
Matthew Pickering's avatar
Matthew Pickering committed
296
      make V=0 -j$THREADS WERROR=-Werror
297
298
    - make binary-dist-prep TAR_COMP_OPTS="-1"
    - make test_bindist TEST_PREP=YES
299
300
    - |
      # Prepare to push git notes.
301
      METRICS_FILE=$CI_PROJECT_DIR/performance-metrics.tsv
302
303
      git config user.email "ben+ghc-ci@smart-cactus.org"
      git config user.name "GHC GitLab CI"
Ben Gamari's avatar
Ben Gamari committed
304
305
    - |
      THREADS=`mk/detect-cpu-count.sh`
306
      make $TEST_TYPE THREADS=$THREADS JUNIT_FILE=../../junit.xml METRICS_FILE=$METRICS_FILE || (METRICS_FILE=$METRICS_FILE .gitlab/push-test-metrics.sh && false)
307
308
309
    - |
      # Push git notes.
      METRICS_FILE=$METRICS_FILE .gitlab/push-test-metrics.sh
310
  dependencies: []
311
312
313
  artifacts:
    reports:
      junit: junit.xml
314
    expire_in: 2 week
315
    paths:
316
      - $BIN_DIST_PREP_TAR_COMP
317
      - junit.xml
318
      - performance-metrics.tsv
319

320
321
322
323
#################################
# x86_64-darwin
#################################

Ben Gamari's avatar
Ben Gamari committed
324
validate-x86_64-darwin:
Ben Gamari's avatar
Ben Gamari committed
325
  extends: .validate
326
  stage: full-build
Ben Gamari's avatar
Ben Gamari committed
327
328
329
  tags:
    - x86_64-darwin
  variables:
330
    GHC_VERSION: 8.6.5
331
    CABAL_INSTALL_VERSION: 2.4.1.0
332
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-apple-darwin.tar.xz"
Ben Gamari's avatar
Ben Gamari committed
333
334
335
    MACOSX_DEPLOYMENT_TARGET: "10.7"
    # Only Sierra and onwards supports clock_gettime. See #12858
    ac_cv_func_clock_gettime: "no"
336
    LANG: "en_US.UTF-8"
337
    CONFIGURE_ARGS: --with-intree-gmp
338
    TEST_ENV: "x86_64-darwin"
Ben Gamari's avatar
Ben Gamari committed
339
  before_script:
Ben Gamari's avatar
Ben Gamari committed
340
    - git clean -xdf && git submodule foreach git clean -xdf
341
342
343
    - git submodule sync --recursive
    - git submodule update --init --recursive
    - git checkout .gitmodules
344
    - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
345

Ben Gamari's avatar
Ben Gamari committed
346
347
348
349
    - bash .gitlab/darwin-init.sh
    - PATH="`pwd`/toolchain/bin:$PATH"
  after_script:
    - cp -Rf $HOME/.cabal cabal-cache
350
351
  artifacts:
    when: always
352
    expire_in: 2 week
Ben Gamari's avatar
Ben Gamari committed
353
  cache:
354
    key: "darwin-$GHC_VERSION"
Ben Gamari's avatar
Ben Gamari committed
355
356
357
358
    paths:
      - cabal-cache
      - toolchain

359
360
# Disabled because of OS X CI capacity
.validate-x86_64-darwin-hadrian:
361
362
363
364
365
366
367
368
369
370
  <<: *only-default
  stage: full-build
  tags:
    - x86_64-darwin
  variables:
    GHC_VERSION: 8.6.3
    MACOSX_DEPLOYMENT_TARGET: "10.7"
    ac_cv_func_clock_gettime: "no"
    LANG: "en_US.UTF-8"
    CONFIGURE_ARGS: --with-intree-gmp
371
    TEST_ENV: "x86_64-darwin-hadrian"
372
    FLAVOUR: "validate"
373
374
375
376
377
378
379
380
381
382
383
384
385
  before_script:
    - git clean -xdf && git submodule foreach git clean -xdf
    - git submodule sync --recursive
    - git submodule update --init --recursive
    - git checkout .gitmodules
    - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"

    - bash .gitlab/darwin-init.sh
    - PATH="`pwd`/toolchain/bin:$PATH"
  script:
    - cabal update
    - ./boot
    - ./configure $CONFIGURE_ARGS
386
    - hadrian/build.cabal.sh --flavour=$FLAVOUR -j`mk/detect-cpu-count.sh` --docs=no-sphinx binary-dist
387
    - mv _build/bindist/ghc*.tar.xz ghc.tar.xz
388
389
390
    - export TOP=$(pwd)
    - cd _build/bindist/ghc-*/ && ./configure --prefix=$TOP/_build/install && make install && cd ../../../
    - hadrian/build.cabal.sh --flavour=$FLAVOUR -j`mk/detect-cpu-count.sh` --docs=no-sphinx test --summary-junit=./junit.xml --test-compiler=$TOP/_build/install/bin/ghc
391
392
393
394
395
396
397
398
399
400
401
  after_script:
    - cp -Rf $HOME/.cabal cabal-cache
  artifacts:
    when: always
    expire_in: 2 week
    reports:
      junit: junit.xml
    paths:
      - ghc.tar.xz
      - junit.xml

Ben Gamari's avatar
Ben Gamari committed
402
.validate-linux:
Ben Gamari's avatar
Ben Gamari committed
403
404
405
406
  extends: .validate
  tags:
    - x86_64-linux
  before_script:
Ben Gamari's avatar
Ben Gamari committed
407
    - git clean -xdf && git submodule foreach git clean -xdf
408
409
410
    - git submodule sync --recursive
    - git submodule update --init --recursive
    - git checkout .gitmodules
411
    - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
412
413
414
415
416
      # Build hyperlinked sources for documentation when building releases
    - |
      if [[ -n "$CI_COMMIT_TAG" ]]; then
        echo "EXTRA_HADDOCK_OPTS += --hyperlinked-source --quickjump" >> mk/build.mk
      fi
417

Ben Gamari's avatar
Ben Gamari committed
418
    - .gitlab/prepare-system.sh
Ben Gamari's avatar
Ben Gamari committed
419
420
421
422
423
    # workaround for docker permissions
    - sudo chown ghc:ghc -R .
  after_script:
    - cp -Rf $HOME/.cabal cabal-cache
  cache:
Ben Gamari's avatar
Ben Gamari committed
424
    key: linux
Ben Gamari's avatar
Ben Gamari committed
425
426
427
428
    paths:
      - cabal-cache
      - toolchain

429
430
431
432
433
#################################
# aarch64-linux-deb9
#################################

.build-aarch64-linux-deb9:
Ben Gamari's avatar
Ben Gamari committed
434
  extends: .validate-linux
435
  stage: full-build
436
  image: "registry.gitlab.haskell.org/ghc/ci-images/aarch64-linux-deb9:$DOCKER_REV"
Ben Gamari's avatar
Ben Gamari committed
437
  allow_failure: true
438
439
  variables:
    TEST_ENV: "aarch64-linux-deb9"
440
    BIN_DIST_PREP_TAR_COMP: "ghc-aarch64-linux-deb9.tar.xz"
Ben Gamari's avatar
Ben Gamari committed
441
442
443
444
445
  cache:
    key: linux-aarch64-deb9
  tags:
    - aarch64-linux

446
447
448
449
450
451
validate-aarch64-linux-deb9:
  extends: .build-aarch64-linux-deb9
  artifacts:
    when: always
    expire_in: 2 week

452
nightly-aarch64-linux-deb9:
453
  extends: .build-aarch64-linux-deb9
454
  artifacts:
455
    expire_in: 2 year
456
457
458
459
460
461
  variables:
    TEST_TYPE: slowtest
  only:
    variables:
      - $NIGHTLY

462
463
464
465
466
467
468
469
470
471
472
#################################
# armv7-linux-deb9
#################################

.build-armv7-linux-deb9:
  extends: .validate-linux
  stage: full-build
  image: "registry.gitlab.haskell.org/ghc/ci-images/armv7-linux-deb9:$DOCKER_REV"
  allow_failure: true
  variables:
    TEST_ENV: "armv7-linux-deb9"
473
    BIN_DIST_PREP_TAR_COMP: "ghc-armv7-linux-deb9.tar.xz"
Ben Gamari's avatar
Ben Gamari committed
474
    CONFIGURE_ARGS: "--host=armv7-linux-gnueabihf --build=armv7-linux-gnueabihf --target=armv7-linux-gnueabihf"
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
  cache:
    key: linux-armv7-deb9
  tags:
    - armv7-linux

validate-armv7-linux-deb9:
  extends: .build-armv7-linux-deb9
  artifacts:
    when: always
    expire_in: 2 week

nightly-armv7-linux-deb9:
  extends: .build-armv7-linux-deb9
  artifacts:
    expire_in: 2 year
  variables:
    TEST_TYPE: slowtest
  only:
    variables:
      - $NIGHTLY

496
497
498
499
500
#################################
# i386-linux-deb9
#################################

.build-i386-linux-deb9:
501
502
  extends: .validate-linux
  stage: full-build
503
  image: "registry.gitlab.haskell.org/ghc/ci-images/i386-linux-deb9:$DOCKER_REV"
504
505
  variables:
    TEST_ENV: "i386-linux-deb9"
506
    BIN_DIST_PREP_TAR_COMP: "ghc-i386-deb9-linux.tar.xz"
507
508
509
510
511
  cache:
    key: linux-i386-deb9

validate-i386-linux-deb9:
  extends: .build-i386-linux-deb9
512
513
514
  artifacts:
    when: always
    expire_in: 2 week
515
516

nightly-i386-linux-deb9:
517
  extends: .build-i386-linux-deb9
518
519
520
521
  variables:
    TEST_TYPE: slowtest
  artifacts:
    when: always
522
    expire_in: 2 week
523
524
525
  only:
    variables:
      - $NIGHTLY
526

527
528
529
#################################
# x86_64-linux-deb9
#################################
Ben Gamari's avatar
Ben Gamari committed
530

531
.build-x86_64-linux-deb9:
Ben Gamari's avatar
Ben Gamari committed
532
  extends: .validate-linux
533
  stage: full-build
534
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
535
536
  variables:
    TEST_ENV: "x86_64-linux-deb9"
537
    BIN_DIST_PREP_TAR_COMP: "./ghc-x86_64-deb9-linux.tar.xz"
Ben Gamari's avatar
Ben Gamari committed
538
539
540
  cache:
    key: linux-x86_64-deb9

541
542
# Disabled to reduce CI load
.validate-x86_64-linux-deb9:
543
  extends: .build-x86_64-linux-deb9
544
545
546
547
  artifacts:
    when: always
    expire_in: 2 week

548
nightly-x86_64-linux-deb9:
549
  extends: .build-x86_64-linux-deb9
550
  artifacts:
551
    expire_in: 2 year
552
553
554
555
556
557
  variables:
    TEST_TYPE: slowtest
  only:
    variables:
      - $NIGHTLY

558
559
560
# N.B. Has DEBUG assertions enabled in stage2
validate-x86_64-linux-deb9-debug:
  extends: .build-x86_64-linux-deb9
561
  stage: build
562
563
  variables:
    BUILD_FLAVOUR: validate
564
565
    # Ensure that stage2 also has DEBUG enabled
    ValidateSpeed: SLOW
Ben Gamari's avatar
Ben Gamari committed
566
567
    # Override validate flavour default; see #16890.
    BUILD_SPHINX_PDF: "YES"
568
    TEST_TYPE: slowtest
569
    TEST_ENV: "x86_64-linux-deb9-debug"
570
    BIN_DIST_PREP_COMP: "ghc-x86_64-deb9-linux-debug.tar.xz"
571
572
573
  artifacts:
    when: always
    expire_in: 2 week
574

575
576
# Disabled to alleviate CI load
.validate-x86_64-linux-deb9-llvm:
577
  extends: .build-x86_64-linux-deb9
578
  stage: full-build
Ben Gamari's avatar
Ben Gamari committed
579
580
  variables:
    BUILD_FLAVOUR: perf-llvm
581
    TEST_ENV: "x86_64-linux-deb9-llvm"
582

583
584
585
586
587
588
589
590
591
592
nightly-x86_64-linux-deb9-llvm:
  extends: .build-x86_64-linux-deb9
  stage: full-build
  variables:
    BUILD_FLAVOUR: perf-llvm
    TEST_ENV: "x86_64-linux-deb9-llvm"
  only:
    variables:
      - $NIGHTLY

593
594
595
596
597
598
validate-x86_64-linux-deb9-integer-simple:
  extends: .build-x86_64-linux-deb9
  stage: full-build
  variables:
    INTEGER_LIBRARY: integer-simple
    TEST_ENV: "x86_64-linux-deb9-integer-simple"
599
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-deb9-linux-integer-simple.tar.xz"
600
601
602
603
604
605
606
607
608
609
610
611
612
613

nightly-x86_64-linux-deb9-integer-simple:
  extends: .build-x86_64-linux-deb9
  stage: full-build
  variables:
    INTEGER_LIBRARY: integer-simple
    TEST_ENV: "x86_64-linux-deb9-integer-simple"
    TEST_TYPE: slowtest
  artifacts:
    expire_in: 2 year
  only:
    variables:
      - $NIGHTLY

614
615
616
617
618
619
620
release-x86_64-linux-deb9-dwarf:
  extends: .validate-linux
  stage: build
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
  variables:
    CONFIGURE_ARGS: "--enable-dwarf-unwind"
    BUILD_FLAVOUR: dwarf
621
    TEST_ENV: "x86_64-linux-deb9-dwarf"
622
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-deb9-linux-dwarf.tar.xz"
623
624
625
626
627
628
  artifacts:
    when: always
    expire_in: 2 week
  cache:
    key: linux-x86_64-deb9

629
630
631
632

#################################
# x86_64-linux-deb8
#################################
Ben Gamari's avatar
Ben Gamari committed
633

634
release-x86_64-linux-deb8:
Ben Gamari's avatar
Ben Gamari committed
635
  extends: .validate-linux
636
  stage: full-build
637
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
638
639
  variables:
    TEST_ENV: "x86_64-linux-deb8"
640
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-deb8-linux.tar.xz"
641
642
  only:
    - tags
Ben Gamari's avatar
Ben Gamari committed
643
644
  cache:
    key: linux-x86_64-deb8
645
646
647
  artifacts:
    when: always
    expire_in: 2 week
Ben Gamari's avatar
Ben Gamari committed
648

649
650
651
652
#################################
# x86_64-linux-alpine
#################################

653
.build-x86_64-linux-alpine:
654
655
656
657
658
659
660
661
  extends: .validate-linux
  stage: full-build
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-alpine:$DOCKER_REV"
  # There are currently a few failing tests
  allow_failure: true
  variables:
    BUILD_SPHINX_PDF: "NO"
    TEST_ENV: "x86_64-linux-alpine"
662
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-alpine-linux.tar.xz"
663
664
665
666
667
668
669
670
    # Can't use ld.gold due to #13958.
    CONFIGURE_ARGS: "--disable-ld-override"
  cache:
    key: linux-x86_64-alpine
  artifacts:
    when: always
    expire_in: 2 week

671
672
673
674
675
676
677
678
679
680
681
release-x86_64-linux-alpine:
  extends: .build-x86_64-linux-alpine
  only:
    - tags

nightly-x86_64-linux-alpine:
  extends: .build-x86_64-linux-alpine
  only:
    variables:
      - $NIGHTLY

682
683
684
685
686
687
688
689
690
691
692
693
694
#################################
# x86_64-linux-centos7
#################################

release-x86_64-linux-centos7:
  extends: .validate-linux
  stage: full-build
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-centos7:$DOCKER_REV"
  variables:
    # The sphinx release shipped with Centos 7 fails to build out documentation
    BUILD_SPHINX_HTML: "NO"
    BUILD_SPHINX_PDF: "NO"
    TEST_ENV: "x86_64-linux-centos7"
695
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-centos7-linux.tar.xz"
696
697
698
699
700
701
702
  only:
    - tags
  cache:
    key: linux-x86_64-centos7
  artifacts:
    when: always
    expire_in: 2 week
703
704
705
706
707

#################################
# x86_64-linux-fedora27
#################################

Ben Gamari's avatar
Ben Gamari committed
708
validate-x86_64-linux-fedora27:
Ben Gamari's avatar
Ben Gamari committed
709
  extends: .validate-linux
710
  stage: full-build
711
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-fedora27:$DOCKER_REV"
712
713
  variables:
    TEST_ENV: "x86_64-linux-fedora27"
714
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-fedora27-linux.tar.xz"
Ben Gamari's avatar
Ben Gamari committed
715
716
  cache:
    key: linux-x86_64-fedora27
717
718
  artifacts:
    when: always
719
    expire_in: 2 week
Ben Gamari's avatar
Ben Gamari committed
720

Ben Gamari's avatar
Ben Gamari committed
721
############################################################
722
# Validation via Pipelines (Windows)
Ben Gamari's avatar
Ben Gamari committed
723
724
############################################################

725
.build-windows:
726
  <<: *only-default
Ben Gamari's avatar
Ben Gamari committed
727
  before_script:
728
    - git clean -xdf
729
    - git submodule foreach git clean -xdf
730
731
732
733
734
735
736
737
738

    # Use a local temporary directory to ensure that concurrent builds don't
    # interfere with one another
    - |
      mkdir tmp
      set TMP=%cd%\tmp
      set TEMP=%cd%\tmp

    - set PATH=C:\msys64\usr\bin;%PATH%
Ben Gamari's avatar
Ben Gamari committed
739
740
741
    - git submodule sync --recursive
    - git submodule update --init --recursive
    - git checkout .gitmodules
742
    - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
Ben Gamari's avatar
Ben Gamari committed
743
    - bash .gitlab/win32-init.sh
744
745
  after_script:
    - rd /s /q tmp
746
    - robocopy /np /nfl /ndl /e "%APPDATA%\cabal" cabal-cache
747
    - bash -c 'make clean || true'
748
  dependencies: []
Tamar Christina's avatar
Tamar Christina committed
749
750
  variables:
    FORCE_SYMLINKS: 1
751
    LANG: "en_US.UTF-8"
752
    SPHINXBUILD: "/mingw64/bin/sphinx-build.exe"
Ben Gamari's avatar
Ben Gamari committed
753
754
755
  cache:
    paths:
      - cabal-cache
756
      - ghc-8.6.5
Ben Gamari's avatar
Ben Gamari committed
757
758
      - ghc-tarballs

759
760
.build-windows-hadrian:
  extends: .build-windows
761
  stage: full-build
Ben Gamari's avatar
Ben Gamari committed
762
  variables:
763
    GHC_VERSION: "8.6.5"
764
    FLAVOUR: "validate"
765
766
  # due to #16574 this currently fails
  allow_failure: true
Ben Gamari's avatar
Ben Gamari committed
767
768
769
  script:
    - |
      python boot
770
      bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
771
    - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh --flavour=$FLAVOUR -j`mk/detect-cpu-count.sh` --flavour=Quick --docs=no-sphinx binary-dist"
772
    - mv _build/bindist/ghc*.tar.xz ghc.tar.xz
773
774
    - bash -c "export TOP=$(pwd); cd _build/bindist/ghc-*/ && PATH=$TOP/toolchain/bin:$PATH ./configure --prefix=$TOP/_build/install && make install && cd ../../../"
    - bash -c "export TOP=$(pwd); PATH=$TOP/toolchain/bin:$PATH hadrian/build.cabal.sh --flavour=$FLAVOUR -j`mk/detect-cpu-count.sh` --flavour=quick test --summary-junit=./junit.xml --skip-perf --test-compiler=$TOP/_build/install/bin/ghc"
775
776
    # skipping perf tests for now since we build a quick-flavoured GHC,
    # which might result in some broken perf tests?
Ben Gamari's avatar
Ben Gamari committed
777
778
  tags:
    - x86_64-windows
779
  artifacts:
780
781
782
    reports:
      junit: junit.xml
    expire_in: 2 week
783
784
785
    when: always
    paths:
      - ghc.tar.xz
786
      - junit.xml
Ben Gamari's avatar
Ben Gamari committed
787

788
validate-x86_64-windows-hadrian:
789
790
791
  extends: .build-windows-hadrian
  variables:
    MSYSTEM: MINGW64
792
    TEST_ENV: "x86_64-windows-hadrian"
793
  cache:
794
    key: "x86_64-windows-hadrian-$WINDOWS_TOOLCHAIN_VERSION"
795
796
797
798
799

nightly-i386-windows-hadrian:
  extends: .build-windows-hadrian
  variables:
    MSYSTEM: MINGW32
800
    TEST_ENV: "i386-windows-hadrian"
801
802
803
804
  only:
    variables:
      - $NIGHTLY
  cache:
805
    key: "i386-windows-hadrian-$WINDOWS_TOOLCHAIN_VERSION"
806
807
808

.build-windows-make:
  extends: .build-windows
809
  stage: full-build
810
  allow_failure: true
811
  variables:
812
    BUILD_FLAVOUR: "quick"
813
    GHC_VERSION: "8.6.5"
814
    BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-mingw32.tar.xz"
815
816
817
  script:
    - |
      python boot
818
      bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex $CONFIGURE_ARGS'
819
    - bash -c "echo include mk/flavours/${BUILD_FLAVOUR}.mk > mk/build.mk"
820
    - bash -c "echo 'GhcLibHcOpts+=-haddock' >> mk/build.mk"
821
    - bash -c "PATH=`pwd`/toolchain/bin:$PATH make -j`mk/detect-cpu-count.sh`"
822
823
    - bash -c "PATH=`pwd`/toolchain/bin:$PATH make binary-dist-prep TAR_COMP_OPTS=-1"
    - bash -c "PATH=`pwd`/toolchain/bin:$PATH make test_bindist TEST_PREP=YES"
824
    - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
825
826
  tags:
    - x86_64-windows
827
828
  artifacts:
    when: always
829
    expire_in: 2 week
830
831
832
    reports:
      junit: junit.xml
    paths:
833
      - $BIN_DIST_PREP_TAR_COMP
834
      - junit.xml
Ben Gamari's avatar
Ben Gamari committed
835

836
837
838
839
validate-x86_64-windows:
  extends: .build-windows-make
  variables:
    MSYSTEM: MINGW64
840
    CONFIGURE_ARGS: "--target=x86_64-unknown-mingw32"
841
    TEST_ENV: "x86_64-windows"
842
  cache:
843
    key: "x86_64-windows-$WINDOWS_TOOLCHAIN_VERSION"
844

845
846
847
848
849
850
851
852
853
854
855
856
857
nightly-x86_64-windows:
  extends: .build-windows-make
  stage: full-build
  variables:
    BUILD_FLAVOUR: "validate"
    MSYSTEM: MINGW64
    CONFIGURE_ARGS: "--target=x86_64-unknown-mingw32"
  only:
    variables:
      - $NIGHTLY
  cache:
    key: "x86_64-windows-$WINDOWS_TOOLCHAIN_VERSION"

858
859
860
861
862
863
# Normal Windows validate builds are profiled; that won't do for releases.
release-x86_64-windows:
  extends: validate-x86_64-windows
  variables:
    MSYSTEM: MINGW64
    BUILD_FLAVOUR: "perf"
864
    CONFIGURE_ARGS: "--target=x86_64-unknown-mingw32"
865
    TEST_ENV: "x86_64-windows"
866
867
868
  only:
    - tags

869
870
871
872
873
874
release-i386-windows:
  extends: .build-windows-make
  only:
    - tags
  variables:
    MSYSTEM: MINGW32
875
    BUILD_FLAVOUR: "perf"
876
    CONFIGURE_ARGS: "--target=i386-unknown-mingw32"
877
878
    # Due to #15934
    BUILD_PROF_LIBS: "NO"
879
    TEST_ENV: "i386-windows"
880
  cache:
881
    key: "i386-windows-$WINDOWS_TOOLCHAIN_VERSION"
882
883
884
885
886
887
888
889

nightly-i386-windows:
  extends: .build-windows-make
  only:
    variables:
      - $NIGHTLY
  variables:
    MSYSTEM: MINGW32
890
    CONFIGURE_ARGS: "--target=i386-unknown-mingw32"
891
892
    # Due to #15934
    BUILD_PROF_LIBS: "NO"
893
    TEST_ENV: "i386-windows"
894
  cache:
895
    key: "i386-windows-$WINDOWS_TOOLCHAIN_VERSION"
896

897
898
899
900
############################################################
# Cleanup
############################################################

901
902
# Note [Cleaning up after shell executor]
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Ben Gamari's avatar
Ben Gamari committed
903
904
905
#
# As noted in [1], gitlab-runner's shell executor doesn't clean up its working
# directory after builds. Unfortunately, we are forced to use the shell executor
906
907
908
909
910
911
912
913
914
# on Darwin. To avoid running out of disk space we add a stage at the end of
# the build to remove the /.../GitLabRunner/builds directory. Since we only run a
# single build at a time on Darwin this should be safe.
#
# We used to have a similar cleanup job on Windows as well however it ended up
# being quite fragile as we have multiple Windows builders yet there is no
# guarantee that the cleanup job is run on the same machine as the build itself
# was run. Consequently we were forced to instead handle cleanup with a separate
# cleanup cron job on Windows.
Ben Gamari's avatar
Ben Gamari committed
915
916
917
#
# [1] https://gitlab.com/gitlab-org/gitlab-runner/issues/3856

918
919
# See Note [Cleanup after shell executor]
cleanup-darwin:
920
  <<: *only-default
921
922
923
924
  stage: cleanup
  tags:
    - x86_64-darwin
  when: always
925
  dependencies: []
926
927
928
929
930
931
932
933
934
935
936
  before_script:
    - echo "Time to clean up"
  script:
    - echo "Let's go"
  after_script:
    - BUILD_DIR=$CI_PROJECT_DIR
    - echo "Cleaning $BUILD_DIR"
    - cd $HOME
    - rm -Rf $BUILD_DIR/*
    - exit 0

937
938
939
940
941
############################################################
# Packaging
############################################################

doc-tarball:
942
  <<: *only-default
943
944
945
  stage: packaging
  tags:
    - x86_64-linux
Ben Gamari's avatar
Ben Gamari committed
946
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
947
  dependencies:
948
    - validate-x86_64-linux-deb9-debug
949
    - validate-x86_64-windows
950
951
952
  variables:
    LINUX_BINDIST: "ghc-x86_64-deb9-linux.tar.xz"
    WINDOWS_BINDIST: "ghc-x86_64-mingw32.tar.xz"
953
954
  # Due to Windows allow_failure
  allow_failure: true
955
956
957
958
959
960
961
962
  artifacts:
    paths:
      - haddock.html.tar.xz
      - libraries.html.tar.xz
      - users_guide.html.tar.xz
      - index.html
      - "*.pdf"
  script:
963
964
965
966
967
968
969
970
971
    - |
      if [ ! -f "$LINUX_BINDIST" ]; then
        echo "Error: $LINUX_BINDIST does not exist. Did the Debian 9 job fail?"
        exit 1
      fi
      if [ ! -f "$WINDOWS_BINDIST" ]; then
        echo "Error: $WINDOWS_BINDIST does not exist. Did the 64-bit Windows job fail?"
        exit 1
      fi
972
    - rm -Rf docs
973
    - bash -ex distrib/mkDocs/mkDocs $LINUX_BINDIST $WINDOWS_BINDIST
974
    - ls -lh
975
    - mv docs/*.tar.xz docs/index.html .
976

977
978
979
980
source-tarball:
  stage: packaging
  tags:
    - x86_64-linux
Ben Gamari's avatar
Ben Gamari committed
981
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
982
  dependencies: []
983
984
985
986
987
988
989
990
991
992
993
994
  only:
    - tags
  artifacts:
    paths:
      - ghc-*.tar.xz
      - version
  script:
    - mk/get-win32-tarballs.sh download all
    - ./boot
    - ./configure
    - make sdist
    - mv sdistprep/*.xz  .
Ben Gamari's avatar
Ben Gamari committed
995
996
997
    - make show! --quiet VALUE=ProjectVersion > version
    - source version
    - echo "$ProjectVersion" > version
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010


############################################################
# Testing via head.hackage
############################################################

# Triggering jobs in the ghc/head.hackage project requires that we have a job
# token for that repository. Furthermore the head.hackage CI job must have
# access to an unprivileged access token with the ability to query the ghc/ghc
# project such that it can find the job ID of the fedora27 job for the current
# pipeline.

.hackage:
1011
  <<: *only-default
1012
  stage: testing
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
  image: ghcci/x86_64-linux-deb9:0.2
  tags:
    - x86_64-linux
  dependencies: []
  variables:
    HEAD_HACKAGE_PROJECT_ID: "78"
  script:
    - bash .gitlab/start-head.hackage.sh

hackage:
  extends: .hackage
  when: manual

hackage-label:
  extends: .hackage
  only:
    variables:
      - $CI_MERGE_REQUEST_LABELS =~ /.*user-facing.*/

nightly-hackage:
  extends: .hackage
  only:
    variables:
      - $NIGHTLY
1037

1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
############################################################
# Nofib testing
############################################################

perf-nofib:
  stage: testing
  dependencies:
    - release-x86_64-linux-deb9-dwarf
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
  only:
    refs:
      - merge_requests
      - master
      - /ghc-[0-9]+\.[0-9]+/
  tags:
    - x86_64-linux
  script:
    - root=$(pwd)/ghc
    - |
      mkdir tmp
1058
      tar -xf ghc-x86_64-deb9-linux-dwarf.tar.xz -C tmp
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
      pushd tmp/ghc-*/
      ./configure --prefix=$root
      make install
      popd
      rm -Rf tmp
    - export BOOT_HC=$(which ghc)
    - cabal update; cabal install -w $BOOT_HC regex-compat
    - export PATH=$root/bin:$PATH
    - make -C nofib boot mode=fast -j$CPUS
    - "make -C nofib EXTRA_RUNTEST_OPTS='-cachegrind +RTS -V0 -RTS' NoFibRuns=1 mode=fast -j$CPUS 2>&1 | tee nofib.log"
  artifacts:
    expire_in: 12 week
    when: always
    paths:
      - nofib.log

############################################################
# Documentation deployment via GitLab Pages
############################################################

1079
1080
1081
1082
1083
pages:
  stage: deploy
  dependencies:
    - doc-tarball
  image: ghcci/x86_64-linux-deb9:0.2
1084
1085
  # Due to Windows allow_failure
  allow_failure: true
1086
1087
1088
1089
1090
1091
1092
  tags:
    - x86_64-linux
  script:
    - mkdir -p public/doc
    - tar -xf haddock.html.tar.xz -C public/doc
    - tar -xf libraries.html.tar.xz -C public/doc
    - tar -xf users_guide.html.tar.xz -C public/doc
1093
1094
1095
1096
1097
1098
    - |
      cat >public/index.html <<EOF
      <!DOCTYPE HTML>
      <meta charset="UTF-8">
      <meta http-equiv="refresh" content="1; url=doc/">
      EOF
1099
1100
1101
1102
1103
1104
    - cp -f index.html public/doc
  only:
    - master
  artifacts:
    paths:
      - public