.gitlab-ci.yml 14.3 KB
Newer Older
1
2
variables:
  GIT_SSL_NO_VERIFY: "1"
3

4
  # Commit of ghc/ci-images repository from which to pull Docker images
5
  DOCKER_REV: 6d19c3adc1f5c28c82aed8c5b1ac40931ac60f3f
6
7
8
9
10
11

before_script:
  - python3 .gitlab/fix-submodules.py
  - git submodule sync --recursive
  - git submodule update --init --recursive
  - git checkout .gitmodules
12
  - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
13

Ben Gamari's avatar
Ben Gamari committed
14
15
16
stages:
  - lint
  - build
17
  - full-build
Ben Gamari's avatar
Ben Gamari committed
18
  - cleanup # See Note [Cleanup on Windows]
Ben Gamari's avatar
Ben Gamari committed
19

Ben Gamari's avatar
Ben Gamari committed
20
############################################################
21
# Runner Tags
Ben Gamari's avatar
Ben Gamari committed
22
23
24
25
26
27
28
29
30
31
############################################################
#
# * x86_64-linux: Any Docker-capable x86_64 Linux machine
# * aarch64-linux: Any Docker-capable AArch64 Linux machine
# * x86_64-windows: A x86_64 Windows machine
# * lint: Any Docker-capable x86_64 Linux machine; distinct from
#         x86_64-linux to ensure low-latency availability.
#


Ben Gamari's avatar
Ben Gamari committed
32
33
34
35
36
37
############################################################
# Linting
############################################################

ghc-linters:
  stage: lint
38
  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
Ben Gamari's avatar
Ben Gamari committed
39
40
41
42
43
44
45
46
47
48
  script:
    - |
      if [ -n "$CI_MERGE_REQUEST_ID" ]; then
        base="$(git merge-base $CI_MERGE_REQUEST_BRANCH_NAME HEAD)"
        validate-commit-msg .git $(git rev-list $base..$CI_COMMIT_SHA)
        submodchecker .git $(git rev-list $base..$CI_COMMIT_SHA)
        validate-whitespace .git $(git rev-list $base..$CI_COMMIT_SHA)
        .gitlab/linters/check-makefiles.py $base $CI_COMMIT_SHA
        .gitlab/linters/check-cpp.py $base $CI_COMMIT_SHA
      fi
49
  dependencies: []
Ben Gamari's avatar
Ben Gamari committed
50
51
  tags:
    - lint
Ben Gamari's avatar
Ben Gamari committed
52

Ben Gamari's avatar
Ben Gamari committed
53
54
55
############################################################
# Validation via Pipelines (hadrian)
############################################################
56

Ben Gamari's avatar
Ben Gamari committed
57
.validate-hadrian:
Ben Gamari's avatar
Ben Gamari committed
58
  allow_failure: true
Ben Gamari's avatar
Ben Gamari committed
59
  script:
60
    - cabal update
61
    - git clean -xdf && git submodule foreach git clean -xdf
Ben Gamari's avatar
Ben Gamari committed
62
63
64
    - bash .circleci/prepare-system.sh
    - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
    - ./boot
Ben Gamari's avatar
Ben Gamari committed
65
    - ./configure $CONFIGURE_ARGS
Ben Gamari's avatar
Ben Gamari committed
66
    - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh`
67
    - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --docs=no-sphinx docs
Ben Gamari's avatar
Ben Gamari committed
68
69
  cache:
    key: hadrian
70
    paths:
Ben Gamari's avatar
Ben Gamari committed
71
      - cabal-cache
72

Ben Gamari's avatar
Ben Gamari committed
73
validate-x86_64-linux-deb8-hadrian:
Ben Gamari's avatar
Ben Gamari committed
74
  extends: .validate-hadrian
75
  stage: build
76
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
Ben Gamari's avatar
Ben Gamari committed
77
78
79
  before_script:
    # workaround for docker permissions
    - sudo chown ghc:ghc -R .
80
81
82
83
    - python3 .gitlab/fix-submodules.py
    - git submodule sync --recursive
    - git submodule update --init --recursive
    - git checkout .gitmodules
84
    - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
Ben Gamari's avatar
Ben Gamari committed
85
86
87
  tags:
    - x86_64-linux

88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
hadrian-ghc-in-ghci:
  stage: build
  image: ghcci/x86_64-linux-deb8:0.1
  before_script:
    # workaround for docker permissions
    - sudo chown ghc:ghc -R .
    - python3 .gitlab/fix-submodules.py
    - git submodule sync --recursive
    - git submodule update --init --recursive
    - git checkout .gitmodules
  tags:
    - x86_64-linux
  script:
    - cabal update
    - git clean -xdf && git submodule foreach git clean -xdf
    - bash .circleci/prepare-system.sh
    - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
    - ./boot
    - ./configure $CONFIGURE_ARGS
    # Load ghc-in-ghci then immediately exit
    - echo ":q" | hadrian/ghci.sh
  cache:
    key: hadrian-ghci
    paths:
      - cabal-cache

Ben Gamari's avatar
Ben Gamari committed
114
115
116
117
118
############################################################
# Validation via Pipelines (make)
############################################################

.validate:
119
120
  variables:
    TEST_TYPE: test
Ben Gamari's avatar
Ben Gamari committed
121
  before_script:
122
    - git clean -xdf && git submodule foreach git clean -xdf
Ben Gamari's avatar
Ben Gamari committed
123
  script:
Ben Gamari's avatar
Ben Gamari committed
124
    - ./boot
Ben Gamari's avatar
Ben Gamari committed
125
    - ./configure $CONFIGURE_ARGS
Ben Gamari's avatar
Ben Gamari committed
126
127
    - |
      THREADS=`mk/detect-cpu-count.sh`
Matthew Pickering's avatar
Matthew Pickering committed
128
      make V=0 -j$THREADS WERROR=-Werror
Ben Gamari's avatar
Ben Gamari committed
129
130
131
    - |
      make binary-dist TAR_COMP_OPTS="-1"
      mv ghc-*.tar.xz ghc.tar.xz
132
133
134
135
136
    - |
      # Prepare to push git notes.
      METRICS_FILE=$(mktemp)
      git config user.email "ben+ghc-ci@smart-cactus.org"
      git config user.name "GHC GitLab CI"
Ben Gamari's avatar
Ben Gamari committed
137
138
    - |
      THREADS=`mk/detect-cpu-count.sh`
139
140
141
142
      make $TEST_TYPE THREADS=$THREADS JUNIT_FILE=../../junit.xml METRICS_FILE=$METRICS_FILE
    - |
      # Push git notes.
      METRICS_FILE=$METRICS_FILE .gitlab/push-test-metrics.sh
143
  dependencies: []
144
145
146
  artifacts:
    reports:
      junit: junit.xml
147
    expire_in: 2 week
148
149
150
151
    paths:
      - ghc.tar.xz
      - junit.xml

152
153
154
155
#################################
# x86_64-darwin
#################################

Ben Gamari's avatar
Ben Gamari committed
156
validate-x86_64-darwin:
Ben Gamari's avatar
Ben Gamari committed
157
  extends: .validate
158
  stage: full-build
Ben Gamari's avatar
Ben Gamari committed
159
160
161
  tags:
    - x86_64-darwin
  variables:
162
    GHC_VERSION: 8.6.3
Ben Gamari's avatar
Ben Gamari committed
163
164
165
    MACOSX_DEPLOYMENT_TARGET: "10.7"
    # Only Sierra and onwards supports clock_gettime. See #12858
    ac_cv_func_clock_gettime: "no"
166
    LANG: "en_US.UTF-8"
167
    CONFIGURE_ARGS: --with-intree-gmp
168
    TEST_ENV: "x86_64-darwin"
Ben Gamari's avatar
Ben Gamari committed
169
  before_script:
Ben Gamari's avatar
Ben Gamari committed
170
    - git clean -xdf && git submodule foreach git clean -xdf
171
    - python3 .gitlab/fix-submodules.py
172
173
174
    - git submodule sync --recursive
    - git submodule update --init --recursive
    - git checkout .gitmodules
175
    - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
176

Ben Gamari's avatar
Ben Gamari committed
177
178
179
180
    - bash .gitlab/darwin-init.sh
    - PATH="`pwd`/toolchain/bin:$PATH"
  after_script:
    - cp -Rf $HOME/.cabal cabal-cache
181
182
  artifacts:
    when: always
183
    expire_in: 2 week
Ben Gamari's avatar
Ben Gamari committed
184
185
186
187
188
189
  cache:
    key: darwin
    paths:
      - cabal-cache
      - toolchain

Ben Gamari's avatar
Ben Gamari committed
190
.validate-linux:
Ben Gamari's avatar
Ben Gamari committed
191
192
193
194
  extends: .validate
  tags:
    - x86_64-linux
  before_script:
Ben Gamari's avatar
Ben Gamari committed
195
    - git clean -xdf && git submodule foreach git clean -xdf
196
197
198
199
    - python3 .gitlab/fix-submodules.py
    - git submodule sync --recursive
    - git submodule update --init --recursive
    - git checkout .gitmodules
200
    - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
201

Ben Gamari's avatar
Ben Gamari committed
202
203
204
205
206
207
    - bash .circleci/prepare-system.sh
    # workaround for docker permissions
    - sudo chown ghc:ghc -R .
  after_script:
    - cp -Rf $HOME/.cabal cabal-cache
  cache:
Ben Gamari's avatar
Ben Gamari committed
208
    key: linux
Ben Gamari's avatar
Ben Gamari committed
209
210
211
212
    paths:
      - cabal-cache
      - toolchain

213
214
215
216
217
#################################
# aarch64-linux-deb9
#################################

.build-aarch64-linux-deb9:
Ben Gamari's avatar
Ben Gamari committed
218
  extends: .validate-linux
219
  stage: full-build
220
  image: "registry.gitlab.haskell.org/ghc/ci-images/aarch64-linux-deb9:$DOCKER_REV"
Ben Gamari's avatar
Ben Gamari committed
221
  allow_failure: true
222
223
  variables:
    TEST_ENV: "aarch64-linux-deb9"
Ben Gamari's avatar
Ben Gamari committed
224
225
226
227
228
  cache:
    key: linux-aarch64-deb9
  tags:
    - aarch64-linux

229
230
231
232
233
234
validate-aarch64-linux-deb9:
  extends: .build-aarch64-linux-deb9
  artifacts:
    when: always
    expire_in: 2 week

235
nightly-aarch64-linux-deb9:
236
  extends: .build-aarch64-linux-deb9
237
  artifacts:
238
    expire_in: 2 year
239
240
241
242
243
244
  variables:
    TEST_TYPE: slowtest
  only:
    variables:
      - $NIGHTLY

245
246
247
248
249
#################################
# i386-linux-deb9
#################################

.build-i386-linux-deb9:
250
251
  extends: .validate-linux
  stage: full-build
252
  image: "registry.gitlab.haskell.org/ghc/ci-images/i386-linux-deb9:$DOCKER_REV"
253
254
  variables:
    TEST_ENV: "i386-linux-deb9"
255
256
257
258
259
  cache:
    key: linux-i386-deb9

validate-i386-linux-deb9:
  extends: .build-i386-linux-deb9
260
261
262
  artifacts:
    when: always
    expire_in: 2 week
263
264

nightly-i386-linux-deb9:
265
  extends: .build-i386-linux-deb9
266
267
268
269
  variables:
    TEST_TYPE: slowtest
  artifacts:
    when: always
270
    expire_in: 2 week
271
272
273
  only:
    variables:
      - $NIGHTLY
274

275
276
277
#################################
# x86_64-linux-deb9
#################################
Ben Gamari's avatar
Ben Gamari committed
278

279
.build-x86_64-linux-deb9:
Ben Gamari's avatar
Ben Gamari committed
280
  extends: .validate-linux
281
  stage: build
282
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
283
284
  variables:
    TEST_ENV: "x86_64-linux-deb9"
Ben Gamari's avatar
Ben Gamari committed
285
286
287
  cache:
    key: linux-x86_64-deb9

288
289
validate-x86_64-linux-deb9:
  extends: .build-x86_64-linux-deb9
290
291
292
293
  artifacts:
    when: always
    expire_in: 2 week

294
nightly-x86_64-linux-deb9:
295
  extends: .build-x86_64-linux-deb9
296
  artifacts:
297
    expire_in: 2 year
298
299
300
301
302
303
  variables:
    TEST_TYPE: slowtest
  only:
    variables:
      - $NIGHTLY

304
305
306
307
308
309
310
# N.B. Has DEBUG assertions enabled in stage2
validate-x86_64-linux-deb9-debug:
  extends: .build-x86_64-linux-deb9
  variables:
    BUILD_FLAVOUR: validate
    TEST_ENV: "x86_64-linux-deb9-debug"

Ben Gamari's avatar
Ben Gamari committed
311
validate-x86_64-linux-deb9-llvm:
312
  extends: .build-x86_64-linux-deb9
313
  stage: full-build
Ben Gamari's avatar
Ben Gamari committed
314
315
  variables:
    BUILD_FLAVOUR: perf-llvm
316
    TEST_ENV: "x86_64-linux-deb9-llvm"
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348

validate-x86_64-linux-deb9-integer-simple:
  extends: .build-x86_64-linux-deb9
  stage: full-build
  variables:
    INTEGER_LIBRARY: integer-simple
    TEST_ENV: "x86_64-linux-deb9-integer-simple"

nightly-x86_64-linux-deb9-integer-simple:
  extends: .build-x86_64-linux-deb9
  stage: full-build
  variables:
    INTEGER_LIBRARY: integer-simple
    TEST_ENV: "x86_64-linux-deb9-integer-simple"
    TEST_TYPE: slowtest
  artifacts:
    expire_in: 2 year
  only:
    variables:
      - $NIGHTLY

validate-x86_64-linux-deb9-unreg:
  extends: .build-x86_64-linux-deb9
  stage: full-build
  variables:
    CONFIGURE_ARGS: --enable-unregisterised
    TEST_ENV: "x86_64-linux-deb9-unreg"


#################################
# x86_64-linux-deb8
#################################
Ben Gamari's avatar
Ben Gamari committed
349

350
release-x86_64-linux-deb8:
Ben Gamari's avatar
Ben Gamari committed
351
  extends: .validate-linux
352
  stage: full-build
353
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
354
355
  variables:
    TEST_ENV: "x86_64-linux-deb8"
356
357
  only:
    - tags
Ben Gamari's avatar
Ben Gamari committed
358
359
  cache:
    key: linux-x86_64-deb8
360
361
362
  artifacts:
    when: always
    expire_in: 2 week
Ben Gamari's avatar
Ben Gamari committed
363

364
365
366
367
368

#################################
# x86_64-linux-fedora27
#################################

369
release-x86_64-linux-fedora27:
Ben Gamari's avatar
Ben Gamari committed
370
  extends: .validate-linux
371
  stage: full-build
372
  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-fedora27:$DOCKER_REV"
373
374
  variables:
    TEST_ENV: "x86_64-linux-fedora27"
375
376
  only:
    - tags
Ben Gamari's avatar
Ben Gamari committed
377
378
  cache:
    key: linux-x86_64-fedora27
379
380
  artifacts:
    when: always
381
    expire_in: 2 week
Ben Gamari's avatar
Ben Gamari committed
382

Ben Gamari's avatar
Ben Gamari committed
383
############################################################
384
# Validation via Pipelines (Windows)
Ben Gamari's avatar
Ben Gamari committed
385
386
############################################################

Ben Gamari's avatar
Ben Gamari committed
387
388
.validate-windows:
  before_script:
389
    - git clean -xdf
390
    - git submodule foreach git clean -xdf
391
392
393
394
395
396
397
398
399

    # Use a local temporary directory to ensure that concurrent builds don't
    # interfere with one another
    - |
      mkdir tmp
      set TMP=%cd%\tmp
      set TEMP=%cd%\tmp

    - set PATH=C:\msys64\usr\bin;%PATH%
Ben Gamari's avatar
Ben Gamari committed
400
401
402
403
    - python .gitlab/fix-submodules.py
    - git submodule sync --recursive
    - git submodule update --init --recursive
    - git checkout .gitmodules
404
    - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
Ben Gamari's avatar
Ben Gamari committed
405
    - bash .gitlab/win32-init.sh
406
407
  after_script:
    - rd /s /q tmp
408
    - robocopy /np /nfl /ndl /e "%APPDATA%\cabal" cabal-cache
409
    - bash -c 'make clean || true'
410
  dependencies: []
Tamar Christina's avatar
Tamar Christina committed
411
412
  variables:
    FORCE_SYMLINKS: 1
Ben Gamari's avatar
Ben Gamari committed
413
414
415
416
417
418
  cache:
    paths:
      - cabal-cache
      - ghc-8.6.2
      - ghc-tarballs

419
validate-x86_64-windows-hadrian:
Ben Gamari's avatar
Ben Gamari committed
420
  extends: .validate-windows
421
  stage: full-build
Ben Gamari's avatar
Ben Gamari committed
422
423
  variables:
    GHC_VERSION: "8.6.2"
Ben Gamari's avatar
Ben Gamari committed
424
    LANG: "en_US.UTF-8"
Ben Gamari's avatar
Ben Gamari committed
425
426
427
428
  script:
    - |
      set MSYSTEM=MINGW64
      python boot
429
      bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
430
    - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick"
431
    - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick --docs=no-sphinx docs
432
433
434
435
    # FIXME: Bindist disabled due to #16073
    #- bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh binary-dist"
    # FIXME: Testsuite disabled due to #16156.
    #- bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
Ben Gamari's avatar
Ben Gamari committed
436
  cache:
Ben Gamari's avatar
Ben Gamari committed
437
    key: x86_64-windows
Ben Gamari's avatar
Ben Gamari committed
438
439
440
  tags:
    - x86_64-windows

441
validate-x86_64-windows:
Ben Gamari's avatar
Ben Gamari committed
442
  extends: .validate-windows
443
  stage: full-build
444
445
  # due to #16084
  allow_failure: true
446
447
  variables:
    GHC_VERSION: "8.6.2"
Ben Gamari's avatar
Ben Gamari committed
448
    LANG: "en_US.UTF-8"
449
450
451
452
  script:
    - |
      set MSYSTEM=MINGW64
      python boot
453
      bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
454
    - bash -c "echo include mk/flavours/quick.mk > mk/build.mk"
455
    - bash -c "PATH=`pwd`/toolchain/bin:$PATH make -j`mk/detect-cpu-count.sh`"
456
457
458
    - |
      bash -c "make binary-dist TAR_COMP_OPTS=-1"
      mv ghc-*.tar.xz ghc.tar.xz
459
    - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
460
  cache:
Ben Gamari's avatar
Ben Gamari committed
461
    key: x86_64-windows
462
463
  tags:
    - x86_64-windows
464
465
466
467
468
469
470
  artifacts:
    when: always
    reports:
      junit: junit.xml
    paths:
      - ghc.tar.xz
      - junit.xml
Ben Gamari's avatar
Ben Gamari committed
471

472
473
474
475
############################################################
# Cleanup
############################################################

476
477
# Note [Cleaning up after shell executor]
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Ben Gamari's avatar
Ben Gamari committed
478
479
480
481
482
483
484
485
486
#
# As noted in [1], gitlab-runner's shell executor doesn't clean up its working
# directory after builds. Unfortunately, we are forced to use the shell executor
# on Windows. To avoid running out of disk space we add a stage at the end of
# the build to remove the \GitLabRunner\builds directory. Since we only run a
# single build at a time on Windows this should be safe.
#
# [1] https://gitlab.com/gitlab-org/gitlab-runner/issues/3856

487
# See Note [Cleanup after shell executor]
Ben Gamari's avatar
Ben Gamari committed
488
489
490
491
492
cleanup-windows:
  stage: cleanup
  tags:
    - x86_64-windows
  when: always
493
  dependencies: []
Ben Gamari's avatar
Ben Gamari committed
494
495
496
497
498
499
500
501
502
  before_script:
    - echo "Time to clean up"
  script:
    - echo "Let's go"
  after_script:
    - set "BUILD_DIR=%CI_PROJECT_DIR%"
    - set "BUILD_DIR=%BUILD_DIR:/=\%"
    - echo "Cleaning %BUILD_DIR%"
    - cd \GitLabRunner
503
504
505
506
    # This is way more complicated than it should be:
    # See https://stackoverflow.com/questions/1965787
    - del %BUILD_DIR%\* /F /Q
    - for /d %%p in (%BUILD_DIR%\*) do rd /Q /S "%%p"
Ben Gamari's avatar
Ben Gamari committed
507
    - exit /b 0
508
509
510
511
512
513
514

# See Note [Cleanup after shell executor]
cleanup-darwin:
  stage: cleanup
  tags:
    - x86_64-darwin
  when: always
515
  dependencies: []
516
517
518
519
520
521
522
523
524
525
526
  before_script:
    - echo "Time to clean up"
  script:
    - echo "Let's go"
  after_script:
    - BUILD_DIR=$CI_PROJECT_DIR
    - echo "Cleaning $BUILD_DIR"
    - cd $HOME
    - rm -Rf $BUILD_DIR/*
    - exit 0