ci.sh 11.1 KB
Newer Older
Ben Gamari's avatar
Ben Gamari committed
1
#!/usr/bin/env bash
2
# shellcheck disable=SC2230
Ben Gamari's avatar
Ben Gamari committed
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43

# This is the primary driver of the GitLab CI infrastructure.

set -e -o pipefail

# Configuration:
hackage_index_state="@1579718451"

# Colors
BLACK="0;30"
GRAY="1;30"
RED="0;31"
LT_RED="1;31"
BROWN="0;33"
LT_BROWN="1;33"
GREEN="0;32"
LT_GREEN="1;32"
BLUE="0;34"
LT_BLUE="1;34"
PURPLE="0;35"
LT_PURPLE="1;35"
CYAN="0;36"
LT_CYAN="1;36"
WHITE="1;37"
LT_GRAY="0;37"

# GitLab Pipelines log section delimiters
# https://gitlab.com/gitlab-org/gitlab-foss/issues/14664
start_section() {
  name="$1"
  echo -e "section_start:$(date +%s):$name\015\033[0K"
}

end_section() {
  name="$1"
  echo -e "section_end:$(date +%s):$name\015\033[0K"
}

echo_color() {
  local color="$1"
  local msg="$2"
Ben Gamari's avatar
Ben Gamari committed
44
  echo -e "\033[${color}m${msg}\033[0m"
Ben Gamari's avatar
Ben Gamari committed
45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77
}

error() { echo_color "${RED}" "$1"; }
warn() { echo_color "${LT_BROWN}" "$1"; }
info() { echo_color "${LT_BLUE}" "$1"; }

fail() { error "error: $1"; exit 1; }

function run() {
  info "Running $*..."
  "$@" || ( error "$* failed"; return 1; )
}

TOP="$(pwd)"

function mingw_init() {
  case "$MSYSTEM" in
    MINGW32)
      triple="i386-unknown-mingw32"
      boot_triple="i386-unknown-mingw32" # triple of bootstrap GHC
      ;;
    MINGW64)
      triple="x86_64-unknown-mingw32"
      boot_triple="x86_64-unknown-mingw32" # triple of bootstrap GHC
      ;;
    *)
      fail "win32-init: Unknown MSYSTEM $MSYSTEM"
      ;;
  esac

  # Bring mingw toolchain into PATH.
  # This is extracted from /etc/profile since this script inexplicably fails to
  # run under gitlab-runner.
78
  # shellcheck disable=SC1091
Ben Gamari's avatar
Ben Gamari committed
79 80 81
  source /etc/msystem
  MINGW_MOUNT_POINT="${MINGW_PREFIX}"
  PATH="$MINGW_MOUNT_POINT/bin:$PATH"
82 83 84

  # We always use mingw64 Python to avoid path length issues like #17483.
  export PYTHON="/mingw64/bin/python3"
Ben Gamari's avatar
Ben Gamari committed
85 86 87 88
}

# This will contain GHC's local native toolchain
toolchain="$TOP/toolchain"
89
mkdir -p "$toolchain/bin"
Ben Gamari's avatar
Ben Gamari committed
90 91 92 93
PATH="$toolchain/bin:$PATH"

export METRICS_FILE="$CI_PROJECT_DIR/performance-metrics.tsv"

94 95
cores="$(mk/detect-cpu-count.sh)"

Ben Gamari's avatar
Ben Gamari committed
96 97
# Use a local temporary directory to ensure that concurrent builds don't
# interfere with one another
98
mkdir -p "$TOP/tmp"
Ben Gamari's avatar
Ben Gamari committed
99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
export TMP="$TOP/tmp"
export TEMP="$TOP/tmp"

function darwin_setup() {
  # It looks like we already have python2 here and just installing python3
  # does not work.
  brew upgrade python
  brew install ghc cabal-install ncurses gmp

  pip3 install sphinx
  # PDF documentation disabled as MacTeX apparently doesn't include xelatex.
  #brew cask install mactex
}

function show_tool() {
  local tool="$1"
  info "$tool = ${!tool}"
  ${!tool} --version
}

function set_toolchain_paths() {
  needs_toolchain=1
  case "$(uname)" in
    Linux) needs_toolchain="" ;;
    *) ;;
  esac

  if [[ -n "$needs_toolchain" ]]; then
      # These are populated by setup_toolchain
128 129 130 131
      GHC="$toolchain/bin/ghc$exe"
      CABAL="$toolchain/bin/cabal$exe"
      HAPPY="$toolchain/bin/happy$exe"
      ALEX="$toolchain/bin/alex$exe"
Ben Gamari's avatar
Ben Gamari committed
132
  else
133 134 135 136
      GHC="$(which ghc)"
      CABAL="/usr/local/bin/cabal"
      HAPPY="$HOME/.cabal/bin/happy"
      ALEX="$HOME/.cabal/bin/alex"
Ben Gamari's avatar
Ben Gamari committed
137
  fi
138 139 140 141
  export GHC
  export CABAL
  export HAPPY
  export ALEX
Ben Gamari's avatar
Ben Gamari committed
142 143 144 145 146 147 148 149 150 151 152 153

  # FIXME: Temporarily use ghc from ports
  case "$(uname)" in
    FreeBSD) GHC="/usr/local/bin/ghc" ;;
    *) ;;
  esac
}

# Extract GHC toolchain
function setup() {
  if [ -d "$TOP/cabal-cache" ]; then
      info "Extracting cabal cache..."
154 155
      mkdir -p "$cabal_dir"
      cp -Rf cabal-cache/* "$cabal_dir"
Ben Gamari's avatar
Ben Gamari committed
156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188
  fi

  if [[ -n "$needs_toolchain" ]]; then
    setup_toolchain
  fi
  case "$(uname)" in
    Darwin) darwin_setup ;;
    *) ;;
  esac

  # Make sure that git works
  git config user.email "ghc-ci@gitlab-haskell.org"
  git config user.name "GHC GitLab CI"

  info "====================================================="
  info "Toolchain versions"
  info "====================================================="
  show_tool GHC
  show_tool CABAL
  show_tool HAPPY
  show_tool ALEX
}

function fetch_ghc() {
  local v="$GHC_VERSION"
  if [[ -z "$v" ]]; then
      fail "GHC_VERSION is not set"
  fi

  if [ ! -e "$GHC" ]; then
      start_section "fetch GHC"
      url="https://downloads.haskell.org/~ghc/${GHC_VERSION}/ghc-${GHC_VERSION}-${boot_triple}.tar.xz"
      info "Fetching GHC binary distribution from $url..."
189
      curl "$url" > ghc.tar.xz || fail "failed to fetch GHC binary distribution"
Ben Gamari's avatar
Ben Gamari committed
190 191 192
      tar -xJf ghc.tar.xz || fail "failed to extract GHC binary distribution"
      case "$(uname)" in
        MSYS_*|MINGW*)
193
          cp -r "ghc-${GHC_VERSION}"/* "$toolchain"
Ben Gamari's avatar
Ben Gamari committed
194 195
          ;;
        *)
196 197 198
          pushd "ghc-${GHC_VERSION}"
          ./configure --prefix="$toolchain"
          "$MAKE" install
Ben Gamari's avatar
Ben Gamari committed
199 200 201
          popd
          ;;
      esac
202
      rm -Rf "ghc-${GHC_VERSION}" ghc.tar.xz
Ben Gamari's avatar
Ben Gamari committed
203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225
      end_section "fetch GHC"
  fi

}

function fetch_cabal() {
  local v="$CABAL_INSTALL_VERSION"
  if [[ -z "$v" ]]; then
      fail "CABAL_INSTALL_VERSION is not set"
  fi

  if [ ! -e "$CABAL" ]; then
      start_section "fetch GHC"
      case "$(uname)" in
        # N.B. Windows uses zip whereas all others use .tar.xz
        MSYS_*|MINGW*)
          case "$MSYSTEM" in
            MINGW32) cabal_arch="i386" ;;
            MINGW64) cabal_arch="x86_64" ;;
            *) fail "unknown MSYSTEM $MSYSTEM" ;;
          esac
          url="https://downloads.haskell.org/~cabal/cabal-install-$v/cabal-install-$v-$cabal_arch-unknown-mingw32.zip"
          info "Fetching cabal binary distribution from $url..."
226 227 228
          curl "$url" > "$TMP/cabal.zip"
          unzip "$TMP/cabal.zip"
          mv cabal.exe "$CABAL"
Ben Gamari's avatar
Ben Gamari committed
229 230 231 232 233 234 235 236 237 238 239
          ;;
        *)
          local base_url="https://downloads.haskell.org/~cabal/cabal-install-$v/"
          case "$(uname)" in
            Darwin) cabal_url="$base_url/cabal-install-$v-x86_64-apple-darwin17.7.0.tar.xz" ;;
            FreeBSD)
              #cabal_url="$base_url/cabal-install-$v-x86_64-portbld-freebsd.tar.xz" ;;
              cabal_url="http://home.smart-cactus.org/~ben/ghc/cabal-install-3.0.0.0-x86_64-portbld-freebsd.tar.xz" ;;
            *) fail "don't know where to fetch cabal-install for $(uname)"
          esac
          echo "Fetching cabal-install from $cabal_url"
240
          curl "$cabal_url" > cabal.tar.xz
Ben Gamari's avatar
Ben Gamari committed
241
          tar -xJf cabal.tar.xz
242
          mv cabal "$toolchain/bin"
Ben Gamari's avatar
Ben Gamari committed
243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328
          ;;
      esac
      end_section "fetch GHC"
  fi
}

# For non-Docker platforms we prepare the bootstrap toolchain
# here. For Docker platforms this is done in the Docker image
# build.
function setup_toolchain() {
  fetch_ghc
  fetch_cabal
  cabal_install="$CABAL v2-install --index-state=$hackage_index_state --installdir=$toolchain/bin"
  # Avoid symlinks on Windows
  case "$(uname)" in
    MSYS_*|MINGW*) cabal_install="$cabal_install --install-method=copy" ;;
    *) ;;
  esac

  if [ ! -e "$HAPPY" ]; then
      info "Building happy..."
      cabal update
      $cabal_install happy
  fi

  if [ ! -e "$ALEX" ]; then
      info "Building alex..."
      cabal update
      $cabal_install alex
  fi
}

function cleanup_submodules() {
  start_section "clean submodules"
  info "Cleaning submodules..."
  # On Windows submodules can inexplicably get into funky states where git
  # believes that the submodule is initialized yet its associated repository
  # is not valid. Avoid failing in this case with the following insanity.
  git submodule sync --recursive || git submodule deinit --force --all
  git submodule update --init --recursive
  git submodule foreach git clean -xdf
  end_section "clean submodules"
}

function prepare_build_mk() {
  if [[ -z "$BUILD_FLAVOUR" ]]; then fail "BUILD_FLAVOUR is not set"; fi
  if [[ -z ${BUILD_SPHINX_HTML:-} ]]; then BUILD_SPHINX_HTML=YES; fi
  if [[ -z ${BUILD_SPHINX_PDF:-} ]]; then BUILD_SPHINX_PDF=YES; fi
  if [[ -z ${INTEGER_LIBRARY:-} ]]; then INTEGER_LIBRARY=integer-gmp; fi

  cat > mk/build.mk <<EOF
V=1
HADDOCK_DOCS=YES
LATEX_DOCS=YES
HSCOLOUR_SRCS=YES
BUILD_SPHINX_HTML=$BUILD_SPHINX_HTML
BUILD_SPHINX_PDF=$BUILD_SPHINX_PDF
BeConservative=YES
INTEGER_LIBRARY=$INTEGER_LIBRARY
XZ_CMD=$XZ

BuildFlavour=$BUILD_FLAVOUR
ifneq "\$(BuildFlavour)" ""
include mk/flavours/\$(BuildFlavour).mk
endif
GhcLibHcOpts+=-haddock
EOF

  if [ -n "$HADDOCK_HYPERLINKED_SOURCES" ]; then
    echo "EXTRA_HADDOCK_OPTS += --hyperlinked-source --quickjump" >> mk/build.mk
  fi

  case "$(uname)" in
    Darwin) echo "libraries/integer-gmp_CONFIGURE_OPTS += --configure-option=--with-intree-gmp" >> mk/build.mk ;;
    *) ;;
  esac

  info "build.mk is:"
  cat mk/build.mk
}

function configure() {
  start_section "booting"
  run python3 boot
  end_section "booting"

329 330 331 332 333
  local target_args=""
  if [[ -n "$triple" ]]; then
    target_args="--target=$triple"
  fi

Ben Gamari's avatar
Ben Gamari committed
334 335 336
  start_section "configuring"
  run ./configure \
    --enable-tarballs-autodownload \
337
    $target_args \
Ben Gamari's avatar
Ben Gamari committed
338
    $CONFIGURE_ARGS \
339 340 341
    GHC="$GHC" \
    HAPPY="$HAPPY" \
    ALEX="$ALEX" \
Ben Gamari's avatar
Ben Gamari committed
342 343 344 345 346 347 348 349 350 351 352 353
    || ( cat config.log; fail "configure failed" )
  end_section "configuring"
}

function build_make() {
  prepare_build_mk
  if [[ -z "$BIN_DIST_PREP_TAR_COMP" ]]; then
    fail "BIN_DIST_PREP_TAR_COMP is not set"
  fi

  echo "include mk/flavours/${BUILD_FLAVOUR}.mk" > mk/build.mk
  echo 'GhcLibHcOpts+=-haddock' >> mk/build.mk
354 355 356
  run "$MAKE" -j"$cores" $MAKE_ARGS
  run "$MAKE" -j"$cores" binary-dist-prep TAR_COMP_OPTS=-1
  ls -lh "$BIN_DIST_PREP_TAR_COMP"
Ben Gamari's avatar
Ben Gamari committed
357 358 359 360
}

function fetch_perf_notes() {
  info "Fetching perf notes..."
361
  "$TOP/.gitlab/test-metrics.sh" pull
Ben Gamari's avatar
Ben Gamari committed
362 363 364 365
}

function push_perf_notes() {
  info "Pushing perf notes..."
366
  "$TOP/.gitlab/test-metrics.sh" push
Ben Gamari's avatar
Ben Gamari committed
367 368 369
}

function test_make() {
370 371 372
  run "$MAKE" test_bindist TEST_PREP=YES
  run "$MAKE" V=0 test \
    THREADS="$cores" \
Ben Gamari's avatar
Ben Gamari committed
373 374 375 376 377 378 379 380
    JUNIT_FILE=../../junit.xml
}

function build_hadrian() {
  if [ -z "$FLAVOUR" ]; then
    fail "FLAVOUR not set"
  fi

381
  run_hadrian binary-dist
Ben Gamari's avatar
Ben Gamari committed
382 383 384 385 386 387

  mv _build/bindist/ghc*.tar.xz ghc.tar.xz
}

function test_hadrian() {
  cd _build/bindist/ghc-*/
388 389
  run ./configure --prefix="$TOP"/_build/install
  run "$MAKE" install
Ben Gamari's avatar
Ben Gamari committed
390 391
  cd ../../../

392
  run_hadrian \
Ben Gamari's avatar
Ben Gamari committed
393 394
    test \
    --summary-junit=./junit.xml \
395
    --test-compiler="$TOP"/_build/install/bin/ghc
Ben Gamari's avatar
Ben Gamari committed
396 397 398 399
}

function clean() {
  rm -R tmp
400
  run "$MAKE" --quiet clean || true
Ben Gamari's avatar
Ben Gamari committed
401 402 403
  run rm -Rf _build
}

404
function run_hadrian() {
405
  run hadrian/build-cabal \
406 407
    --flavour="$FLAVOUR" \
    -j"$cores" \
Ben Gamari's avatar
Ben Gamari committed
408
    --broken-test="$BROKEN_TESTS" \
409 410 411 412
    $HADRIAN_ARGS \
    $@
}

413 414 415 416 417 418 419 420 421
# A convenience function to allow debugging in the CI environment.
function shell() {
  local cmd=$@
  if [ -z "$cmd" ]; then
    cmd="bash -i"
  fi
  run $cmd
}

Ben Gamari's avatar
Ben Gamari committed
422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449
# Determine Cabal data directory
case "$(uname)" in
  MSYS_*|MINGW*) exe=".exe"; cabal_dir="$APPDATA/cabal" ;;
  *) cabal_dir="$HOME/.cabal"; exe="" ;;
esac

# Platform-specific environment initialization
MAKE="make"
case "$(uname)" in
  MSYS_*|MINGW*) mingw_init ;;
  Darwin) boot_triple="x86_64-apple-darwin" ;;
  FreeBSD)
    boot_triple="x86_64-portbld-freebsd"
    MAKE="gmake"
    ;;
  Linux) ;;
  *) fail "uname $(uname) is not supported" ;;
esac

set_toolchain_paths

case $1 in
  setup) setup && cleanup_submodules ;;
  configure) configure ;;
  build_make) build_make ;;
  test_make) fetch_perf_notes; test_make; push_perf_notes ;;
  build_hadrian) build_hadrian ;;
  test_hadrian) fetch_perf_notes; test_hadrian; push_perf_notes ;;
450
  run_hadrian) run_hadrian $@ ;;
Ben Gamari's avatar
Ben Gamari committed
451
  clean) clean ;;
452
  shell) shell $@ ;;
Ben Gamari's avatar
Ben Gamari committed
453 454
  *) fail "unknown mode $1" ;;
esac