Rewrite Travis CI script.

Lots of changes:

    - When possible, we use the container infrastructure (sudo: false)
      rather than Google Compute Engine infrastructure (sudo: required).
      Unfortunately, we can't use GCE for the Linux builds, where
      reduced RAM available hoses are GHC build.

    - Switched from using ./Setup and old-style cabal to new-build.
      There are numerous great benefits but the best is that
      .cabal/store can be cached on Travis, leading to huge speedups
      on the build.  Downside is we need to string-and-ceiling-wax
      support for test/haddock/etc.

    - I stopped bootstrapping on every build we do; instead there
      is a separate bootstrap build we do to make sure that that
      is working.  This also speeds up the basic builds since
      we are not building Cabal/cabal-install multiple times.

    - There are some hacks.  The big one is setting CABAL_BUILDDIR
      explicitly; this smooths over quite a few infelicities in
      the current new-build implementation.

Signed-off-by: Edward Z. Yang <ezyang@cs.stanford.edu>

.travis.yml

@@ -2,36 +2,53 @@
 # We specify language: c, so it doesn't default to e.g. ruby
 language: c
 
-sudo: required
+sudo: false
 
 # The following enables several GHC versions to be tested; often it's enough to
 # test only against the last release in a major GHC version. Feel free to omit
 # lines listings versions you don't need/want testing for.
 matrix:
   include:
-   - env: GHCVER=7.4.2
+   - env: GHCVER=none SCRIPT=meta
      os: linux
-   - env: GHCVER=7.6.3
+   # These don't have -dyn/-prof whitelisted yet, so we have to
+   # do the old-style installation
+   - env: GHCVER=7.4.2 SCRIPT=script
      os: linux
-   - env: GHCVER=7.8.4
+     sudo: required
+   - env: GHCVER=7.6.3 SCRIPT=script
      os: linux
-   - env: GHCVER=7.10.3
+     sudo: required
+   - env: GHCVER=7.8.4 SCRIPT=script
      os: linux
-   - env: GHCVER=8.0.1 TEST_OLDER=NO DEPLOY_DOCS=YES
+     sudo: required
+   # Ugh, we'd like to drop 'sudo: required' and use the
+   # apt plugin for the next two
+   # but the GCE instance we get has more memory, which makes
+   # a big difference
+   - env: GHCVER=7.10.3 SCRIPT=script
+     os: linux
+     sudo: required
+   - env: GHCVER=8.0.1 SCRIPT=script DEPLOY_DOCS=YES
+     sudo: required
+     os: linux
+   - env: GHCVER=8.0.1 SCRIPT=bootstrap
+     sudo: required
      os: linux
    # It's not worth the trouble to make older GHC work with clang's cpp
-   # Obviously TEST_OLDER doesn't work with OSX
+   # TEST_OLDER is not implemented on OSX
    #
    # Also we might want to specify OSX version
    # https://docs.travis-ci.com/user/osx-ci-environment/#OS-X-Version
-   - env: GHCVER=7.8.4
+   - env: GHCVER=7.8.4 SCRIPT=script
      os: osx
-   - env: GHCVER=7.10.3
+   - env: GHCVER=7.10.3 SCRIPT=script
      os: osx
-   - env: GHCVER=8.0.1
+   - env: GHCVER=8.0.1 SCRIPT=script
      os: osx
   allow_failures:
    - env: GHCVER=head
+     sudo: required
 
  # TODO add PARSEC_BUNDLED=YES when it's so
  # It seems pointless to run head if we're going to ignore the results.
@@ -39,13 +56,18 @@ matrix:
 
 # Note: the distinction between `before_install` and `install` is not important.
 before_install:
- - ./travis-install.sh
  - export PATH=/opt/ghc/$GHCVER/bin:$PATH
- - export PATH=$HOME/.ghc-install/$GHCVER/bin:$PATH;
+ - export PATH=$HOME/.ghc-install/$GHCVER/bin:$PATH
+ - export PATH=$HOME/bin:$PATH
+ - export PATH=$HOME/.cabal/bin:$PATH
+ - export PATH=/opt/cabal/1.24/bin:$PATH
+ - export PATH=/opt/happy/1.19.5/bin:$PATH
+ - ./travis-install.sh
 
  # Set up deployment to the haskell/cabal-website repo.
  # NB: these commands MUST be in .travis.yml, otherwise the secret key can be
  # leaked! See https://github.com/travis-ci/travis.rb/issues/423.
+ # umask to get the permissions to be 400.
  - if [ "x$TRAVIS_PULL_REQUEST" = "xfalse" -a "x$TRAVIS_BRANCH" = "xmaster" -a "x$DEPLOY_DOCS" = "xYES"  ]; then (umask 377 && openssl aes-256-cbc -K $encrypted_edaf6551664d_key -iv $encrypted_edaf6551664d_iv -in id_rsa_cabal_website.aes256.enc -out ~/.ssh/id_rsa -d); fi
 
 install:
@@ -57,7 +79,20 @@ install:
 # ./dist/setup/setup here instead of cabal-install to avoid breakage when the
 # build config format changed.
 script:
- - ./travis-script.sh -j
+ - ./travis-${SCRIPT}.sh
+
+cache:
+ directories:
+ - $HOME/.cabal/packages
+ - $HOME/.cabal/store
+ - $HOME/.cabal/bin
+
+# We remove the index because it churns quite a bit and we don't want
+# to pay the cost of repeatedly caching it even though we don't care
+# about most changing packages.
+before_cache:
+ - rm -fv $HOME/.cabal/packages/hackage.haskell.org/build-reports.log
+ - rm -fv $HOME/.cabal/packages/hackage.haskell.org/00-index*
 
 # Deploy Haddocks to the haskell/cabal-website repo.
 after_success:

Cabal/Distribution/Compat/Environment.hs

@@ -51,7 +51,7 @@ lookupEnv name = (Just `fmap` System.getEnv name) `catchIO` const (return Nothin
 -- empty string or contains an equals sign.
 setEnv :: String -> String -> IO ()
 setEnv key value_
-  | null value = error "Distribuiton.Compat.setEnv: empty string"
+  | null value = error "Distribution.Compat.setEnv: empty string"
   | otherwise  = setEnv_ key value
   where
     -- NOTE: Anything that follows NUL is ignored on both POSIX and Windows. We

cabal-install/tests/IntegrationTests.hs

@@ -9,7 +9,7 @@ module Main
 
 -- Modules from Cabal.
 import Distribution.Compat.CreatePipe (createPipe)
-import Distribution.Compat.Environment (setEnv)
+import Distribution.Compat.Environment (setEnv, getEnvironment)
 import Distribution.Compat.Internal.TempFile (createTempDirectory)
 import Distribution.Simple.Configure (findDistPrefOrDefault)
 import Distribution.Simple.Program.Builtin (ghcPkgProgram)
@@ -100,7 +100,11 @@ run cwd path args = do
     (hReadStdOut, hWriteStdOut) <- createPipe
     (hReadStdErr, hWriteStdErr) <- createPipe
     -- Run the process
-    pid <- runProcess path' args (Just cwd) Nothing Nothing (Just hWriteStdOut) (Just hWriteStdErr)
+    env0 <- getEnvironment
+    -- CABAL_BUILDDIR can interfere with test running, so
+    -- be sure to clear it out.
+    let env = filter ((/= "CABAL_BUILDDIR") . fst) env0
+    pid <- runProcess path' args (Just cwd) (Just env) Nothing (Just hWriteStdOut) (Just hWriteStdErr)
     -- Return the pid and read ends of the pipes
     return (pid, hReadStdOut, hReadStdErr)
   -- Read subprocess output using asynchronous threads; we need to

cabal.project.travis

+-- Force error messages to be better
+-- Parallel new-build error messages are non-existent.
+-- Turn off parallelization to get good errors.
+jobs: 1
+
+-- The -fno-warn-orphans is a hack to make Cabal-1.24
+-- build properly (unfortunately the flags here get applied
+-- to the dependencies too!)
+package Cabal
+  ghc-options: -Werror -fno-warn-orphans
+
+package cabal-install
+  ghc-options: -Werror

travis-bootstrap.sh

+#!/bin/sh
+
+. ./travis-common.sh
+
+# ---------------------------------------------------------------------
+# Bootstrap cabal, to verify bootstrap.sh script works.
+# ---------------------------------------------------------------------
+
+bootstrap_jobs="-j"
+
+(cd cabal-install && timed env EXTRA_CONFIGURE_OPTS="" ./bootstrap.sh $bootstrap_jobs --no-doc)
+timed $HOME/.cabal/bin/cabal --version
+PATH=$HOME/.cabal/bin:$PATH
+
+# ---------------------------------------------------------------------
+# Verify that installation from tarball works.
+# ---------------------------------------------------------------------
+
+# The following scriptlet checks that the resulting source distribution can be
+# built & installed.
+install_from_tarball() {
+   SRC_TGZ=$(cabal info . | awk '{print $2 ".tar.gz";exit}') ;
+   export SRC_TGZ
+   if [ -f "dist/$SRC_TGZ" ]; then
+      cabal install --force-reinstalls $jobs "dist/$SRC_TGZ" -v2;
+   else
+      echo "expected 'dist/$SRC_TGZ' not found";
+      exit 1;
+   fi
+}
+
+timed cabal update
+
+echo Cabal
+(cd Cabal && timed cabal sdist)
+(cd Cabal && timed install_from_tarball)
+
+echo cabal-install
+(cd cabal-install && timed cabal sdist)
+(cd cabal-install && timed install_from_tarball)

travis-common.sh

+set -e
+
+CABAL_VERSION="1.25.0.0"
+
+# ---------------------------------------------------------------------
+# Timing / diagnostic output
+# ---------------------------------------------------------------------
+
+timed() {
+    echo "\$ $*"
+    start_time=$(date +%s)
+    $*
+    end_time=$(date +%s)
+    duration=$((end_time - start_time))
+    echo "$* took $duration seconds."
+    echo "----"
+}

travis-install.sh

@@ -5,10 +5,14 @@ travis_retry () {
     $*  || (sleep 1 && $*) || (sleep 2 && $*)
 }
 
+if [ "$GHCVER" = "none" ]; then
+    exit 0
+fi
+
 if [ "$TRAVIS_OS_NAME" = "linux" ]; then
     travis_retry sudo add-apt-repository -y ppa:hvr/ghc
     travis_retry sudo apt-get update
-    travis_retry sudo apt-get install --force-yes ghc-$GHCVER-prof ghc-$GHCVER-dyn
+    travis_retry sudo apt-get install --force-yes cabal-install-1.24 happy-1.19.5 ghc-$GHCVER-prof ghc-$GHCVER-dyn
     if [ "$TEST_OLDER" == "YES" ]; then travis_retry sudo apt-get install --force-yes ghc-7.0.4-prof ghc-7.0.4-dyn ghc-7.2.2-prof ghc-7.2.2-dyn; fi
 
 elif [ "$TRAVIS_OS_NAME" = "osx" ]; then
@@ -50,6 +54,12 @@ elif [ "$TRAVIS_OS_NAME" = "osx" ]; then
     make install;
     cd ..;
 
+    travis_retry curl -L https://www.haskell.org/cabal/release/cabal-install-1.24.0.0/cabal-install-1.24.0.0-x86_64-apple-darwin-yosemite.tar.gz -o cabal-install.tar.gz
+    TAR=$PWD/cabal-install.tar.gz
+    mkdir "${HOME}/bin"
+    (cd "${HOME}/bin" && tar -xzf "$TAR")
+    "${HOME}/bin/cabal" --version
+
 else
     echo "Not linux or osx: $TRAVIS_OS_NAME"
     false

travis-meta.sh

+#!/bin/sh
+
+. ./travis-common.sh
+
+# ---------------------------------------------------------------------
+# Check that auto-generated files/fields are up to date.
+# ---------------------------------------------------------------------
+
+# Regenerate the CONTRIBUTORS file.
+# Currently doesn't work because Travis uses --depth=50 when cloning.
+#./Cabal/misc/gen-authors.sh > AUTHORS
+
+# Regenerate the 'extra-source-files' field in Cabal.cabal.
+(cd Cabal && timed ./misc/gen-extra-source-files.sh Cabal.cabal)
+
+# Regenerate the 'extra-source-files' field in cabal-install.cabal.
+(cd cabal-install && ../Cabal/misc/gen-extra-source-files.sh cabal-install.cabal)
+
+# Fail if the diff is not empty.
+timed ./Cabal/misc/travis-diff-files.sh

travis-script.sh

 #!/bin/sh
-set -ev
-
-usage() {
-    echo "Usage: travis-script.sh"
-    echo "-h  Print this help string"
-    echo "-j  Number of concurrent workers to use (Default: 1)"
-    echo "    -j without an argument will use all available cores"
-}
-
-jobs="-j1"
-# We use a separate jobs var for bootstrap.sh because the arg parser in
-# bootstrap.sh wouldn't be able to interpret -j1 (or any -jN).
-bootstrap_jobs="-j 1"
-while getopts ":hj:" opt; do
-    case $opt in
-        h)
-            usage
-            exit 0
-            ;;
-        j)
-            jobs="-j$OPTARG"
-            bootstrap_jobs="-j $OPTARG"
-            ;;
-        :)
-            # Argument-less -j
-            if [ "$OPTARG" = "j" ]; then
-                jobs="-j"
-                bootstrap_jobs="-j"
-            fi
-            ;;
-        \?)
-            echo "Invalid option: $OPTARG"
-            usage
-            exit 1
-            ;;
-    esac
-done
-shift $((OPTIND-1))
-
-# Do not try to use -j with GHC older than 7.8
-case $GHCVER in
-    7.4*|7.6*)
-        jobs=""
-        bootstrap_jobs=""
-        ;;
-    *)
-        ;;
-esac
+
+# ATTENTION! Before editing this file, maybe you can make a
+# separate script to do your test?  We don't want individual
+# Travis builds to take too long (they time out at 50min and
+# it's generally unpleasant if the build takes that long.)
+# If you make a separate matrix entry in .travis.yml it can
+# be run in parallel.
+
+. ./travis-common.sh
+
+CABAL_BDIR="${PWD}/dist-newstyle/build/Cabal-${CABAL_VERSION}"
+CABAL_INSTALL_BDIR="${PWD}/dist-newstyle/build/cabal-install-${CABAL_VERSION}"
+CABAL_INSTALL_SETUP="${CABAL_INSTALL_BDIR}/setup/setup"
+# --hide-successes uses terminal control characters which mess up
+# Travis's log viewer.  So just print them all!
+TEST_OPTIONS=""
 
 # ---------------------------------------------------------------------
-# Timing
+# Update the Cabal index
 # ---------------------------------------------------------------------
 
-timed() {
-    echo "$1"
-    start_time=$(date +%s)
-    $2
-    end_time=$(date +%s)
-    duration=$((end_time - start_time))
-    echo "$1 took $duration seconds."
-    echo "----"
-}
+timed cabal update
 
 # ---------------------------------------------------------------------
-# Bootstrap cabal, to verify bootstrap.sh script works.
+# Install happy if necessary
 # ---------------------------------------------------------------------
 
-bootstrap() {
-    OLD_CWD=$PWD
-
-    # Bootstrap
-    cd cabal-install
-    env EXTRA_CONFIGURE_OPTS="" ./bootstrap.sh $bootstrap_jobs --no-doc
-    ~/.cabal/bin/cabal --version
+if ! command -v happy; then
+    timed cabal install happy
+fi
 
-    # Move cabal for local use.
-    mkdir ~/fresh-cabal
-    mv -i ~/.cabal/bin/cabal ~/fresh-cabal/
+# ---------------------------------------------------------------------
+# Setup our local project
+# ---------------------------------------------------------------------
 
-    # Clean up after ourselves.
-    rm -r ~/.ghc ~/.cabal
+cp cabal.project.travis cabal.project.local
 
-    # From here on we use the freshly built cabal executable.
-    export PATH="$HOME/fresh-cabal/:$PATH"
+# ---------------------------------------------------------------------
+# Cabal
+# ---------------------------------------------------------------------
 
-    cd "$OLD_CWD"
-}
+export CABAL_BUILDDIR="${CABAL_BDIR}"
 
-install_parsec() {
-    # Initial working directory: base directory of Git repository
+# NB: Best to do everything for a single package together as it's
+# more efficient (since new-build will uselessly try to rebuild
+# Cabal otherwise).
+timed cabal new-build Cabal Cabal:package-tests Cabal:unit-tests
 
-    cabal update
+# Run tests
+(cd Cabal && timed ${CABAL_BDIR}/build/package-tests/package-tests $TEST_OPTIONS)
+(cd Cabal && timed ${CABAL_BDIR}/build/unit-tests/unit-tests       $TEST_OPTIONS)
 
-    # We depend on parsec nowadays, which isn't distributed with GHC <8.0
-    if [ "$PARSEC_BUNDLED" != "YES" ]; then
-        cabal install $jobs parsec
-    fi
-}
+# Run haddock (hack: use the Setup script from package-tests!)
+(cd Cabal && timed cabal act-as-setup --build-type=Simple -- haddock --builddir=${CABAL_BDIR})
 
-timed "bootstrap.sh script" bootstrap
-timed "Installing parsec" install_parsec
+# Redo the package tests with different versions of GHC
+# TODO: reenable me
+#   if [ "x$TEST_OLDER" = "xYES" -a "x$TRAVIS_OS_NAME" = "xlinux" ]; then
+#       CABAL_PACKAGETESTS_WITH_GHC=/opt/ghc/7.0.4/bin/ghc \
+#           ./dist/setup/setup test package-tests --show-details=streaming
+#       CABAL_PACKAGETESTS_WITH_GHC=/opt/ghc/7.2.2/bin/ghc \
+#           ./dist/setup/setup test package-tests --show-details=streaming
+#   fi
 
-# ---------------------------------------------------------------------
-# Check that auto-generated files/fields are up to date.
-# ---------------------------------------------------------------------
+# Check for package warnings
+(cd Cabal && timed cabal check)
 
-generated_files() {
-    # Regenerate the CONTRIBUTORS file.
-    # Currently doesn't work because Travis uses --depth=50 when cloning.
-    #./Cabal/misc/gen-authors.sh > AUTHORS
+# Test that an sdist can be created
+(cd Cabal && timed cabal sdist --builddir=${CABAL_BDIR})
 
-    # Regenerate the 'extra-source-files' field in Cabal.cabal.
-    cd Cabal
-    ./misc/gen-extra-source-files.sh Cabal.cabal
+unset CABAL_BUILDDIR
 
-    # Regenerate the 'extra-source-files' field in cabal-install.cabal.
-    cd ../cabal-install
-    ../Cabal/misc/gen-extra-source-files.sh cabal-install.cabal
-    cd ..
+# ---------------------------------------------------------------------
+# cabal-install
+# ---------------------------------------------------------------------
 
-    # Fail if the diff is not empty.
-    ./Cabal/misc/travis-diff-files.sh
-}
+# Setting the build directory here helps avoid sdist bugs.
+export CABAL_BUILDDIR="${CABAL_INSTALL_BDIR}"
 
-if [ "$TRAVIS_OS_NAME" = "linux" ]; then
-  timed "Generated files check" generated_files
-fi
+timed cabal new-build cabal-install:cabal \
+                      cabal-install:integration-tests \
+                      cabal-install:integration-tests2 \
+                      cabal-install:unit-tests \
+                      cabal-install:solver-quickcheck
 
-# ---------------------------------------------------------------------
-# Cabal
-# ---------------------------------------------------------------------
+# Run tests
+(cd cabal-install && timed ${CABAL_INSTALL_BDIR}/build/unit-tests/unit-tests         $TEST_OPTIONS)
+(cd cabal-install && timed ${CABAL_INSTALL_BDIR}/build/solver-quickcheck/solver-quickcheck  $TEST_OPTIONS --quickcheck-tests=1000)
+(cd cabal-install && timed ${CABAL_INSTALL_BDIR}/build/integration-tests/integration-tests  $TEST_OPTIONS)
+(cd cabal-install && timed ${CABAL_INSTALL_BDIR}/build/integration-tests2/integration-tests2 $TEST_OPTIONS)
 
-# The following scriptlet checks that the resulting source distribution can be
-# built & installed.
-install_from_tarball() {
-   SRC_TGZ=$(cabal info . | awk '{print $2 ".tar.gz";exit}') ;
-   export SRC_TGZ
-   if [ -f "dist/$SRC_TGZ" ]; then
-      cabal install $jobs "dist/$SRC_TGZ" -v2;
-   else
-      echo "expected 'dist/$SRC_TGZ' not found";
-      exit 1;
-   fi
-}
-
-cabal_dependencies() {
-    cd Cabal
-
-    # Build the setup script in the same way that cabal-install would:
-    mkdir -p ./dist/setup
-    cp Setup.hs ./dist/setup/setup.hs
-    ghc --make \
-        $jobs \
-        -odir ./dist/setup -hidir ./dist/setup -i -i. \
-        ./dist/setup/setup.hs -o ./dist/setup/setup \
-        -Wall -Werror -threaded
-
-    # Install test dependencies only after setup is built
-    cabal install $jobs --only-dependencies --enable-tests --enable-benchmarks
-}
-
-cabal_build() {
-    ./dist/setup/setup configure \
-        --user --ghc-option=-Werror --enable-tests --enable-benchmarks \
-        -v2 # -v2 provides useful information for debugging
-
-    # Build all libraries and executables (including tests/benchmarks)
-    ./dist/setup/setup build $jobs
-    ./dist/setup/setup haddock # see https://github.com/haskell/cabal/issues/2198
-    ./dist/setup/setup test --show-details=streaming --test-option=--hide-successes
-
-    # Redo the package tests with different versions of GHC
-    if [ "x$TEST_OLDER" = "xYES" -a "x$TRAVIS_OS_NAME" = "xlinux" ]; then
-        CABAL_PACKAGETESTS_WITH_GHC=/opt/ghc/7.0.4/bin/ghc \
-            ./dist/setup/setup test package-tests --show-details=streaming
-        CABAL_PACKAGETESTS_WITH_GHC=/opt/ghc/7.2.2/bin/ghc \
-            ./dist/setup/setup test package-tests --show-details=streaming
-    fi
-
-    cabal check
-    cabal sdist   # tests that a source-distribution can be generated
-    install_from_tarball
-
-    cd ..
-}
-
-timed "Cabal dependencies" cabal_dependencies
-timed "Cabal build" cabal_build
+# Haddock
+(cd cabal-install && timed ${CABAL_INSTALL_SETUP} haddock --builddir=${CABAL_INSTALL_BDIR} )
 
-# ---------------------------------------------------------------------
-# cabal-install
-# ---------------------------------------------------------------------
+(cd cabal-install && timed cabal check)
+(cd cabal-install && timed cabal sdist --builddir=${CABAL_INSTALL_BDIR})
 
-cabalinstall_dependencies() {
-    cd cabal-install
-
-    cabal install $jobs happy
-    cabal install $jobs --only-dependencies --enable-tests --enable-benchmarks
-}
-
-cabalinstall_build() {
-    cabal configure \
-        --user --ghc-option=-Werror --enable-tests --enable-benchmarks \
-        -v2 # -v2 provides useful information for debugging
-    cabal build $jobs
-    cabal haddock # see https://github.com/haskell/cabal/issues/2198
-    cabal test unit-tests --show-details=streaming --test-option=--hide-successes
-    cabal test integration-tests --show-details=streaming --test-option=--hide-successes
-    cabal test integration-tests2 --show-details=streaming --test-option=--hide-successes
-    cabal test solver-quickcheck --show-details=streaming --test-option=--hide-successes \
-        --test-option=--quickcheck-tests=1000
-    cabal check
-    ./dist/setup/setup sdist
-    install_from_tarball
-
-    cd ..
-}
-
-timed "cabal-install dependencies" cabalinstall_dependencies
-timed "cabal-install build" cabalinstall_build
+unset CABAL_BUILDDIR
 
 # Check what we got
-~/.cabal/bin/cabal --version
+${CABAL_INSTALL_BDIR}/build/cabal/cabal --version