Use parsec, drop parsec flag

- Manually generate Lexer.hs
- Temporarily disable parser-hackage-tests on appveyor (needs
  01-index.tar)
- Add root Makefile to handle commmon dev tasks (genering Lexer.hs)

.travis.yml

@@ -54,9 +54,6 @@ matrix:
    - env: GHCVER=8.0.2 SCRIPT=solver-debug-flags USE_GOLD=YES
      sudo: required
      os: linux
-   - env: GHCVER=8.0.2 SCRIPT=script PARSEC=YES TAGSUFFIX="-parsec" USE_GOLD=YES
-     os: linux
-     sudo: required
    - env: GHCVER=8.0.2 SCRIPT=script DEBUG_EXPENSIVE_ASSERTIONS=YES TAGSUFFIX="-fdebug-expensive-assertions" USE_GOLD=YES
      os: linux
      sudo: required

Cabal/Cabal.cabal

@@ -64,11 +64,6 @@ flag old-directory
   description:  Use directory < 1.2 and old-time
   default:      False
 
-flag parsec
-  description:  Use parsec parser
-  default:      False
-  manual:       True
-
 flag parsec-struct-diff
   description:  Use StructDiff in parsec tests. Affects only parsec tests.
   default:      False
@@ -267,26 +262,22 @@ library
     Language.Haskell.Extension
     Distribution.Compat.Binary
 
-  if flag(parsec)
-    cpp-options: -DCABAL_PARSEC
-    build-depends:
-      transformers,
-      parsec >= 3.1.9 && <3.2
-    build-tools:
-      alex >=3.1.4 && <3.3
-    exposed-modules:
-      Distribution.Compat.Parsec
-      Distribution.PackageDescription.Parsec
-      Distribution.PackageDescription.Parsec.FieldDescr
-      Distribution.Parsec.Class
-      Distribution.Parsec.ConfVar
-      Distribution.Parsec.Lexer
-      Distribution.Parsec.LexerMonad
-      Distribution.Parsec.Parser
-      Distribution.Parsec.Types.Common
-      Distribution.Parsec.Types.Field
-      Distribution.Parsec.Types.FieldDescr
-      Distribution.Parsec.Types.ParseResult
+  build-depends:
+    transformers,
+    parsec >= 3.1.9 && <3.2
+  exposed-modules:
+    Distribution.Compat.Parsec
+    Distribution.PackageDescription.Parsec
+    Distribution.PackageDescription.Parsec.FieldDescr
+    Distribution.Parsec.Class
+    Distribution.Parsec.ConfVar
+    Distribution.Parsec.Lexer
+    Distribution.Parsec.LexerMonad
+    Distribution.Parsec.Parser
+    Distribution.Parsec.Types.Common
+    Distribution.Parsec.Types.Field
+    Distribution.Parsec.Types.FieldDescr
+    Distribution.Parsec.Types.ParseResult
 
   other-modules:
     Distribution.Backpack.PreExistingComponent
@@ -382,9 +373,6 @@ test-suite unit-tests
   default-language: Haskell2010
 
 test-suite parser-tests
-  if !flag(parsec)
-    buildable: False
-
   type: exitcode-stdio-1.0
   hs-source-dirs: tests
   main-is: ParserTests.hs
@@ -400,15 +388,18 @@ test-suite parser-tests
   default-language: Haskell2010
 
 test-suite parser-hackage-tests
-  if !flag(parsec)
-    buildable: False
-
   type: exitcode-stdio-1.0
   main-is: ParserHackageTests.hs
 
+  -- TODO: need to get 01-index.tar on appveyor
+  if os(windows)
+    buildable: False
+
   hs-source-dirs: tests
   build-depends:
     base,
+    base-orphans == 0.6.*,
+    base-compat >=0.9.3 && <0.10,
     containers,
     tar >=0.5 && <0.6,
     bytestring,

Cabal/Distribution/PackageDescription/Parsec.hs

@@ -168,7 +168,8 @@ parseGenericPackageDescription' lexWarnings fs = do
     gpd <-  goFields emptyGpd fs'
     -- Various post checks
     maybeWarnCabalVersion syntax (packageDescription gpd)
-    checkForUndefinedFlags gpd
+    -- TODO: this does nothing
+    -- checkForUndefinedFlags gpd
     -- TODO: do other validations
     return gpd
   where
@@ -326,42 +327,6 @@ parseGenericPackageDescription' lexWarnings fs = do
 
     maybeWarnCabalVersion _ _ = return ()
 
-{-
-    handleFutureVersionParseFailure :: Version -> ParseResult a -> ParseResult GenericPackageDescription
-    handleFutureVersionParseFailure _cabalVersionNeeded _parseBody =
-        error "handleFutureVersionParseFailure"
--}
-
- {-
-      undefined (unless versionOk (warning message) >> parseBody)
-        `catchParseError` \parseError -> case parseError of
-        TabsError _   -> parseFail parseError
-        _ | versionOk -> parseFail parseError
-          | otherwise -> fail message
-      where versionOk = cabalVersionNeeded <= cabalVersion
-            message   = "This package requires at least Cabal version "
-                     ++ display cabalVersionNeeded
-    -}
-
-    checkForUndefinedFlags
-        :: GenericPackageDescription
-        -> ParseResult ()
-    checkForUndefinedFlags _gpd = pure ()
-{-
-        let definedFlags = map flagName flags
-        mapM_ (checkCondTreeFlags definedFlags) (maybeToList mlib)
-        mapM_ (checkCondTreeFlags definedFlags . snd) sub_libs
-        mapM_ (checkCondTreeFlags definedFlags . snd) exes
-        mapM_ (checkCondTreeFlags definedFlags . snd) tests
-
-    checkCondTreeFlags :: [FlagName] -> CondTree ConfVar c a -> PM ()
-    checkCondTreeFlags definedFlags ct = do
-        let fv = nub $ freeVars ct
-        unless (all (`elem` definedFlags) fv) $
-            fail $ "These flags are used without having been defined: "
-                ++ intercalate ", " [ n | FlagName n <- fv \\ definedFlags ]
--}
-
 parseName :: Position -> [SectionArg Position] -> ParseResult String
 parseName pos args = case args of
     [SecArgName _pos secName] ->

Cabal/Distribution/Simple.hs

@@ -105,12 +105,8 @@ import Distribution.Compat.GetShortPathName (getShortPathName)
 
 import Data.List       (unionBy, (\\))
 
-#ifdef CABAL_PARSEC
 import Distribution.PackageDescription.Parsec
 import Distribution.PackageDescription.Parse (readHookedBuildInfo)
-#else
-import Distribution.PackageDescription.Parse
-#endif
 
 -- | A simple implementation of @main@ for a Cabal setup script.
 -- It reads the package description file using IO, and performs the
@@ -237,9 +233,7 @@ confPkgDescr hooks verbosity mb_path = do
         pdfile <- case mb_path of
                     Nothing -> defaultPackageDesc verbosity
                     Just path -> return path
-#ifdef CABAL_PARSEC
         info verbosity "Using Parsec parser"
-#endif
         descr  <- readGenericPackageDescription verbosity pdfile
         return (Just pdfile, descr)
 

Cabal/tests/ParserHackageTests.hs

+{-# LANGUAGE CPP #-}
 {-# LANGUAGE Rank2Types #-}
 module Main where
 
+import Prelude ()
+import Prelude.Compat
+
 import           Control.Applicative
-                 (Applicative (..), (<$>), Const (..))
+                 (Const (..))
 import           Control.Monad                          (when, unless)
 import           Data.Foldable
-                 (foldMap, for_, traverse_)
+                 (for_, traverse_)
 import           Data.List                              (isPrefixOf, isSuffixOf)
 import           Data.Maybe                             (mapMaybe, listToMaybe)
-import           Data.Monoid                            (Monoid (..), Sum (..))
-import           Data.Traversable                       (traverse)
+import           Data.Monoid                            (Sum (..))
 import           Distribution.Simple.Utils              (fromUTF8LBS, ignoreBOM)
 import           System.Directory
                  (getAppUserDataDirectory)
@@ -21,6 +24,8 @@ import           Distribution.Types.Dependency
 import           Distribution.Types.UnqualComponentName
 import           Distribution.PackageDescription
 
+import           Data.Orphans ()
+
 import qualified Codec.Archive.Tar                      as Tar
 import qualified Data.ByteString                        as B
 import qualified Data.ByteString.Char8                  as B8
@@ -75,7 +80,7 @@ parseIndex' action path = do
        fpath = Tar.entryPath entry
 
 readFieldTest :: FilePath -> BSL.ByteString -> IO ()
-readFieldTest fpath bsl = case Parsec.readFields $ BSL.toStrict bsl of
+readFieldTest fpath bsl = case Parsec.readFields $ bslToStrict bsl of
     Right _  -> return ()
     Left err -> putStrLn $ fpath ++ "\n" ++ show err
 
@@ -101,7 +106,7 @@ compareTest pfx fpath bsl
         ReadP.ParseFailed err -> print err >> exitFailure
     traverse_ (putStrLn . ReadP.showPWarning fpath) readpWarnings
 
-    let (warnings, errors, parsec') = Parsec.runParseResult $ Parsec.parseGenericPackageDescription (BSL.toStrict bsl)
+    let (warnings, errors, parsec') = Parsec.runParseResult $ Parsec.parseGenericPackageDescription (bslToStrict bsl)
     traverse_ (putStrLn . Parsec.showPWarning fpath) warnings
     traverse_ (putStrLn . Parsec.showPError fpath) errors
     parsec <- maybe (print readp >> exitFailure) return parsec'
@@ -158,7 +163,7 @@ parseReadpTest fpath bsl = unless (any ($ fpath) problematicFiles) $ do
 
 parseParsecTest :: FilePath -> BSL.ByteString -> IO ()
 parseParsecTest fpath bsl = unless (any ($ fpath) problematicFiles) $ do
-    let bs = BSL.toStrict bsl
+    let bs = bslToStrict bsl
     let (_warnings, errors, parsec) = Parsec.runParseResult $ Parsec.parseGenericPackageDescription bs
     case parsec of
         Just _ -> return ()
@@ -213,6 +218,18 @@ main = do
         for_ (Map.toList warn) $ \(t, Sum c) ->
             putStrLn $ " - " ++ show t ++ " : " ++ show c
 
+-------------------------------------------------------------------------------
+--
+-------------------------------------------------------------------------------
+
+bslToStrict :: BSL.ByteString -> B.ByteString
+#if MIN_VERSION_bytestring(0,10,0)
+bslToStrict = BSL.toStrict
+#else
+-- Not effective!
+bslToStrict = B.concat . BSL.toChunks
+#endif
+
 -------------------------------------------------------------------------------
 -- Index shuffling
 -------------------------------------------------------------------------------

Makefile

+.PHONY : all lexer lib exe
+
+LEXER_HS:=Cabal/Distribution/Parsec/Lexer.hs
+
+all : exe lib
+
+lexer : $(LEXER_HS)
+
+$(LEXER_HS) : boot/Lexer.x
+	alex --ghc -o $@ $^
+
+lib : $(LEXER_HS)
+	cabal new-build --enable-tests Cabal
+
+exe : $(LEXER_HS)
+	cabal new-build  --enable-tests cabal

appveyor.yml

@@ -5,13 +5,15 @@ install:
   - refreshenv
   # See http://help.appveyor.com/discussions/problems/6312-curl-command-not-found#comment_42195491
   # NB: Do this after refreshenv, otherwise it will be clobbered!
-  - set PATH=C:\Program Files\Git\mingw64\bin;%PATH%
+  - set PATH=%APPDATA%\cabal\bin;C:\Program Files\Git\mingw64\bin;%PATH%
   # TODO: remove --insecure, this is to workaround haskell.org
   # failing to send intermediate cert; see https://github.com/haskell/cabal/pull/4172
   - curl -o cabal.zip --insecure --progress-bar https://www.haskell.org/cabal/release/cabal-install-1.24.0.0/cabal-install-1.24.0.0-x86_64-unknown-mingw32.zip
   - 7z x -bd cabal.zip
   - cabal --version
   - cabal update
+  # Install parsec, text and mtl, also alex and happy
+  - echo "" | appveyor-retry cabal install parsec-3.1.11 text-1.2.2.2 mtl-2.2.1 alex-3.1.7 happy-1.19.5
 
 build_script:
   - cd Cabal
@@ -42,7 +44,6 @@ build_script:
   # - Setup test --show-details=streaming --test-option=--hide-successes
   - cd ..\cabal-install
   - ghc --make -threaded -i -i. Setup.hs -Wall -Werror
-  - echo "" | ..\appveyor-retry ..\cabal install happy
   - echo "" | ..\appveyor-retry ..\cabal install --only-dependencies --enable-tests -flib
   - ..\cabal configure --user --ghc-option=-Werror --enable-tests -flib
   - ..\cabal build

Cabal/Distribution/Parsec/Lexer.x → boot/Lexer.x

@@ -13,11 +13,24 @@
 #ifdef CABAL_PARSEC_DEBUG
 {-# LANGUAGE PatternGuards #-}
 #endif
+{-# OPTIONS_GHC -fno-warn-unused-imports #-}
 module Distribution.Parsec.Lexer
   (ltest, lexToken, Token(..), LToken(..)
   ,bol_section, in_section, in_field_layout, in_field_braces
   ,mkLexState) where
 
+-- [Note: boostrapping parsec parser]
+--
+-- We manually produce the `Lexer.hs` file from `boot/Lexer.x` (make lexer)
+-- because boostrapping cabal-install would be otherwise tricky.
+-- Alex is (atm) tricky package to build, cabal-install has some magic
+-- to move bundled generated files in place, so rather we don't depend
+-- on it before we can build it ourselves.
+-- Therefore there is one thing less to worry in bootstrap.sh, which is a win.
+--
+-- See also https://github.com/haskell/cabal/issues/4633
+--
+
 import Prelude ()
 import qualified Prelude as Prelude
 import Distribution.Compat.Prelude

cabal-install/Distribution/Client/Check.hs

@@ -18,12 +18,7 @@ module Distribution.Client.Check (
 
 import Control.Monad ( when, unless )
 
-#ifdef CABAL_PARSEC
 import Distribution.PackageDescription.Parsec ( readGenericPackageDescription )
-#else
-import Distribution.PackageDescription.Parse ( readGenericPackageDescription )
-#endif
-
 import Distribution.PackageDescription.Check
 import Distribution.PackageDescription.Configuration
          ( flattenPackageDescription )

cabal-install/Distribution/Client/Configure.hs

@@ -65,13 +65,8 @@ import Distribution.Package
 import Distribution.Types.Dependency
          ( Dependency(..), thisPackageVersion )
 import qualified Distribution.PackageDescription as PkgDesc
-#ifdef CABAL_PARSEC
 import Distribution.PackageDescription.Parsec
          ( readGenericPackageDescription )
-#else
-import Distribution.PackageDescription.Parse
-         ( readGenericPackageDescription )
-#endif
 import Distribution.PackageDescription.Configuration
          ( finalizePD )
 import Distribution.Version

cabal-install/Distribution/Client/GenBounds.hs

@@ -29,13 +29,8 @@ import Distribution.PackageDescription
          ( buildDepends )
 import Distribution.PackageDescription.Configuration
          ( finalizePD )
-#ifdef CABAL_PARSEC
 import Distribution.PackageDescription.Parsec
          ( readGenericPackageDescription )
-#else
-import Distribution.PackageDescription.Parse
-         ( readGenericPackageDescription )
-#endif
 import Distribution.Types.ComponentRequestedSpec
          ( defaultComponentRequestedSpec )
 import Distribution.Types.Dependency

cabal-install/Distribution/Client/IndexUtils.hs

@@ -71,19 +71,9 @@ import Distribution.Simple.Utils
 import Distribution.Client.Setup
          ( RepoContext(..) )
 
-#ifdef CABAL_PARSEC
 import Distribution.PackageDescription.Parsec
          ( parseGenericPackageDescriptionMaybe )
 import qualified Distribution.PackageDescription.Parsec as PackageDesc.Parse
-#else
-import Distribution.ParseUtils
-         ( ParseResult(..) )
-import Distribution.PackageDescription.Parse
-         ( parseGenericPackageDescription )
-import Distribution.Simple.Utils
-         ( fromUTF8LBS, ignoreBOM )
-import qualified Distribution.PackageDescription.Parse as PackageDesc.Parse
-#endif
 
 import           Distribution.Solver.Types.PackageIndex (PackageIndex)
 import qualified Distribution.Solver.Types.PackageIndex as PackageIndex
@@ -470,19 +460,11 @@ extractPkg verbosity entry blockNo = case Tar.entryContent entry of
           Just ver -> Just . return $ Just (NormalPackage pkgid descr content blockNo)
             where
               pkgid  = PackageIdentifier (mkPackageName pkgname) ver
-#ifdef CABAL_PARSEC
               parsed = parseGenericPackageDescriptionMaybe (BS.toStrict content)
               descr = case parsed of
                   Just d  -> d
                   Nothing -> error $ "Couldn't read cabal file "
                                     ++ show fileName
-#else
-              parsed = parseGenericPackageDescription . ignoreBOM . fromUTF8LBS $ content
-              descr  = case parsed of
-                ParseOk _ d -> d
-                _           -> error $ "Couldn't read cabal file "
-                                    ++ show fileName
-#endif
           _ -> Nothing
         _ -> Nothing
 
@@ -740,15 +722,9 @@ packageListFromCache verbosity mkPkg hnd Cache{..} mode = accum mempty [] mempty
 
     readPackageDescription :: ByteString -> IO GenericPackageDescription
     readPackageDescription content =
-#ifdef CABAL_PARSEC
       case parseGenericPackageDescriptionMaybe (BS.toStrict content) of
         Just gpd -> return gpd
         Nothing  -> interror "failed to parse .cabal file"
-#else
-      case parseGenericPackageDescription . ignoreBOM . fromUTF8LBS $ content of
-        ParseOk _ d -> return d
-        _           -> interror "failed to parse .cabal file"
-#endif
 
     interror :: String -> IO a
     interror msg = die' verbosity $ "internal error when reading package index: " ++ msg

cabal-install/Distribution/Client/Outdated.hs

@@ -45,13 +45,8 @@ import Distribution.Verbosity                        (Verbosity, silent)
 import Distribution.Version
        (Version, LowerBound(..), UpperBound(..)
        ,asVersionIntervals, majorBoundVersion)
-#ifdef CABAL_PARSEC
 import Distribution.PackageDescription.Parsec
        (readGenericPackageDescription)
-#else
-import Distribution.PackageDescription.Parse
-       (readGenericPackageDescription)
-#endif
 
 import qualified Data.Set as S
 import System.Directory                              (getCurrentDirectory)

cabal-install/Distribution/Client/ProjectConfig.hs

@@ -77,13 +77,8 @@ import Distribution.System
          ( Platform )
 import Distribution.PackageDescription
          ( SourceRepo(..) )
-#if CABAL_PARSEC
 import Distribution.PackageDescription.Parsec
          ( readGenericPackageDescription )
-#else
-import Distribution.PackageDescription.Parse
-         ( readGenericPackageDescription )
-#endif
 import Distribution.Simple.Compiler
          ( Compiler, compilerInfo )
 import Distribution.Simple.Program

cabal-install/Distribution/Client/Sandbox.hs

@@ -80,11 +80,7 @@ import Distribution.Client.Utils              ( inDir, tryCanonicalizePath
                                               , tryFindAddSourcePackageDesc)
 import Distribution.PackageDescription.Configuration
                                               ( flattenPackageDescription )
-#ifdef CABAL_PARSEC
 import Distribution.PackageDescription.Parsec ( readGenericPackageDescription )
-#else
-import Distribution.PackageDescription.Parse  ( readGenericPackageDescription )
-#endif
 import Distribution.Simple.Compiler           ( Compiler(..), PackageDB(..) )
 import Distribution.Simple.Configure          ( configCompilerAuxEx
                                               , getPackageDBContents

cabal-install/Distribution/Client/SetupWrapper.hs

@@ -40,13 +40,8 @@ import Distribution.PackageDescription
          ( GenericPackageDescription(packageDescription)
          , PackageDescription(..), specVersion
          , BuildType(..), knownBuildTypes, defaultRenaming )
-#ifdef CABAL_PARSEC
 import Distribution.PackageDescription.Parsec
          ( readGenericPackageDescription )
-#else
-import Distribution.PackageDescription.Parse
-         ( readGenericPackageDescription )
-#endif
 import Distribution.Simple.Configure
          ( configCompilerEx )
 import Distribution.Compiler

cabal-install/Distribution/Client/SrcDist.hs

@@ -20,13 +20,8 @@ import Distribution.PackageDescription
          ( PackageDescription )
 import Distribution.PackageDescription.Configuration
          ( flattenPackageDescription )
-#ifdef CABAL_PARSEC
 import Distribution.PackageDescription.Parsec
          ( readGenericPackageDescription )
-#else
-import Distribution.PackageDescription.Parse
-         ( readGenericPackageDescription )
-#endif
 import Distribution.Simple.Utils
          ( createDirectoryIfMissingVerbose, defaultPackageDesc
          , warn, die', notice, withTempDirectory )

cabal-install/Distribution/Client/Targets.hs

@@ -91,15 +91,8 @@ import Distribution.Verbosity (Verbosity)
 import Distribution.Simple.Utils
          ( die', warn, lowercase )
 
-#ifdef CABAL_PARSEC
 import Distribution.PackageDescription.Parsec
          ( readGenericPackageDescription, parseGenericPackageDescriptionMaybe )
-#else
-import Distribution.PackageDescription.Parse
-         ( readGenericPackageDescription, parseGenericPackageDescription, ParseResult(..) )
-import Distribution.Simple.Utils
-         ( fromUTF8LBS, ignoreBOM )
-#endif
 
 -- import Data.List ( find, nub )
 import Data.Either
@@ -557,15 +550,8 @@ readPackageTarget verbosity = traverse modifyLocation
           _                 -> False
 
     parsePackageDescription' :: BS.ByteString -> Maybe GenericPackageDescription
-#ifdef CABAL_PARSEC
     parsePackageDescription' bs = 
         parseGenericPackageDescriptionMaybe (BS.toStrict bs)
-#else
-    parsePackageDescription' content =
-      case parseGenericPackageDescription . ignoreBOM . fromUTF8LBS $ content of
-        ParseOk _ pkg -> Just pkg
-        _             -> Nothing
-#endif
 
 -- ------------------------------------------------------------
 -- * Checking package targets

cabal-install/bootstrap.sh

@@ -504,13 +504,15 @@ do_pkg   "deepseq"      ${DEEPSEQ_VER} ${DEEPSEQ_VER_REGEXP}
 do_pkg   "binary"       ${BINARY_VER}  ${BINARY_VER_REGEXP}
 do_pkg   "time"         ${TIME_VER}    ${TIME_VER_REGEXP}
 
-# Install the Cabal library from the local Git clone if possible.
-do_Cabal_pkg
-
+# Cabal might depend on these
 do_pkg   "transformers" ${TRANS_VER}   ${TRANS_VER_REGEXP}
 do_pkg   "mtl"          ${MTL_VER}     ${MTL_VER_REGEXP}
 do_pkg   "text"         ${TEXT_VER}    ${TEXT_VER_REGEXP}
 do_pkg   "parsec"       ${PARSEC_VER}  ${PARSEC_VER_REGEXP}
+
+# Install the Cabal library from the local Git clone if possible.
+do_Cabal_pkg
+
 do_pkg   "network"      ${NETWORK_VER} ${NETWORK_VER_REGEXP}
 
 # We conditionally install network-uri, depending on the network version.