Add solver-benchmarks to validate.sh

.docker/validate-8.8.1.dockerfile

@@ -49,4 +49,4 @@ RUN     cabal v2-install -w ghc-8.8.1 --lib \
 # Validate
 WORKDIR /build
 COPY    . /build
-RUN     sh ./validate.sh -w ghc-8.8.1 -v -D
+RUN     sh ./validate.sh -w ghc-8.8.1 -v -D -b

cabal.project.validate

-packages: Cabal/ cabal-testsuite/ cabal-install/
+packages: Cabal/ cabal-testsuite/ cabal-install/ solver-benchmarks/
 
 write-ghc-environment-files: never
 

solver-benchmarks/HackageBenchmark.hs

@@ -27,7 +27,9 @@ import Statistics.Test.MannWhitneyU ( PositionTest(..), TestResult(..)
                                     , mannWhitneyUCriticalValue
                                     , mannWhitneyUtest)
 import Statistics.Types (PValue, mkPValue)
+import System.Directory (getTemporaryDirectory)
 import System.Exit (ExitCode(..), exitFailure)
+import System.FilePath ((</>))
 import System.IO ( BufferMode(LineBuffering), hPutStrLn, hSetBuffering, stderr
                  , stdout)
 import System.Process ( StdStream(CreatePipe), CreateProcess(..), callProcess
@@ -173,6 +175,8 @@ hackageBenchmarkMain = do
 runCabal :: Int -> FilePath -> [String] -> PackageName -> IO CabalTrial
 runCabal timeoutSeconds cabal flags pkg = do
   ((exitCode, err), time) <- timeEvent $ do
+    tmpDir <- getTemporaryDirectory
+
     let timeout = "timeout --foreground -sINT " ++ show timeoutSeconds
         cabalCmd = unwords $
             [ cabal
@@ -180,7 +184,7 @@ runCabal timeoutSeconds cabal flags pkg = do
               -- A non-existent store directory prevents cabal from reading the
               -- store, which would cause the size of the store to affect run
               -- time.
-            , "--store-dir=non-existent-store-dir"
+            , "--store-dir=" ++ (tmpDir </> "non-existent-store-dir")
 
             , "v2-install"
 

solver-benchmarks/solver-benchmarks.cabal

@@ -30,6 +30,8 @@ library
     base,
     bytestring,
     Cabal >= 2.3,
+    directory,
+    filepath,
     optparse-applicative,
     process,
     time,

validate.sh

@@ -14,6 +14,7 @@ CABALSUITETESTS=true
 CABALONLY=false
 DEPSONLY=false
 DOCTEST=false
+BENCHMARKS=false
 VERBOSE=false
 
 # Help
@@ -23,7 +24,7 @@ show_usage() {
 cat <<EOF
 ./validate.sh - build & test
 
-Usage: ./validate.sh [ -j JOBS | -l | -C | -c | -s | -w HC | -x CABAL | -y CABALPLAN | -d | -D | -v ]
+Usage: ./validate.sh [ -j JOBS | -l | -C | -c | -s | -w HC | -x CABAL | -y CABALPLAN | -d | -D | -b | -v ]
   A script which runs all the tests.
 
 Available options:
@@ -37,6 +38,7 @@ Available options:
   -y CABALPLAN   With cabal-plan
   -d             Build dependencies only
   -D             Run doctest
+  -b             Run benchmarks (quick run, verify they work)
   -v             Verbose
 EOF
 exit 0
@@ -112,7 +114,7 @@ footer() {
 # getopt
 #######################################################################
 
-while getopts 'j:lCcsw:x:y:dDv' flag; do
+while getopts 'j:lCcsw:x:y:dDbv' flag; do
     case $flag in
         j) JOBS="$OPTARG"
             ;;
@@ -134,6 +136,8 @@ while getopts 'j:lCcsw:x:y:dDv' flag; do
             ;;
         D) DOCTEST=true
             ;;
+        b) BENCHMARKS=true
+            ;;
         v) VERBOSE=true
             ;;
         ?) show_usage
@@ -170,6 +174,7 @@ cabal-testsuite:     $CABALSUITETESTS
 library only:        $CABALONLY
 dependencies only:   $DEPSONLY
 doctest:             $DOCTEST
+benchmarks:          $BENCHMARKS
 verbose:             $VERBOSE
 
 EOF
@@ -343,6 +348,27 @@ CMD="$($CABALPLANLISTBIN cabal-testsuite:exe:cabal-tests) --builddir=$CABAL_TEST
 
 fi # CABALSUITETESTS
 
+# solver-benchmarks
+#######################################################################
+
+if $BENCHMARKS; then
+echo "$CYAN=== solver-benchmarks: build =========================== $(date +%T) === $RESET"
+
+timed $CABALNEWBUILD solver-benchmarks:hackage-benchmark solver-benchmarks:unit-tests --enable-tests
+
+echo "$CYAN=== solver-benchmarks: test ============================ $(date +%T) === $RESET"
+
+CMD="$($CABALPLANLISTBIN solver-benchmarks:test:unit-tests)"
+(cd Cabal && timed $CMD) || exit 1
+
+echo "$CYAN=== solver-benchmarks: run ============================= $(date +%T) === $RESET"
+
+SOLVEPKG=Chart-diagrams
+CMD="$($CABALPLANLISTBIN solver-benchmarks:exe:hackage-benchmark) --cabal1=$CABAL --cabal2=$($CABALPLANLISTBIN cabal-install:exe:cabal) --trials=5 --packages=$SOLVEPKG --print-trials"
+(cd Cabal && timed $CMD) || exit 1
+
+fi
+
 # END
 #######################################################################