diff --git a/.docker/validate-8.8.1.dockerfile b/.docker/validate-8.8.1.dockerfile index ca1fae605c35b4577308e655bc0424c8b5602c97..0a797ae4a3b19c2295ccb1fdc7c84f7dfe18b7ee 100644 --- a/.docker/validate-8.8.1.dockerfile +++ b/.docker/validate-8.8.1.dockerfile @@ -49,4 +49,4 @@ RUN cabal v2-install -w ghc-8.8.1 --lib \ # Validate WORKDIR /build COPY . /build -RUN sh ./validate.sh -w ghc-8.8.1 -v -D +RUN sh ./validate.sh -w ghc-8.8.1 -v -D -b diff --git a/cabal.project.validate b/cabal.project.validate index a11193fc85354235cc61adae0bddc6790cf1b5b5..81f99f296c99e2c7ff2cc386a0063758c4046d74 100644 --- a/cabal.project.validate +++ b/cabal.project.validate @@ -1,4 +1,4 @@ -packages: Cabal/ cabal-testsuite/ cabal-install/ +packages: Cabal/ cabal-testsuite/ cabal-install/ solver-benchmarks/ write-ghc-environment-files: never diff --git a/solver-benchmarks/HackageBenchmark.hs b/solver-benchmarks/HackageBenchmark.hs index b014e6b8f46011fe819101a8fbef4fa80bd52c4e..ef5d9efa5989fb81e95aee9811364bd5b1d02646 100644 --- a/solver-benchmarks/HackageBenchmark.hs +++ b/solver-benchmarks/HackageBenchmark.hs @@ -27,7 +27,9 @@ import Statistics.Test.MannWhitneyU ( PositionTest(..), TestResult(..) , mannWhitneyUCriticalValue , mannWhitneyUtest) import Statistics.Types (PValue, mkPValue) +import System.Directory (getTemporaryDirectory) import System.Exit (ExitCode(..), exitFailure) +import System.FilePath ((</>)) import System.IO ( BufferMode(LineBuffering), hPutStrLn, hSetBuffering, stderr , stdout) import System.Process ( StdStream(CreatePipe), CreateProcess(..), callProcess @@ -173,6 +175,8 @@ hackageBenchmarkMain = do runCabal :: Int -> FilePath -> [String] -> PackageName -> IO CabalTrial runCabal timeoutSeconds cabal flags pkg = do ((exitCode, err), time) <- timeEvent $ do + tmpDir <- getTemporaryDirectory + let timeout = "timeout --foreground -sINT " ++ show timeoutSeconds cabalCmd = unwords $ [ cabal @@ -180,7 +184,7 @@ runCabal timeoutSeconds cabal flags pkg = do -- A non-existent store directory prevents cabal from reading the -- store, which would cause the size of the store to affect run -- time. - , "--store-dir=non-existent-store-dir" + , "--store-dir=" ++ (tmpDir </> "non-existent-store-dir") , "v2-install" diff --git a/solver-benchmarks/solver-benchmarks.cabal b/solver-benchmarks/solver-benchmarks.cabal index ab28af0ae75442c0955fb87c95f9cc6385bd9d1d..2046a0b614a95fcb5428a2de630940fbaccf87d2 100644 --- a/solver-benchmarks/solver-benchmarks.cabal +++ b/solver-benchmarks/solver-benchmarks.cabal @@ -30,6 +30,8 @@ library base, bytestring, Cabal >= 2.3, + directory, + filepath, optparse-applicative, process, time, diff --git a/validate.sh b/validate.sh index 2c2e3804b1182f499fc00c7d65e3df9d405e200d..1ec1ff18c1f1fca26742c1fcbf094c8c04c3e585 100755 --- a/validate.sh +++ b/validate.sh @@ -14,6 +14,7 @@ CABALSUITETESTS=true CABALONLY=false DEPSONLY=false DOCTEST=false +BENCHMARKS=false VERBOSE=false # Help @@ -23,7 +24,7 @@ show_usage() { cat <<EOF ./validate.sh - build & test -Usage: ./validate.sh [ -j JOBS | -l | -C | -c | -s | -w HC | -x CABAL | -y CABALPLAN | -d | -D | -v ] +Usage: ./validate.sh [ -j JOBS | -l | -C | -c | -s | -w HC | -x CABAL | -y CABALPLAN | -d | -D | -b | -v ] A script which runs all the tests. Available options: @@ -37,6 +38,7 @@ Available options: -y CABALPLAN With cabal-plan -d Build dependencies only -D Run doctest + -b Run benchmarks (quick run, verify they work) -v Verbose EOF exit 0 @@ -112,7 +114,7 @@ footer() { # getopt ####################################################################### -while getopts 'j:lCcsw:x:y:dDv' flag; do +while getopts 'j:lCcsw:x:y:dDbv' flag; do case $flag in j) JOBS="$OPTARG" ;; @@ -134,6 +136,8 @@ while getopts 'j:lCcsw:x:y:dDv' flag; do ;; D) DOCTEST=true ;; + b) BENCHMARKS=true + ;; v) VERBOSE=true ;; ?) show_usage @@ -170,6 +174,7 @@ cabal-testsuite: $CABALSUITETESTS library only: $CABALONLY dependencies only: $DEPSONLY doctest: $DOCTEST +benchmarks: $BENCHMARKS verbose: $VERBOSE EOF @@ -343,6 +348,27 @@ CMD="$($CABALPLANLISTBIN cabal-testsuite:exe:cabal-tests) --builddir=$CABAL_TEST fi # CABALSUITETESTS +# solver-benchmarks +####################################################################### + +if $BENCHMARKS; then +echo "$CYAN=== solver-benchmarks: build =========================== $(date +%T) === $RESET" + +timed $CABALNEWBUILD solver-benchmarks:hackage-benchmark solver-benchmarks:unit-tests --enable-tests + +echo "$CYAN=== solver-benchmarks: test ============================ $(date +%T) === $RESET" + +CMD="$($CABALPLANLISTBIN solver-benchmarks:test:unit-tests)" +(cd Cabal && timed $CMD) || exit 1 + +echo "$CYAN=== solver-benchmarks: run ============================= $(date +%T) === $RESET" + +SOLVEPKG=Chart-diagrams +CMD="$($CABALPLANLISTBIN solver-benchmarks:exe:hackage-benchmark) --cabal1=$CABAL --cabal2=$($CABALPLANLISTBIN cabal-install:exe:cabal) --trials=5 --packages=$SOLVEPKG --print-trials" +(cd Cabal && timed $CMD) || exit 1 + +fi + # END #######################################################################