...
  View open merge request
Commits (46)
......@@ -2,10 +2,10 @@ TOP = ../..
include $(TOP)/mk/boilerplate.mk
ifeq "$(HEAP)" "LARGE"
SRC_RUNTEST_OPTS += +RTS -H16m -RTS
PROG_ARGS += +RTS -H16m -RTS
endif
ifeq "$(HEAP)" "OLD"
SRC_RUNTEST_OPTS += +RTS -H10m -RTS
PROG_ARGS += +RTS -H10m -RTS
endif
include $(TOP)/mk/target.mk
......
......@@ -7,10 +7,10 @@ NORM_OPTS = 8 3000
SLOW_OPTS = 8 5000
ifeq "$(HEAP)" "LARGE"
SRC_RUNTEST_OPTS += +RTS -H256m -RTS
PROG_ARGS += +RTS -H256m -RTS
endif
ifeq "$(HEAP)" "OLD"
SRC_RUNTEST_OPTS += +RTS -H30m -RTS
PROG_ARGS += +RTS -H30m -RTS
endif
include $(TOP)/mk/target.mk
......@@ -7,10 +7,10 @@ NORM_OPTS = 10
SLOW_OPTS = 11
ifeq "$(HEAP)" "LARGE"
SRC_RUNTEST_OPTS += +RTS -H330m -RTS
PROG_ARGS += +RTS -H330m -RTS
endif
ifeq "$(HEAP)" "OLD"
SRC_RUNTEST_OPTS += +RTS -H10m -RTS
PROG_ARGS += +RTS -H10m -RTS
endif
include $(TOP)/mk/target.mk
......@@ -2,15 +2,16 @@ TOP = ../..
include $(TOP)/mk/boilerplate.mk
NORM_OPTS = 300000
SLOW_OPTS = 1000000
SRC_HC_OPTS += -package array
SRC_RUNTEST_OPTS += +RTS -K64m -RTS
PROG_ARGS += +RTS -K64m -RTS
ifeq "$(HEAP)" "LARGE"
SRC_RUNTEST_OPTS += +RTS -H128m -RTS
PROG_ARGS += +RTS -H128m -RTS
endif
ifeq "$(HEAP)" "OLD"
SRC_RUNTEST_OPTS += +RTS -H10m -RTS
PROG_ARGS += +RTS -H10m -RTS
endif
include $(TOP)/mk/target.mk
......@@ -11,10 +11,10 @@ NORM_OPTS = 9
SLOW_OPTS = 9
ifeq "$(HEAP)" "LARGE"
SRC_RUNTEST_OPTS += +RTS -H160m -RTS
PROG_ARGS += +RTS -H160m -RTS
endif
ifeq "$(HEAP)" "OLD"
SRC_RUNTEST_OPTS += +RTS -H10m -RTS
PROG_ARGS += +RTS -H10m -RTS
endif
SRC_RUNTEST_OPTS += -stdout-binary
......
......@@ -13,7 +13,7 @@ NORM_OPTS = 18 500000 4 19
SLOW_OPTS = 19 500000 5 22
ifeq "$(HEAP)" "LARGE"
SRC_RUNTEST_OPTS += +RTS -H180m -RTS
PROG_ARGS += +RTS -H180m -RTS
endif
include $(TOP)/mk/target.mk
......
......@@ -3,14 +3,15 @@ TOP = ../..
include $(TOP)/mk/boilerplate.mk
NORM_OPTS = TestInput.y
SLOW_OPTS = TestInput.y
SRC_HC_OPTS += -cpp -package containers
EXCLUDED_SRCS += TestInput.hs
ifeq "$(HEAP)" "LARGE"
SRC_RUNTEST_OPTS += +RTS -H128m -RTS
PROG_ARGS += +RTS -H128m -RTS
endif
ifeq "$(HEAP)" "OLD"
SRC_RUNTEST_OPTS += +RTS -H10m -RTS
PROG_ARGS += +RTS -H10m -RTS
endif
include $(TOP)/mk/target.mk
......@@ -6,7 +6,7 @@ NORM_OPTS = 5000000
SLOW_OPTS = 100000000
ifeq "$(HEAP)" "LARGE"
SRC_RUNTEST_OPTS += +RTS -H430m -RTS
PROG_ARGS += +RTS -H430m -RTS
endif
include $(TOP)/mk/target.mk
......@@ -6,7 +6,7 @@ NORM_OPTS = 1 2 2000 1000 1001 4000
SLOW_OPTS = 1 2 4000 1000 1001 4000
ifeq "$(HEAP)" "LARGE"
SRC_RUNTEST_OPTS += +RTS -H256m -RTS
PROG_ARGS += +RTS -H256m -RTS
endif
include $(TOP)/mk/target.mk
......@@ -3,12 +3,13 @@ TOP = ../..
include $(TOP)/mk/boilerplate.mk
NORM_OPTS = 14
SLOW_OPTS = 17
ifeq "$(HEAP)" "LARGE"
SRC_RUNTEST_OPTS += +RTS -H8m -RTS
PROG_ARGS += +RTS -H8m -RTS
endif
ifeq "$(HEAP)" "OLD"
SRC_RUNTEST_OPTS += +RTS -H10m -RTS
PROG_ARGS += +RTS -H10m -RTS
endif
include $(TOP)/mk/target.mk
......@@ -8,8 +8,8 @@ NORM_OPTS = 80
SLOW_OPTS = 90
ifeq "$(HEAP)" "LARGE"
SRC_RUNTEST_OPTS += +RTS -H16m -RTS
PROG_ARGS += +RTS -H16m -RTS
endif
ifeq "$(HEAP)" "OLD"
SRC_RUNTEST_OPTS += +RTS -H10m -RTS
PROG_ARGS += +RTS -H10m -RTS
endif
......@@ -9,12 +9,13 @@ input : words
SRC_HC_OPTS += -package containers
NORM_OPTS = words input
SLOW_OPTS = words input
ifeq "$(HEAP)" "LARGE"
SRC_RUNTEST_OPTS += +RTS -H32m -RTS
PROG_ARGS += +RTS -H32m -RTS
endif
ifeq "$(HEAP)" "OLD"
SRC_RUNTEST_OPTS += +RTS -H10m -RTS
PROG_ARGS += +RTS -H10m -RTS
endif
include $(TOP)/mk/target.mk
......@@ -5,10 +5,10 @@ include $(TOP)/mk/boilerplate.mk
PROG_ARGS = 27000.1 27000.2
ifeq "$(HEAP)" "LARGE"
SRC_RUNTEST_OPTS += +RTS -H32m -RTS
PROG_ARGS += +RTS -H32m -RTS
endif
ifeq "$(HEAP)" "OLD"
SRC_RUNTEST_OPTS += +RTS -H24m -RTS
PROG_ARGS += +RTS -H24m -RTS
endif
include $(TOP)/mk/target.mk
......@@ -3,7 +3,7 @@ include $(TOP)/mk/boilerplate.mk
-include opts.mk
# Seems to be a real memory hog, this one
SRC_RUNTEST_OPTS += +RTS -M300m -RTS
PROG_ARGS += +RTS -M300m -RTS
include $(TOP)/mk/target.mk
......
TOP = ../..
include $(TOP)/mk/boilerplate.mk
SRC_RUNTEST_OPTS += 8400
PROG_ARGS += 8400
include $(TOP)/mk/target.mk
......@@ -4,7 +4,7 @@ include $(TOP)/mk/boilerplate.mk
# Override default SRCS; the default is all source files
SRCS=parfact.hs
SRC_RUNTEST_OPTS += 8000000 1000
PROG_ARGS += 8000000 1000
SRC_HC_OPTS += -package concurrent
include $(TOP)/mk/target.mk
......
......@@ -3,7 +3,7 @@ include $(TOP)/mk/boilerplate.mk
# Override default SRCS; the default is all source files
SRCS=Main.hs
SRC_RUNTEST_OPTS += 20
PROG_ARGS += 20
SRC_HC_OPTS += -cpp -package concurrent
include $(TOP)/mk/target.mk
......
......@@ -2,10 +2,10 @@ TOP = ../..
include $(TOP)/mk/boilerplate.mk
# This version just counts the results, and runs in constant space:
# SRC_RUNTEST_OPTS += 7 1163
# PROG_ARGS += 7 1163
# This version builds a list of the results, and needs a lot of memory:
SRC_RUNTEST_OPTS += 3 873
PROG_ARGS += 3 873
SRC_HC_OPTS += -package parallel
......
......@@ -35,6 +35,8 @@ instance Applicative Pure where
instance Monad Pure where
Pure x >>= k = k x
return = Pure
instance MonadFail Pure where
fail s = error s
instance MonadEval Pure where
......@@ -313,6 +315,8 @@ instance Monad Abs where
AbsState r s' -> runAbs (k r) s'
AbsFail m -> AbsFail m)
return = pure
instance MonadFail Abs where
fail s = Abs (\ n -> AbsFail s)
instance MonadEval Abs where
......
......@@ -5,7 +5,7 @@ SRC_HC_OPTS += -cpp -DSTRATEGIES -package random -package parallel
# 28 = version
# 83 = input
PROG_ARGS = 28 83
PROG_ARGS += 28 83
include $(TOP)/mk/target.mk
TOP = ../..
include $(TOP)/mk/boilerplate.mk
SRC_RUNTEST_OPTS += -2.0 -2.0 2.0 2.0 1024 1024 256
PROG_ARGS += -2.0 -2.0 2.0 2.0 1024 1024 256
SRC_RUNTEST_OPTS += -stdout-binary
SRC_HC_OPTS += -package parallel
......
......@@ -7,9 +7,9 @@ SRCS = Board.hs \
Tree.hs \
Wins.hs \
Main.hs
HC_OPTS += -package parallel -package random
SRC_HC_OPTS += -package parallel -package random
PROG_ARGS = 4 6
PROG_ARGS += 4 6
include $(TOP)/mk/target.mk
TOP = ../..
include $(TOP)/mk/boilerplate.mk
PROG_ARGS = 3000
PROG_ARGS += 3000
SRC_HC_OPTS += -package parallel
include $(TOP)/mk/target.mk
......
TOP = ../..
include $(TOP)/mk/boilerplate.mk
SRC_RUNTEST_OPTS += 43 11
PROG_ARGS += 43 11
SRC_HC_OPTS += -package parallel
include $(TOP)/mk/target.mk
TOP = ../..
include $(TOP)/mk/boilerplate.mk
PROG_ARGS = 300 100
PROG_ARGS += 300 100
SRC_HC_OPTS += -package parallel
include $(TOP)/mk/target.mk
......
TOP = ../..
include $(TOP)/mk/boilerplate.mk
PROG_ARGS = 500000
PROG_ARGS += 500000
SRC_HC_OPTS += -package parallel
include $(TOP)/mk/target.mk
......
TOP = ../..
include $(TOP)/mk/boilerplate.mk
SRC_RUNTEST_OPTS += 13
PROG_ARGS += 13
SRC_HC_OPTS += -package parallel
include $(TOP)/mk/target.mk
TOP = ../..
include $(TOP)/mk/boilerplate.mk
PROG_ARGS = 1000
PROG_ARGS += 1000
SRC_HC_OPTS += -package parallel
include $(TOP)/mk/target.mk
......
TOP = ../..
include $(TOP)/mk/boilerplate.mk
PROG_ARGS = 38 8000 100
PROG_ARGS += 38 8000 100
SRC_HC_OPTS += -package parallel
# FAST_OPTS =
......
......@@ -13,7 +13,7 @@ SRC_HC_OPTS += -O0
# 10 is about optimal for 7.1, greater degrades perf (less so for local-gc)
# dummy = 999 (always)
# delay = larger for
PROG_ARGS = 4 10 10 999 24
PROG_ARGS += 4 10 10 999 24
include $(TOP)/mk/target.mk
......@@ -13,7 +13,7 @@
TOP = ../..
include $(TOP)/mk/boilerplate.mk
SRC_RUNTEST_OPTS += 400 400
PROG_ARGS += 400 400
SRC_HC_OPTS += -package parallel
include $(TOP)/mk/target.mk
TOP = ../..
include $(TOP)/mk/boilerplate.mk
SRC_RUNTEST_OPTS += +RTS -K1m -RTS
PROG_ARGS += +RTS -K1m -RTS
include $(TOP)/mk/target.mk
......
{-# LANGUAGE GeneralizedNewtypeDeriving #-}
module CachegrindParse where
import Data.Maybe
import qualified Data.Map as M
newtype EventName = EventName { getEventName :: String }
deriving (Show, Eq, Ord)
parse :: FilePath -> IO (M.Map EventName Integer)
parse fname = parse' <$> readFile fname
parse' :: String -> M.Map EventName Integer
parse' content =
M.fromList $ zip summary events
where
events = case mapMaybe isEventList $ lines content of
[] -> error "No event list found"
[x] -> x
_ -> error "More than one event list found"
summary = case mapMaybe isSummaryList $ lines content of
[] -> error "No event summary found"
[x] -> x
_ -> error "More than one event summary found"
isEventList :: String -> Maybe [Integer]
isEventList line
| "summary:" : rest <- words line = Just $ map read rest
| otherwise = Nothing
isSummaryList :: String -> Maybe [EventName]
isSummaryList line
| "events:" : rest <- words line = Just $ map EventName rest
| otherwise = Nothing
{-# LANGUAGE FlexibleInstances #-}
{-# LANGUAGE OverloadedStrings #-}
import Data.Align
import Data.Maybe
import Data.These
import Data.List (intercalate, group)
import Data.Foldable
import Numeric
import qualified Data.Sequence as Seq
import qualified Data.Tree as Tree
import Data.Tree (Tree)
import qualified Data.Map.Strict as M
import System.FilePath
import Text.Tabular
import Text.Tabular.AsciiArt
import Options.Applicative
import qualified MeasurementTree as MTree
import MeasurementTree (Measurements, Label)
import LabelMatch
data Mode = RelativeMode | AbsoluteMode
args :: Parser (Mode, [FilePath])
args =
(,)
<$> flag RelativeMode AbsoluteMode (long "absolute" <> short 'a' <> help "show absolute metrics")
<*> some (argument str (metavar "FILE" <> help "results.json file"))
-- | Pick out the interesting part of a set of filenames for use as a friendly
-- name. Essentially we just break up all paths into their components and drop
-- all components common to all.
friendlyNameFiles :: [FilePath] -> M.Map String FilePath
friendlyNameFiles files =
M.fromList
[ (joinPath $ toList name, path)
| (path, name) <- M.toList friendlyNames
]
where
friendlyNames :: M.Map FilePath (Seq.Seq String)
friendlyNames =
fmap (fmap snd . Seq.filter (not . fst) . Seq.zip isCommon) splitPaths
where
isCommon :: Seq.Seq Bool
isCommon = Seq.fromList $ fmap isCommonComp [0..maximum $ fmap length splitPaths]
splitPaths :: M.Map FilePath (Seq.Seq String)
splitPaths = M.fromList [ (f, Seq.fromList $ splitPath f) | f <- files ]
isCommonComp :: Int -> Bool
isCommonComp n = allEqual $ M.elems $ fmap (Seq.!? n) splitPaths
allEqual :: Eq a => [a] -> Bool
allEqual xs = length (group xs) == 1
main :: IO ()
main = do
(mode, files) <- execParser $ info (helper <*> args) mempty
let filesWithFriendlyNames = friendlyNameFiles files
-- Yuck. Figure out friendly name of reference set
[ref] = [ name
| (name, fname) <- M.toList filesWithFriendlyNames
, fname == head files
]
mtrees <- mapM (fmap (MTree.toMap head . fromJust) . MTree.readFile) filesWithFriendlyNames
let aligned = alignMany mtrees
let tabulate :: (Show a, Ord a) => String -> (LabelMatcher a) -> IO ()
tabulate heading pred
| null xs = putStrLn $ heading ++ ": no measurements"
| otherwise = do
putStrLn $ unlines
[ heading
, replicate 20 '='
, ""
, case mode of
RelativeMode ->
let showRel = (maybe "-" (showRelative $ showGFloat (Just 3)))
table = toTable $ toRelativeChange ref xs
in render show show showRel table
AbsoluteMode ->
let showVal = (maybe "-" (\x -> showGFloat (Just 3) x ""))
table = toTable xs
in render show show showVal table
, ""
]
where
xs = M.fromList
$ mapMaybe (\(k,v) -> (\k' -> (k',v)) <$> match pred k)
$ M.toList aligned
-- compile-time metrics
tabulate "compiler allocations" $ objectCompilerRtsStats <* "bytes allocated"
tabulate "compiler mutator time" $ objectCompilerRtsStats <* "mutator_cpu_seconds"
tabulate "compiler GC time" $ objectCompilerRtsStats <* "GC_cpu_seconds"
tabulate "executable size" $ testName <* "executable size"
-- run-time metrics
tabulate "bytes allocated" $ runRtsStats <* "bytes allocated"
tabulate "mutator time" $ runRtsStats <* ("mutator_cpu_seconds" <|> "mut_cpu_seconds")
tabulate "GC time" $ runRtsStats <* "GC_cpu_seconds"
-- cachegrind
tabulate "instructions" $ cachegrindStats <* "Ir"
tabulate "LLC cache misses" $ cachegrindStats <* "DLmr"
tabulate "L1 cache misses" $ cachegrindStats <* "D1mr"
alignMany :: (Align f, Ord k) => M.Map k (f a) -> f (M.Map k a)
alignMany mtrees =
foldl1 (alignWith (mergeThese M.union))
[ fmap (M.singleton label) mtree
| (label, mtree) <- M.toList mtrees
]
type TestName = String
type ModuleName = String
testName :: LabelMatcher TestName
testName = wildcard
objectCompilerRtsStats :: LabelMatcher (TestName, ModuleName)
objectCompilerRtsStats = (,) <$> testName <* "objects" <*> wildcard <* "rts stats"
runRtsStats :: LabelMatcher TestName
runRtsStats = testName <* "run" <* "rts stats"
cachegrindStats :: LabelMatcher TestName
cachegrindStats = testName <* "run" <* "cachegrind"
toTable :: (Ord r, Ord c)
=> M.Map r (M.Map c a)
-> Table r c (Maybe a)
toTable xs =
Table
(Group NoLine $ map Header $ M.keys xs)
(Group NoLine $ map Header allKeys)
[ [ M.lookup k ys
| k <- allKeys
]
| ys <- M.elems xs
]
where
allKeys = toList $ foldMap M.keysSet xs
data Relative a = Reference a
| Relative a
| NoReference
showRelative :: (Ord a, RealFloat a, Show a)
=> (a -> ShowS) -> Relative a -> String
showRelative showRef (Reference n) = showRef n ""
showRelative _showRef (Relative n) = plus . showFFloat (Just 2) (100*n) $ "%"
where plus = if n > 0 then showChar '+' else id
showRelative _showRef NoReference = "no ref"
toRelativeChange :: (Ord r, Ord c, RealFrac a)
=> c
-> M.Map r (M.Map c a)
-> M.Map r (M.Map c (Relative a))
toRelativeChange ref xs = fmap f xs
where
f ys
| Just x0 <- M.lookup ref ys =
M.mapWithKey (\k y -> if k == ref then Reference x0 else Relative ((y-x0) / x0)) ys
| otherwise = fmap (const NoReference) ys
Copyright (c) 2019, Ben Gamari
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following
disclaimer in the documentation and/or other materials provided
with the distribution.
* Neither the name of Ben Gamari nor the names of other
contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
{-# LANGUAGE DeriveFunctor #-}
{-# LANGUAGE GADTs #-}
module LabelMatch where
import Control.Applicative
import Control.Monad
import Data.String
import MeasurementTree (Label(..))
import qualified MeasurementTree as MTree
newtype LabelMatcher a = LabelMatcher (Label -> Maybe (Label, a))
deriving (Functor)
match :: LabelMatcher a -> Label -> Maybe a
match (LabelMatcher f) l = snd <$> f l
instance (a ~ ()) => IsString (LabelMatcher a) where
fromString = matchPart
instance Applicative LabelMatcher where
pure x = LabelMatcher $ \lbl -> Just (lbl, x)
(<*>) = ap
instance Alternative LabelMatcher where
LabelMatcher f <|> LabelMatcher g =
LabelMatcher $ \lbl -> f lbl <|> g lbl
empty = LabelMatcher $ const Nothing
instance Monad LabelMatcher where
LabelMatcher f >>= g = LabelMatcher $ \lbl ->
case f lbl of
Nothing -> Nothing
Just (lbl', x) -> let LabelMatcher h = g x
in h lbl'
matchPart :: String -> LabelMatcher ()
matchPart s = LabelMatcher f
where
f (Label (x:xs))
| x == s = Just (Label xs, ())
f _ = Nothing
wildcard :: LabelMatcher String
wildcard = LabelMatcher f
where
f (Label (x:xs)) = Just (Label xs, x)
f (Label []) = Nothing
end :: LabelMatcher ()
end = LabelMatcher f
where
f (Label []) = Just (Label [], ())
f (Label _) = Nothing
{-# LANGUAGE RecordWildCards, DeriveDataTypeable #-}
module Main(main) where
-- Standard libraries
import Control.Concurrent
import Control.Exception
import Control.Monad
import Data.Bifunctor
import Data.Char
import Data.Foldable
import Data.List
import Data.Maybe
import qualified Data.Map.Strict as M
import Data.Time.Clock
import qualified System.Directory as IO
import qualified Data.ByteString.Char8 as BS
import qualified Data.ByteString.Lazy.Char8 as BSL
import System.Exit
import System.Info
import System.IO
import System.Process
import System.Console.CmdArgs
-- Shake - build system
import Development.Shake hiding ((*>))
import Development.Shake.FilePath hiding (exe)
import qualified MeasurementTree as MTree
import MeasurementTree (Measurements, Label(..))
import qualified ParseResults
import qualified CachegrindParse
import qualified PerfStatParse
ml = MTree.mkLabel
---------------------------------------------------------------------
-- TEST CONFIGURATION - which tests are available to run
-- | These are directories that contain tests.
testRoots :: [String]
testRoots = words "imaginary spectral real parallel spectral/hartel gc"
-- | These are tests that are under testRoots, but should be skipped (all are skipped by the Makefile system)
disabledTests :: [String]
disabledTests = words "hartel last-piece secretary triangle ebnf2ps HMMS PolyGP rx cfd dcbm linsolv warshall"
-- | These tests are compiled by the Makefile system, but don't work for me (mostly GHC 7.4 breaks)
newlyDisabledTests :: [String]
newlyDisabledTests = words "power lift fulsom fluid eff"
-- | Directories containing tests that the system can run.
allTests :: IO [FilePath]
allTests = do
xs <- forM testRoots $ \x -> do
ys <- IO.getDirectoryContents x
return [x </> y | y <- ys, '.' `notElem` y, y `notElem` disabledTests, y `notElem` newlyDisabledTests]
fmap sort $ flip filterM (concat xs) $ \x -> do
b <- IO.doesDirectoryExist x
if not b then return False else
IO.doesFileExist $ x </> "Makefile"
---------------------------------------------------------------------
-- ARGUMENT PARSING - mostly based on CmdArgs
data Nofib
= Clean
| Build
{clean :: Bool
,tests :: [String]
,way :: [String]
,threads :: Int
,compiler :: String
,compiler_args :: String
,tag :: String
,output :: String
,cachegrind :: Bool
,cachegrind_args :: String
,perf :: Bool
,perf_args :: String
,run :: Bool
,speed :: Speed
,rts_args :: [String]
,times :: Int
,skip_check :: Bool
}
deriving (Data, Show)
data Speed = Fast | Norm | Slow
deriving (Data, Show)
nofibMode :: Mode (CmdArgs Nofib)
nofibMode = cmdArgsMode $ modes
[Clean
&= help "Clean the build"
,Build
{clean = False &= groupname "Building" &= help "Clean before building"
,tests = [] &= args &= typ "TEST"
,way = [] &= help "Which way to build, defaults to -O1"
,threads = 1 &= name "j" &= typ "NUM" &= help "Number of threads, defaults to 1"
,compiler = "ghc" &= help "Compiler to use, defaults to ghc"
,compiler_args = "" &= help "Extra arguments to pass to the Compiler when building tests"
,tag = "" &= help "Tag to name the compiler, defaults to compiler --version"
,output = "" &= help "Where to put created files under _make, defaults to tag/way"
,cachegrind = False &= groupname "Running" &= help "Run the tests under cachegrind"
,cachegrind_args = "" &= groupname "Running" &= help "Extra arguments to pass to cachegrind"
,perf = False &= groupname "Running" &= help "Run the tests under `perf stat`"
,perf_args = "" &= groupname "Running" &= help "Extra arguments to pass to `perf stat`"
,run = False &= groupname "Running" &= help "Run the tests"
,speed = Norm &= groupname "Running" &= help "Test speed (Fast,Norm,Slow)"
,times = 1 &= help "Number of times to run each test"
,rts_args = [] &= help "Which RTS options to pass when running"
,skip_check = False &= help "Skip checking the results of the tests"
} &= auto &= help "Build"
&= help "Build and run"
]
&= summary "Nofib benchmark suite"
-- | Create a clean set of arguments, with any defaults filled in
nofibArgs :: IO Nofib
nofibArgs = do
args <- cmdArgsRun nofibMode
case args of
Clean -> return args
Build{..} -> do
way <- return $ let xs = concatMap words way in if null xs then ["-O1"] else xs
tag <- if tag == "" then compilerTag compiler else return tag
tests <- resolveTests tests
output <- return $ "_make" </> (if null output then tag </> intercalate "_" way else output)
return Build{..}
-- | Given the tests the user asked for, expand them out, e.g. real is the full real suite.
resolveTests :: [String] -> IO [String]
resolveTests [] = allTests
resolveTests finds = do
let slash1 x = "/" ++ map (\i -> if i == '\\' then '/' else i) x
slash2 x = slash1 x ++ "/"
tests <- allTests
let whole = filter (\test -> any (\find -> slash2 find `isInfixOf` slash2 test) finds) tests -- whole matches
let prefix = filter (\test -> any (\find -> slash1 find `isInfixOf` slash2 test) finds) tests -- prefix matches
let res = if null whole then prefix else whole
when (null res) $
error $ "The targets failed to match any programs: " ++ unwords finds
return res
-- | Find the default compiler string, e.g. ghc-7.4.1
compilerTag :: String -> IO String
compilerTag compiler = do
(_,stdout,_) <- readProcessWithExitCode compiler ["--version"] ""
let ver = takeWhile (\x -> isDigit x || x == '.') $ dropWhile (not . isDigit) stdout
return $ if null ver then "unknown" else ver
---------------------------------------------------------------------
-- MAIN DRIVER
-- | Main program, just interpret the arguments and dispatch the tasks.
main = do
args <- nofibArgs
case args of
Clean -> removeDirectoryRecursive "_make"
Build{..} -> do
when clean $
removeDirectoryRecursive output
putStrLn $ "Running: " ++ unwords tests
shake shakeOptions
{shakeThreads=threads
,shakeFiles=output ++ "/"
,shakeReport=[output ++ "/shake_report.html"]
,shakeStaunch=True
,shakeVerbosity=Development.Shake.Loud} $
buildRules args
putStrLn "Build completed"
when run $ do
ns <- mapM (runTest args) tests
let tot = length ns
bad = length $ filter not ns
t i = if i == 1 then "1 test" else show i ++ " tests"
if bad == 0 then
putStrLn $ "Ran " ++ t tot ++ " successfully"
else
putStrLn $ "WARNING: " ++ t bad ++ " failed, out of " ++ t tot
-- | Rules to build the given tests. For each test, there are three files
-- we care about:
--
-- * config.txt - a cleaned up version of the configuration out of Makefile,
-- created by convertConfig. Also contains "MAIN" which points at the name
-- of the Main module.
--
-- * Main.exe - the actual binary, produced by ghc linking everything.
--
-- * Main.deps - the files that Main.exe depends on, ghc -M.
--
-- * .hi/.o - files produced by ghc -c.
--
-- * .result - the stdout and stderr output from the build
--
-- Most complication comes from modules not named Main, which still produce
-- Main.o object files (I think ghc -M gets these wrong).
buildRules :: Nofib -> Rules ()
buildRules nofib@Build{..} = do
r <- newResource "ghc linker" 1
let unoutput x =
let f x = if hasExtension x then f $ takeDirectory x else x
in f $ takeDirectory $ drop (length output + 1) x
want $ concat
[ [s </> "Main" <.> exe, s </> "config.txt"] | t <- tests, let s = output </> t]
when cachegrind $ want [ "cachegrind" ]
when perf $ want [ "perf" ]
"//all-results" %> \out -> do
let results = [ s </> "Main" <.> exe <.> "result"
| t <- tests
, let s = output </> t
]
need results
xs <- mapM readFileLines results
writeFileLines out (concat xs)
"//perf" %> \out -> do
let results = [ s </> "Main" <.> "perf" <.> "result"
| t <- tests
, let s = output </> t
]
need results
xs <- mapM readFileLines results
writeFileLines out (concat xs)
"//cachegrind" %> \out -> do
let results = [ s </> "Main" <.> "cachegrind" <.> "result"
| t <- tests
, let s = output </> t
]
need results
xs <- mapM readFileLines results
writeFileLines out (concat xs)
"//config.txt" %> \out -> do
let dir = unoutput out
src <- readFileLines $ dir </> "Makefile"
let poss = ["Main.hs","Main.lhs","Era.hs","SumEuler.hs",takeFileName dir <.> "hs",takeFileName dir <.> "lhs"]
bs <- filterM (doesFileExist . (dir </>)) poss
let mainMod = case bs of
[] -> error $ "Could not find Main file for " ++ dir
x:_ -> "MAIN = " ++ x
writeFileLines out $ mainMod : convertConfig src
["//Main" <.> exe, "//Main" <.> exe <.> "result", "//Main.results.json"] &%> \[out, result, resultsJson] -> do
deps <- readFile' $ replaceExtension out "deps"
let os = nub [ if isLower $ head $ takeFileName x then replaceExtension out "o" else output </> x
| x <- words deps, takeExtension x == ".o"]
need os
config <- readConfig' $ takeDirectory out </> "config.txt"
let dir = unoutput out
obj = takeDirectory out
name = takeFileName dir
resultHdl <- liftIO $ openFile result WriteMode
-- Add results from object compilation to results output
objectResults <- forM os $ \o -> do
ls <- liftIO $ BS.readFile (o <.> "result")
liftIO $ BS.hPutStr resultHdl ls
Just ms <- liftIO $ MTree.readFile (o <.> "results.json")
return ms
-- Link executable
liftIO $ hPutStrLn resultHdl $ "==nofib== " ++ name ++ " : time to link " ++ name ++ " follows..."
Stdouterr out_err <- withResource r 1 $
cmd compiler $ ["-Rghc-timing","-rtsopts","-o", out] ++ os ++ way ++ words (config "SRC_HC_OPTS")
liftIO $ BS.hPutStr resultHdl out_err
liftIO $ hPutStrLn resultHdl $ "==nofib== " ++ name ++ ": size of " ++ name ++ " follows..."
-- Report executable size
Stdout out_err <- cmd "size" [out]
let execSize = ParseResults.parseCodeSize $ BS.unpack out_err
liftIO $ BS.hPutStr resultHdl out_err
liftIO $ hClose resultHdl
liftIO $ MTree.writeFile resultsJson
$ fold objectResults
<> MTree.singleton (ml "executable size") (realToFrac execSize)
["//*.o","//*.hi","//*.o.result","//*.o.results.json"] &%> \[o,hi,result,resultsJson] -> do
let dir = unoutput o
obj = output </> dir
config <- readConfig' $ obj </> "config.txt"
let mod = let x = dropExtension $ drop (length obj + 1) o
in if x == "Main" then dropExtension $ config "MAIN" else x
src <- do b <- doesFileExist $ dir </> mod <.> "hs"
return $ dir </> mod <.> (if b then "hs" else "lhs")
deps <- readFileLines $ obj </> "Main.deps"
need [ if takeExtension r `elem` [".h",".hs",".lhs"] then r else output </> r
| lhs:":":rhs <- map words $ deps, dir </> mod <.> "o" == lhs, r <- rhs]
let test = takeFileName dir
resultHdl <- liftIO $ openFile result WriteMode
liftIO $ hPutStrLn resultHdl $ "==nofib== " ++ test ++ " : time to compile " ++ mod ++ " follows..."
let rts_args = [ "+RTS", "--machine-readable", "-t"++o++".stats" ]
Stdouterr out_err <- cmd compiler $ ["-Rghc-timing","-c",src,"-w","-i"++obj,"-odir="++obj,"-hidir="++obj] ++
way ++ words (config "SRC_HC_OPTS") ++ [compiler_args] ++ rts_args
liftIO $ BS.hPutStr resultHdl out_err
liftIO $ hPutStrLn resultHdl $ "==nofib== " ++ test ++ ": size of " ++ takeFileName o ++ " follows..."
Stdout out_err <- cmd "size" [o]
liftIO $ BS.hPutStr resultHdl out_err
rtsStats <- liftIO $ readRtsStats $ o++".stats"
liftIO $ MTree.writeFile resultsJson
$ MTree.prefix (MTree.mkLabel test <> ml "objects" <> MTree.mkLabel (takeFileName o))
$ MTree.singleton (ml "size") (realToFrac $ ParseResults.parseCodeSize $ BS.unpack out_err)
<> MTree.prefix (ml "rts stats") rtsStats
liftIO $ hClose resultHdl
"//Main.deps" %> \out -> do
let dir = unoutput out
config <- readConfig' $ takeDirectory out </> "config.txt"
cmd_ compiler $ ["-w","-M",dir </> config "MAIN","-i" ++ dir,"-dep-makefile=" ++ out, "-dep-suffix", ""] ++
words (config "SRC_HC_OPTS")
src <- liftIO $ readFile out
need [x | x <- words src, takeExtension x `elem` [".hs",".lhs",".h"]]
["//Main.perf.result", "//Main.perf.results.json"] &%> \[out, resultsJson] -> do
need [takeDirectory out </> "config.txt"]
need [replaceExtensions out exe]
let test = unoutput out
(stdin, args) <- liftIO $ getTestCmdline nofib test
executable <- liftIO $ IO.canonicalizePath $ output </> test </> "Main" <.> exe
let rtsStatsOut = executable <.> "stats"
out' <- liftIO $ IO.canonicalizePath out
liftIO $ writeFile out "" -- truncate output
replicateM_ times $ cmd_ (Cwd test) (EchoStdout False) (StdinBS stdin)
"perf" "stat" perf_args "-x," ("--output="++out') "--append" "--"
executable args "+RTS" rts_args "--machine-readable" ("-t"++rtsStatsOut)
stats <- liftIO $ PerfStatParse.readPerfStat out'
liftIO $ MTree.writeFile resultsJson
$ MTree.fromList
[ (MTree.mkLabel test <> ml "run" <> ml "perf" <> lbl, v)
| (eventName, vs) <- M.toList stats
, v <- vs
, let lbl = MTree.mkLabel $ PerfStatParse.getEventName eventName
]
["//Main.cachegrind.result", "//Main.cachegrind.results.json"] &%> \[out, resultsJson] -> do
need [takeDirectory out </> "config.txt"]
need [replaceExtensions out exe]
let test = unoutput out
(stdin, args) <- liftIO $ getTestCmdline nofib test
executable <- liftIO $ IO.canonicalizePath $ output </> test </> "Main" <.> exe
let rtsStatsOut = executable <.> "stats"
out' <- liftIO $ IO.canonicalizePath out
cmd_ (Cwd test) (EchoStdout False) (StdinBS stdin)
"valgrind" "--tool=cachegrind" cachegrind_args ("--cachegrind-out-file="++out')
executable args "+RTS" rts_args "--machine-readable" ("-t"++rtsStatsOut)
stats <- liftIO $ CachegrindParse.parse out'
rtsStats <- liftIO $ readRtsStats rtsStatsOut
liftIO $ MTree.writeFile resultsJson
$ MTree.fromList
[ (MTree.mkLabel test <> ml "run" <> ml "cachegrind" <> lbl, realToFrac v)
| (eventName, v) <- M.toList stats
, let lbl = MTree.mkLabel $ CachegrindParse.getEventName eventName
]
<> MTree.prefix (MTree.mkLabel test <> ml "run" <> ml "rts stats") rtsStats
where
objectsForExecutable :: FilePath -> Action [FilePath]
objectsForExecutable executable = do
deps <- readFile' $ replaceExtension executable "deps"
return $ nub [ if isLower $ head $ takeFileName x then replaceExtension executable "o" else output </> x
| x <- words deps, takeExtension x == ".o"]
getTestCmdline :: Nofib -> String -> IO (BSL.ByteString, [String])
getTestCmdline nofib@Build{..} test = do
config <- readConfig $ output </> test </> "config.txt"
let args = words (config "PROG_ARGS") ++ words (config $ map toUpper (show speed) ++ "_OPTS")
stdin_path <-
let s = config "STDIN_FILE"
in if s == ""
then grab "stdin"
else pure $ Just $ test </> s
putStrLn $ "test " <> test <> " stdin: " <> show stdin_path
stdin <- maybe (pure BSL.empty) BSL.readFile stdin_path
return (stdin, args)
where
grab :: String -> IO (Maybe FilePath)
grab ext = do
let s = [test </> takeFileName test <.> map toLower (show speed) ++ ext
,test </> takeFileName test <.> ext]
ss <- filterM IO.doesFileExist s
return $ listToMaybe ss
readRtsStats :: FilePath -> IO (Measurements Double)
readRtsStats fname = do
rtsStats <- ParseResults.parseRtsStats <$> readFile fname
return $ MTree.fromList $ map (first MTree.mkLabel) $ M.toList rtsStats
-- | Run a test, checking stdout/stderr are as expected, and reporting time.
-- Return True if the test passes.
runTest :: Nofib -> String -> IO Bool
runTest nofib@Build{..} test = do
putStrLn $ "==nofib== " ++ takeDirectory1 test ++ ": time to run " ++ takeDirectory1 test ++ " follows..."
(stdin, args) <- getTestCmdline nofib test
stats <- IO.canonicalizePath $ output </> test </> "stat.txt"
executable <- liftIO $ IO.canonicalizePath $ output </> test </> "Main" <.> exe
fmap and $ replicateM times $ do
start <- getCurrentTime
(code,stdout,stderr) <- readProcessWithExitCodeAndWorkingDirectory
test executable (args++"+RTS":rts_args++["--machine-readable", "-t"++stats]) stdin
end <- getCurrentTime
stdoutWant <- grab "stdout"
stderrWant <- grab "stderr"
BSL.writeFile (output </> test </> "stdout") stdout
BSL.writeFile (output </> test </> "stderr") stderr
putStrLn $ show (floor $ fromRational (toRational $ end `diffUTCTime` start) * 1000) ++ "ms"
putStr =<< readFile stats
rtsStats <- readRtsStats stats
MTree.writeFile (output </> test </> "run.results.json")
$ MTree.prefix (MTree.mkLabel test <> ml "run" <> ml "rts stats") rtsStats
err <- return $
if not skip_check && stderr /= stderrWant then "FAILED STDERR\nWANTED: " ++ snip stderrWant ++ "\nGOT: " ++ snip stderr
else if not skip_check && stdout /= stdoutWant then "FAILED STDOUT\nWANTED: " ++ snip stdoutWant ++ "\nGOT: " ++ snip stdout
else if not skip_check && code /= ExitSuccess then "FAILED EXIT CODE " ++ show code
else ""
if null err then return True else putStrLn err >> return False
where
snip :: BSL.ByteString -> String
snip x = if BSL.length x > 200 then BSL.unpack (BSL.take 200 x) ++ "..." else BSL.unpack x
grab :: String -> IO BSL.ByteString
grab ext = do
let s = [test </> takeFileName test <.> map toLower (show speed) ++ ext
,test </> takeFileName test <.> ext]
ss <- filterM IO.doesFileExist s
maybe (return BSL.empty) BSL.readFile $ listToMaybe ss
---------------------------------------------------------------------
-- CONFIGURATION UTILITIES
-- The Makefile's are slurped for configuration, to produce a cleaned-up config file
-- | Given the source of a Makefile, slurp out the configuration strings.
convertConfig :: [String] -> [String]
convertConfig xs =
[ k ++ " = " ++ v
| (k, v) <- M.toList vars
]
where
vars =
M.fromListWith (\a b -> a ++ " " ++ b)
[ (remap a, b)
| x <- xs
, let (a,b) = separate x, a `elem` keep
]
keep = words "PROG_ARGS SRC_HC_OPTS SLOW_OPTS NORM_OPTS FAST_OPTS STDIN_FILE"
remap x = x
separate x = (name,rest)
where (name,x2) = span (\x -> isAlpha x || x == '_') x
rest = dropWhile isSpace $ dropWhile (`elem` "+=") $ dropWhile isSpace x2
-- | Read a configuration file (new format) into a function supplying options.
readConfig :: FilePath -> IO (String -> String)
readConfig x = do
src <- readFile x
let res = [ (reverse $ dropWhile isSpace $ reverse a, dropWhile isSpace $ drop 1 b)
| y <- lines src, let (a,b) = break (== '=') y]
return $ \x -> fromMaybe "" $ lookup x res
-- | readConfig lifted into the Action monad.
readConfig' :: FilePath -> Action (String -> String)
readConfig' x = do
need [x]
liftIO $ readConfig x
---------------------------------------------------------------------
-- GENERAL UTILITIES
-- | The executable extension on this platform.
exe :: String
exe = if os == "mingw32" then "exe" else ""
-- | Like the standard removeDirectoryRecursive, but doesn't fail if the path is missing.
removeDirectoryRecursive :: FilePath -> IO ()
removeDirectoryRecursive x = do
b <- IO.doesDirectoryExist x
when b $ IO.removeDirectoryRecursive x
-- | Source for readProcessWithExitCode, plus addition of cwd
readProcessWithExitCodeAndWorkingDirectory
:: FilePath -- ^ directory to use
-> FilePath -- ^ command to run
-> [String] -- ^ any arguments
-> BSL.ByteString -- ^ standard input
-> IO (ExitCode, BSL.ByteString, BSL.ByteString) -- ^ exitcode, stdout, stderr
readProcessWithExitCodeAndWorkingDirectory cwd cmd args input = do
(Just inh, Just outh, Just errh, pid) <-
createProcess (proc cmd args){ cwd = Just cwd,
std_in = CreatePipe,
std_out = CreatePipe,
std_err = CreatePipe }
outMVar <- newEmptyMVar
out <- BSL.hGetContents outh
_ <- forkIO $ evaluate (BSL.length out) >> putMVar outMVar ()
err <- BSL.hGetContents errh
_ <- forkIO $ evaluate (BSL.length err) >> putMVar outMVar ()
when (not (BSL.null input)) $ do BSL.hPutStr inh input; hFlush inh
hClose inh
takeMVar outMVar
takeMVar outMVar
hClose outh
hClose errh
ex <- waitForProcess pid
return (ex, out, err)
import Data.Foldable
import Options.Applicative
import qualified MeasurementTree as MTree
import MeasurementTree (Measurements, Label)
args :: Parser (FilePath, [FilePath])
args =
(,)
<$> option str (long "output" <> short 'o' <> help "output file")
<*> some (argument str (metavar "FILE" <> help "results.json files"))
main :: IO ()
main = do
(output, files) <- execParser $ info (helper <*> args) mempty
trees <- mapM (fmap (maybe (error "uh oh") id) . MTree.readFile) files
MTree.writeFile output (fold trees)
module ParseResults
( parseCodeSize
, parseRtsStats
) where
import qualified Data.Map.Strict as M
import Data.Maybe
-- | Parse text section size out of Berkley-format @size@ output
parseCodeSize :: String -> Integer
parseCodeSize content
| header : sizes : _ <- lines content
, "text" : _ <- words header =
read $ head $ words sizes
| otherwise = error "unrecognized size output"
parseRtsStats :: String -> M.Map String Double
parseRtsStats = foldMap parseValue . readPairs . dropFirstLine
where
parseValue (name, value)
| (x, ""):_ <- reads value = M.singleton name x
| otherwise = M.empty
dropFirstLine = unlines . drop 1 . lines
readPairs :: String -> [(String, String)]
readPairs s
| (x, _):_ <- reads s = x
| otherwise = error $ "Failed to parse RTS statistics: " ++ s
{-# LANGUAGE OverloadedStrings #-}
module PerfStatParse (readPerfStat, EventName(..)) where
import qualified Data.Text as T
import qualified Data.Text.IO as TIO
import qualified Data.Map as M
newtype EventName = EventName { getEventName :: String }
deriving (Show, Eq, Ord)
-- | Read metrics from @perf stat -x,@ invocation.
readPerfStat :: FilePath -> IO (M.Map EventName [Double])
readPerfStat path = do
parsePerfStat <$> TIO.readFile path
parsePerfStat :: T.Text -> M.Map EventName [Double]
parsePerfStat = M.unionsWith (++) . map parseLine . T.lines
where
parseLine line
| "#" `T.isPrefixOf` line = mempty
| value:_:metric:_ <- T.splitOn "," line =
M.singleton (EventName $ T.unpack metric) [read $ T.unpack value]
| otherwise = mempty
import Distribution.Simple
main = defaultMain
cabal-version: >=1.10
name: nofib
version: 0.1.0.0
-- synopsis:
-- description:
-- bug-reports:
license: BSD3
license-file: LICENSE
author: Ben Gamari
maintainer: ben@smart-cactus.org
-- copyright:
-- category:
build-type: Simple
library
exposed-modules: MeasurementTree
hs-source-dirs: src
build-depends: base >=4.10 && <4.13,
aeson,
these >= 0.8,
containers,
bytestring
default-language: Haskell2010
executable nofib-run
main-is: Main.hs
other-modules: CachegrindParse, PerfStatParse, ParseResults
other-extensions: RecordWildCards, DeriveDataTypeable
build-depends: base >=4.10 && <4.13,
time >=1.8 && <1.9,
containers,
bytestring,
directory >=1.3 && <1.4,
process >=1.6 && <1.7,
text,
cmdargs,
shake,
nofib
default-language: Haskell2010
executable nofib-compare
main-is: Compare.hs
other-modules: LabelMatch
other-extensions: RecordWildCards, DeriveDataTypeable
build-depends: base >=4.10 && <4.13,
these,
containers,
filepath,
aeson,
ansi-wl-pprint,
tabular,
these,
semialign,
optparse-applicative,
nofib
default-language: Haskell2010
executable nofib-merge
main-is: Merge.hs
build-depends: base >=4.10 && <4.13,
optparse-applicative,
nofib
default-language: Haskell2010