Commit c867876a authored by Andreas Klebinger's avatar Andreas Klebinger

Factor out common parts of running benchmarks

parent e5930f81
......@@ -319,61 +319,92 @@ buildRules nofib@Build{..} = do
-- Run tests under perf stat
["//Main.perf.result", "//Main.perf.results.tsv"] &%> \[out, resultsTsv] -> do
need [takeDirectory out </> "config.txt"]
need [replaceExtensions out exe]
let test = unoutput out
(stdin, args) <- liftIO $ getTestCmdline nofib test
executable <- liftIO $ IO.canonicalizePath $ output </> testDir test </> "Main" <.> exe
let rtsStatsOut = executable <.> "stats"
out' <- liftIO $ IO.canonicalizePath out
liftIO $ writeFile out "" -- truncate output
replicateM_ times $ cmd_ (Cwd $ testDir test) (EchoStdout False) (StdinBS stdin)
"perf" "stat" perf_args "-x," ("--output="++out') "--append" "--"
executable args "+RTS" rts_args "--machine-readable" ("-t"++rtsStatsOut)
stats <- liftIO $ PerfStatParse.readPerfStat out'
liftIO $ Ms.writeFileTsv resultsTsv
$ Ms.fromList
[ (testLabel test <> ml "run" <> ml "perf" <> lbl, v)
| (eventName, vs) <- M.toList stats
, v <- vs
, let lbl = Ms.mkLabel $ PerfStatParse.getEventName eventName
]
let test = testFromResultTsv nofib resultsTsv
let args = ["perf", "stat"] <> perf_args <> ["-x,", ("--output="++out'), "--append", "--"]
let parse_perf = do
stats <- PerfStatParse.readPerfStat out'
return $ Ms.fromList
[ (testLabel test <> ml "run" <> ml "perf" <> lbl, v)
| (eventName, vs) <- M.toList stats
, v <- vs
, let lbl = Ms.mkLabel $ PerfStatParse.getEventName eventName
]
runTest nofib (ModeWrapped args parse_perf) resultsTsv
-- Run tests normally
["//Main.run.results.tsv"] &%> \[resultsTsv] -> do
need [takeDirectory resultsTsv </> "config.txt"]
need [replaceExtensions resultsTsv exe]
let test = unoutput resultsTsv :: TestName
(stdin, args) <- liftIO $ getTestCmdline nofib test
executable <- liftIO $ IO.canonicalizePath $ output </> testDir test </> "Main" <.> exe
let rtsStatsOut = executable <.> "stats"
cmd_ (Cwd $ testDir test) (EchoStdout False) (StdinBS stdin)
executable args "+RTS" rts_args "--machine-readable" ("-t"++rtsStatsOut)
rtsStats <- liftIO $ readRtsStats rtsStatsOut
liftIO $ Ms.writeFileTsv resultsTsv
$ Ms.prefix (testLabel test <> ml "run" <> ml "rts stats") rtsStats
runTest nofib ModeRun resultsTsv
-- Run tests under cachegrind
["//Main.cachegrind.result", "//Main.cachegrind.results.tsv"] &%> \[out, resultsTsv] -> do
need [takeDirectory out </> "config.txt"]
need [replaceExtensions out exe]
let test = unoutput out
out' <- liftIO $ IO.canonicalizePath out
let test = testFromResultTsv nofib resultsTsv
let wrapper_args = ["valgrind", "--tool=cachegrind"] <> cachegrind_args <>
[("--cachegrind-out-file="++out)]
let parse_cachegrind = do
stats <- CachegrindParse.parse out'
return $ Ms.fromList
[ (testLabel test <> ml "run" <> ml "cachegrind" <> lbl, realToFrac v)
| (eventName, v) <- M.toList stats
, let lbl = Ms.mkLabel $ CachegrindParse.getEventName eventName
]
runTest nofib (ModeWrapped wrapper_args parse_cachegrind) resultsTsv
data RunMode = ModeRun -- ^ Regular runtime measurement
| ModeWrapped [String] (IO (Measurements Double))
-- ^ Wrap the executable by a call to another executable like
-- perf or valgrind.
getWrapperArgs :: RunMode -> [String]
getWrapperArgs ModeRun = []
getWrapperArgs (ModeWrapped args _) = args
getWrapperParser :: RunMode -> IO (Measurements Double)
getWrapperParser ModeRun = return mempty
getWrapperParser (ModeWrapped _ parser) = parser
---------------------------------------------------------------------
-- RULES
-- | "foo/results.tsv" => TestName foo
testFromResultTsv :: Nofib -> String -> TestName
testFromResultTsv Build{..}=
let f path
| hasExtension path = f (takeDirectory path)
| otherwise = path
in TestName . f . takeDirectory . drop (length output + 1)
runTest :: Nofib
-> RunMode
-> String
-> Action ()
runTest nofib@Build{..} runMode resultsTsv = do
-- Build executable first
need [takeDirectory resultsTsv </> "config.txt"]
need [replaceExtensions resultsTsv exe]
let test = testFromResultTsv nofib resultsTsv :: TestName
-- Construct benchmark invocation
(stdin, args) <- liftIO $ getTestCmdline nofib test
executable <- liftIO $ IO.canonicalizePath $ output </> testDir test </> "Main" <.> exe
let rtsStatsOut = executable <.> "stats"
out' <- liftIO $ IO.canonicalizePath out
-- Benchmark arguments
exec_args :: [String]
exec_args = [executable] <> args <> ["+RTS"] <> rts_args <> ["--machine-readable", ("-t"++rtsStatsOut)]
cmd_ (Cwd $ testDir test) (EchoStdout False) (StdinBS stdin)
"valgrind" "--tool=cachegrind" cachegrind_args ("--cachegrind-out-file="++out')
executable args "+RTS" rts_args "--machine-readable" ("-t"++rtsStatsOut)
stats <- liftIO $ CachegrindParse.parse out'
(getWrapperArgs runMode) exec_args
wrapper_measurements <- liftIO (getWrapperParser runMode)
rtsStats <- liftIO $ readRtsStats rtsStatsOut
liftIO $ Ms.writeFileTsv resultsTsv
$ Ms.fromList
[ (testLabel test <> ml "run" <> ml "cachegrind" <> lbl, realToFrac v)
| (eventName, v) <- M.toList stats
, let lbl = Ms.mkLabel $ CachegrindParse.getEventName eventName
]
<> Ms.prefix (testLabel test <> ml "run" <> ml "rts stats") rtsStats
liftIO $ Ms.writeFileTsv resultsTsv $
wrapper_measurements <>
(Ms.prefix (testLabel test <> ml "run" <> ml "rts stats") rtsStats)
getTestConfig :: Nofib -> TestName -> Action (String -> String)
getTestConfig Build{..} test =
......
......@@ -45,7 +45,7 @@ data Nofib
,threads :: Int
,compiler :: String
,compiler_args :: [String]
,output :: String
,output :: String -- ^ Where to put the results
,cachegrind :: Bool
,cachegrind_args :: [String]
,perf :: Bool
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment