Commit 68b4e7bb authored by Andreas Klebinger's avatar Andreas Klebinger

Compare expected and actual stdout

parent aba73840
......@@ -160,7 +160,7 @@ main = do
{ shakeThreads = threads
, shakeFiles = output ++ "/"
, shakeReport = [output ++ "/shake_report.html"]
, shakeStaunch = False
, shakeStaunch = True
, shakeVerbosity = Development.Shake.Loud
}
......@@ -421,9 +421,6 @@ buildRules nofib@Build{..} = do
$ Ms.singleton (ml "size") objSize
<> Ms.prefix (ml "rts stats") rtsStats
"//*.src_dir" %> \o -> do
fail $ "Dependency should have been created in Main.exe rule:" ++ o
-- Compute build dependencies
"//.depends" %> \out -> do
-- TODO: Maybe it would be better to look for C dependencies here, than in the main.exe rule
......@@ -543,16 +540,37 @@ runTest nofib@Build{..} runMode resultsTsv = do
-- Construct benchmark invocation
(stdin, args) <- getTestCmdline nofib test
(stdin, args, stdout) <- getTestCmdline nofib test
executable <- liftIO $ IO.canonicalizePath $ output </> src_dir </> "Main" <.> exe
-- Create stats.0, stats.1, etc.
let doRun :: Int -> Action ()
doRun n = do
let rtsStatsOut = executable <.> "stats" <.> show n
cmd_ (Cwd $ src_dir) (EchoStdout False) (StdinBS stdin)
(Exit c, Stderr err, Stdout out) <-
cmd (Cwd $ src_dir) (EchoStdout False) (StdinBS stdin)
(getWrapperArgs runMode $ n)
executable args "+RTS" rts_args "--machine-readable" ("-t"++rtsStatsOut)
unless (c == ExitSuccess) $ do
fail $ "Benchmark failed:" ++ src_dir ++ " exit code:" ++ show c
unless (BSL.null err) $ do
fail $ "Benchmark failed:" ++ src_dir ++ " unexpected stderr:" ++ show err
-- We simply drop '\r' from both expected and actual output when comparing
-- them. This avoids windows line ending issues.
-- Given that checking the output is merely a sanity check
-- that seems reasonable.
when (isJust stdout &&
(BSL.filter (/= '\r') out) /= (BSL.filter (/= '\r') (fromJust stdout))) $ do
liftIO $ do
BSL.writeFile (output </> src_dir </> ".actual" ) out
BSL.writeFile (output </> src_dir </> ".expected" ) $ fromJust stdout
putStrLn "Start of actual output:"
BSL.putStrLn $ BSL.take 800 out
putStrLn "Start of expected output:"
BSL.putStrLn $ BSL.take 800 (fromJust stdout)
fail $ "Benchmark failed:" ++ src_dir ++ " unexpected stdout:^"
return ()
-- Run benchmarks n times
forM_ [1..times] $ \n -> doRun n
......@@ -596,7 +614,8 @@ getModeArgs benchSettings speed = words $
| otherwise
= (benchSettings "NORM_OPTS")
getTestCmdline :: Nofib -> TestName -> Action (BSL.ByteString, [String])
-- | Get stdin, arguments, expected stdout for benchmarks
getTestCmdline :: Nofib -> TestName -> Action (BSL.ByteString, [String], Maybe (BSL.ByteString))
getTestCmdline Build{..} test = do
config <- readConfig' $ output </> src_dir </> "config.txt"
......@@ -607,14 +626,25 @@ getTestCmdline Build{..} test = do
let args = words (config "PROG_ARGS")
++ speed_args
-- Check if there is stdin data, default to ""
stdin_path <-
let s = config "STDIN_FILE"
in if s == ""
then liftIO $ grab "stdin"
else pure $ Just $ testDir test </> s
liftIO $ putStrLn $ "test " <> unTestName test <> " stdin: " <> show stdin_path
stdin <- liftIO $maybe (pure BSL.empty) BSL.readFile stdin_path
return (stdin, args)
stdin <- liftIO $ maybe (pure BSL.empty) BSL.readFile stdin_path
-- Check if there is an stdout file. Default to Nothing
stdout_path <-
let s = config "STDOUT_FILE"
in if null s
then liftIO $ grab "stdout"
else pure $ Just $ testDir test </> s :: Action (Maybe FilePath)
liftIO $ putStrLn $ "test " <> unTestName test <> " stdout: " <> show stdout_path
stdout <- liftIO $ sequence (BSL.readFile <$> stdout_path)
return (stdin, args, stdout)
where
src_dir = testDir test
-- Grab stdin/out
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment