Commit 54d3a1fd authored by Ben Gamari's avatar Ben Gamari Committed by Ben Gamari

testsuite: Produce JUnit output

Test Plan: Validate, try ingesting into Jenkins.

Reviewers: austin

Subscribers: rwbarton, thomie

GHC Trac Issues: #13716

Differential Revision: https://phabricator.haskell.org/D3796
parent 5e940bd3
......@@ -171,6 +171,7 @@ _darcs/
/rts/package.conf.install.raw
/stage3.package.conf
/testsuite_summary*.txt
/testsuite*.xml
/testlog*
/utils/mkUserGuidePart/mkUserGuidePart.cabal
/utils/runghc/runghc.cabal
......
from datetime import datetime
import xml.etree.ElementTree as ET
def junit(t):
testsuites = ET.Element('testsuites')
testsuite = ET.SubElement(testsuites, 'testsuite',
id = "0",
package = 'ghc',
tests = str(t.total_tests),
failures = str(len(t.unexpected_failures) + len(t.unexpected_stat_failures)),
errors = str(len(t.framework_failures)),
timestamp = datetime.now().isoformat())
for result, group in [('stat failure', t.unexpected_stat_failures),
('unexpected failure', t.unexpected_failures)]:
for (directory, testname, reason, way) in group:
testcase = ET.SubElement(testsuite, 'testcase',
classname = testname,
name = way)
result = ET.SubElement(testcase, 'failure',
type = result,
message = reason)
for (directory, testname, reason, way) in t.framework_failures:
testcase = ET.SubElement(testsuite, 'testcase',
classname = testname,
name = way)
result = ET.SubElement(testcase, 'error',
type = "framework failure",
message = reason)
for (directory, testname, way) in t.expected_passes:
testcase = ET.SubElement(testsuite, 'testcase',
classname = testname,
name = way)
return ET.ElementTree(testsuites)
......@@ -26,6 +26,7 @@ import subprocess
from testutil import *
from testglobals import *
from junit import junit
# Readline sometimes spews out ANSI escapes for some values of TERM,
# which result in test failures. Thus set TERM to a nice, simple, safe
......@@ -57,6 +58,7 @@ parser.add_argument("--threads", type=int, help="threads to run simultaneously")
parser.add_argument("--check-files-written", help="check files aren't written by multiple tests") # NOTE: This doesn't seem to exist?
parser.add_argument("--verbose", type=int, choices=[0,1,2,3,4,5], help="verbose (Values 0 through 5 accepted)")
parser.add_argument("--skip-perf-tests", action="store_true", help="skip performance tests")
parser.add_argument("--junit", type=argparse.FileType('wb'), help="output testsuite summary in JUnit format")
args = parser.parse_args()
......@@ -310,6 +312,9 @@ else:
with open(config.summary_file, 'w') as file:
summary(t, file)
if args.junit:
junit(t).write(args.junit)
cleanup_and_exit(0)
# Note [Running tests in /tmp]
......
......@@ -140,6 +140,7 @@ class TestRun:
self.framework_failures = []
self.framework_warnings = []
self.expected_passes = []
self.unexpected_passes = []
self.unexpected_failures = []
self.unexpected_stat_failures = []
......
......@@ -863,6 +863,7 @@ def do_test(name, way, func, args, files):
if passFail == 'pass':
if _expect_pass(way):
t.expected_passes.append((directory, name, way))
t.n_expected_passes += 1
else:
if_verbose(1, '*** unexpected pass for %s' % full_name)
......
......@@ -246,6 +246,10 @@ RUNTEST_OPTS += \
RUNTEST_OPTS += -e "config.stage=$(GhcStage)"
ifneq "$(JUNIT_FILE)" ""
RUNTEST_OPTS += \
--junit "$(JUNIT_FILE)"
endif
ifneq "$(SUMMARY_FILE)" ""
RUNTEST_OPTS += \
--summary-file "$(SUMMARY_FILE)"
......
......@@ -296,6 +296,7 @@ rm -f testsuite_summary.txt testsuite_summary_stage1.txt
$make -C testsuite/tests $BINDIST $PYTHON_ARG \
$MAKE_TEST_TARGET stage=2 LOCAL=0 $TEST_VERBOSITY THREADS=$threads \
NO_PRINT_SUMMARY=YES SUMMARY_FILE=../../testsuite_summary.txt \
JUNIT_FILE=../../testsuite.xml \
2>&1 | tee testlog
# Run a few tests using the stage1 compiler.
......@@ -304,6 +305,7 @@ $make -C testsuite/tests $BINDIST $PYTHON_ARG \
$make -C testsuite/tests/stage1 $PYTHON_ARG \
$MAKE_TEST_TARGET stage=1 LOCAL=0 $TEST_VERBOSITY THREADS=$threads \
NO_PRINT_SUMMARY=YES SUMMARY_FILE=../../../testsuite_summary_stage1.txt \
JUNIT_FILE=../../../testsuite_stage1.xml \
2>&1 | tee testlog-stage1
echo
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment