Commit d0063e89 authored by thomie's avatar thomie

Make validate more quiet

  * By default use V=0, and call the testsuite with VERBOSE=2, which we
    did before only with validate --quiet. This disables printing the
    test commands it runs.

  * When --quiet is used, call the testsuite with VERBOSE=1. This
    disables printing the '====> Scanning' lines, and doesn't print
    which test is being run. So it only prints something when a test
    accidentally prints to stdout or when it fails.

    Don't set this option on Travis, as Travis will cancel a build if it
    doesn't see any output for more than 10 minutes.

  * When --quiet is used, set the new test option NO_PRINT_SUMMARY,
    which skips printing the test summary. Only the list of unexpected
    failures is printed, if there are any. Note that the full summary
    can still be found in testsuite_summary.txt

  * When --quiet is used, don't pass the `-v` flag to `ghc-pkg check`

  * When --quiet is used, don't print the Oops! header. It shoud be
    clear from the list of failing tests that something is wrong.

This is all done to get the most out of 30 lines of logfile. These changes can
be disabled later by simply not passing the --quiet flag to validate.

Differential Revision: https://phabricator.haskell.org/D942
parent 5dd02864
......@@ -34,5 +34,6 @@ script:
- echo 'DYNAMIC_GHC_PROGRAMS = NO' >> mk/validate.mk
- echo 'GhcLibWays = v' >> mk/validate.mk
- if [ "$DEBUG_STAGE" = "YES" ]; then echo 'GhcStage2HcOpts += -DDEBUG' >> mk/validate.mk; fi
# use --quiet, otherwise we hit log file limits on travis.
- CPUS=2 SKIP_PERF_TESTS=YES ./validate --fast --quiet
# Don't use --quiet, as it might cause the testsuite to not print output for
# over 10 minutes, causing Travis to kill our job.
- CPUS=2 SKIP_PERF_TESTS=YES ./validate --fast
......@@ -156,8 +156,8 @@ endif
.PHONY: fasttest
fasttest:
$(MAKE) -C testsuite/tests CLEANUP=1 OUTPUT_SUMMARY=../../testsuite_summary.txt fast
$(MAKE) -C testsuite/tests CLEANUP=1 SUMMARY_FILE=../../testsuite_summary.txt fast
.PHONY: fulltest test
fulltest test:
$(MAKE) -C testsuite/tests CLEANUP=1 OUTPUT_SUMMARY=../../testsuite_summary.txt
$(MAKE) -C testsuite/tests CLEANUP=1 SUMMARY_FILE=../../testsuite_summary.txt
......@@ -43,7 +43,8 @@ long_options = [
"configfile=", # config file
"config=", # config field
"rootdir=", # root of tree containing tests (default: .)
"output-summary=", # file in which to save the (human-readable) summary
"summary-file=", # file in which to save the (human-readable) summary
"no-print-summary=", # should we print the summary?
"only=", # just this test (can be give multiple --only= flags)
"way=", # just this way
"skipway=", # skip this way
......@@ -71,8 +72,11 @@ for opt,arg in opts:
if opt == '--rootdir':
config.rootdirs.append(arg)
if opt == '--output-summary':
config.output_summary = arg
if opt == '--summary-file':
config.summary_file = arg
if opt == '--no-print-summary':
config.no_print_summary = True
if opt == '--only':
config.only.append(arg)
......@@ -306,10 +310,10 @@ else:
break
oneTest()
summary(t, sys.stdout)
summary(t, sys.stdout, config.no_print_summary)
if config.output_summary != '':
summary(t, open(config.output_summary, 'w'))
if config.summary_file != '':
summary(t, open(config.summary_file, 'w'))
sys.exit(0)
......@@ -29,7 +29,16 @@ class TestConfig:
self.accept = 0
# File in which to save the summary
self.output_summary = ''
self.summary_file = ''
# Should we print the summary?
# Disabling this is useful for Phabricator/Harbormaster
# logfiles, which are truncated to 30 lines. TODO. Revise if
# this is still true.
# Note that we have a separate flag for this, instead of
# overloading --verbose, as you might want to see the summary
# with --verbose=0.
self.no_print_summary = False
# File in which to save the times
self.times_file = ''
......
......@@ -2186,10 +2186,15 @@ def findTFiles_(path):
# -----------------------------------------------------------------------------
# Output a test summary to the specified file object
def summary(t, file):
def summary(t, file, short=False):
file.write('\n')
printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures, t.unexpected_stat_failures])
if short:
# Only print the list of unexpected tests above.
return
file.write('OVERALL SUMMARY for test run started at '
+ time.strftime("%c %Z", t.start_time) + '\n'
+ str(datetime.timedelta(seconds=
......
......@@ -223,9 +223,13 @@ RUNTEST_OPTS += \
--config 'gs=$(call quote_path,$(GS))' \
--config 'timeout_prog=$(call quote_path,$(TIMEOUT_PROGRAM))'
ifneq "$(OUTPUT_SUMMARY)" ""
ifneq "$(SUMMARY_FILE)" ""
RUNTEST_OPTS += \
--output-summary "$(OUTPUT_SUMMARY)"
--summary-file "$(SUMMARY_FILE)"
endif
ifeq "$(NO_PRINT_SUMMARY)" "YES"
RUNTEST_OPTS += \
--no-print-summary 1
endif
RUNTEST_OPTS += \
......
......@@ -83,14 +83,21 @@ do
done
check_packages () {
echo "== Start $1 package check"
if [ "$bindistdir" = "" ]
then
inplace/bin/ghc-pkg check -v
ghc_pkg=inplace/bin/ghc-pkg
else
"$bindistdir"/bin/ghc-pkg check -v
ghc_pkg="$bindistdir"/bin/ghc-pkg
fi
if [ $be_quiet -eq 1 ]
then
"$ghc_pkg" check
else
echo "== Start $1 package check"
"$ghc_pkg" check -v
echo "== End $1 package check"
fi
echo "== End $1 package check"
}
detect_cpu_count () {
......@@ -133,17 +140,9 @@ fi
if type gmake > /dev/null 2> /dev/null
then
if [ $be_quiet -eq 1 ]; then
make="gmake -s"
else
make="gmake"
fi
make="gmake -s"
else
if [ $be_quiet -eq 1 ]; then
make="make -s"
else
make="make"
fi
make="make -s"
fi
if [ $testsuite_only -eq 0 ]; then
......@@ -170,10 +169,7 @@ thisdir=`utils/ghc-pwd/dist-boot/ghc-pwd`
echo "Validating=YES" > mk/are-validating.mk
echo "ValidateSpeed=$speed" >> mk/are-validating.mk
echo "ValidateHpc=$hpc" >> mk/are-validating.mk
if [ $be_quiet -eq 1 ]; then
echo "V=0" >> mk/are-validating.mk # Less gunk
fi
echo "V=0" >> mk/are-validating.mk # Less gunk
if [ $use_dph -eq 1 ]; then
echo "BUILD_DPH=YES" >> mk/are-validating.mk
......@@ -238,12 +234,13 @@ FAST)
;;
esac
verbosity=3
if [ $be_quiet -eq 1 ]; then
verbosity=2
TEST_VERBOSITY="VERBOSE=1 NO_PRINT_SUMMARY=YES"
else
TEST_VERBOSITY="VERBOSE=2"
fi
$make $MAKE_TEST_TARGET stage=2 $BINDIST VERBOSE=$verbosity THREADS=$threads 2>&1 | tee testlog
$make $MAKE_TEST_TARGET stage=2 $BINDIST $TEST_VERBOSITY THREADS=$threads 2>&1 | tee testlog
check_packages post-testsuite
......@@ -252,16 +249,16 @@ then
utils/hpc/hpc markup --hpcdir=. --srcdir=compiler --srcdir=testsuite/hpc_output --destdir=testsuite/hpc_output testsuite/hpc_output/ghc.tix
fi
echo "-------------------------------------------------------------------"
if
grep '\<0 caused framework failures' testlog >/dev/null 2>/dev/null &&
grep '\<0 unexpected passes' testlog >/dev/null 2>/dev/null &&
grep '\<0 unexpected failures' testlog >/dev/null 2>/dev/null &&
grep '\<0 unexpected stat failures' testlog >/dev/null 2>/dev/null &&
! grep 'Some files are written by multiple tests' testlog >/dev/null 2>/dev/null ; then
grep '\<0 caused framework failures' testsuite_summary.txt >/dev/null 2>/dev/null &&
grep '\<0 unexpected passes' testsuite_summary.txt >/dev/null 2>/dev/null &&
grep '\<0 unexpected failures' testsuite_summary.txt >/dev/null 2>/dev/null &&
grep '\<0 unexpected stat failures' testsuite_summary.txt >/dev/null 2>/dev/null &&
! grep 'Some files are written by multiple tests' testsuite_summary.txt >/dev/null 2>/dev/null ; then
if [ $testsuite_only -eq 0 ] && [ $no_clean -eq 0 ]
then
cat <<EOF
-------------------------------------------------------------------
Congratulations! This tree has passed minimal testing.
NOTE: If you have made changes that may cause failures not tested for by
......@@ -283,6 +280,7 @@ EOF
EOF
else
cat <<EOF
-------------------------------------------------------------------
I didn't find any problems, but this wasn't a complete validate run,
so be careful!
......@@ -292,10 +290,14 @@ the minimal testing procedure, please do further testing as necessary.
EOF
fi
else
cat <<EOF
if [ $be_quiet -eq 0 ]
then
cat <<EOF
-------------------------------------------------------------------
Oops! Looks like you have some unexpected test results or framework failures.
Please fix them before pushing/sending patches.
-------------------------------------------------------------------
EOF
fi
exit 1
fi
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment