Commit 6e24a0be authored by David Eichmann's avatar David Eichmann 🏋

Skip all performance tests if not in a git repo.

Reviewers: bgamari, tdammers, osa1

Reviewed By: tdammers

Subscribers: osa1, tdammers, rwbarton, carter

GHC Trac Issues: #15923

Differential Revision: https://phabricator.haskell.org/D5367
parent f10df65f
......@@ -20,6 +20,15 @@ from math import ceil, trunc
from testutil import passed, failBecause
# Check if "git status" can be run successfully.
# True implies the current directory is a git repo.
def can_git_status():
try:
subprocess.check_call(['git', 'status'])
return True
except subprocess.CalledProcessError:
return False
#
# Some data access functions. A the moment this uses git notes.
#
......
......@@ -25,7 +25,7 @@ import subprocess
from testutil import getStdout, Watcher, str_warn, str_info
from testglobals import getConfig, ghc_env, getTestRun, TestOptions, brokens
from perf_notes import MetricChange
from perf_notes import MetricChange, can_git_status
from junit import junit
# Readline sometimes spews out ANSI escapes for some values of TERM,
......@@ -84,6 +84,7 @@ if args.rootdir:
config.rootdirs = args.rootdir
config.metrics_file = args.metrics_file
hasMetricsFile = bool(config.metrics_file)
config.summary_file = args.summary_file
config.no_print_summary = args.no_print_summary
......@@ -117,7 +118,12 @@ if args.threads:
if args.verbose is not None:
config.verbose = args.verbose
config.skip_perf_tests = args.skip_perf_tests
# Note force skip perf tests: skip if this is not a git repo (estimated with can_git_status)
# and no metrics file is given. In this case there is no way to read the previous commit's
# perf test results, nor a way to store new perf test results.
canGitStatus = can_git_status()
forceSkipPerfTests = not hasMetricsFile and not canGitStatus
config.skip_perf_tests = args.skip_perf_tests or forceSkipPerfTests
config.only_perf_tests = args.only_perf_tests
if args.test_env:
......@@ -351,12 +357,24 @@ else:
# flush everything before we continue
sys.stdout.flush()
# Warn if had to force skip perf tests (see Note force skip perf tests).
spacing = " "
if forceSkipPerfTests and not args.skip_perf_tests:
print()
print(str_warn('Skipping All Performance Tests') + ' `git status` exited with non-zero exit code.')
print(spacing + 'Git is required because performance test results are compared with the previous git commit\'s results (stored with git notes).')
print(spacing + 'You can still run the tests without git by specifying an output file with --metrics-file FILE.')
# Warn of new metrics.
new_metrics = [metric for (change, metric) in t.metrics if change == MetricChange.NewMetric]
spacing = " "
if any(new_metrics):
if canGitStatus:
reason = 'the previous git commit doesn\'t have recorded metrics for the following tests.' + \
' If the tests exist on the previous commit, then check it out and run the tests to generate the missing metrics.'
else:
reason = 'this is not a git repo so the previous git commit\'s metrics cannot be loaded from git notes:'
print()
print(str_warn('New Metrics') + ' the previous git commit doesn\'t have metrics for the following tests:')
print(str_warn('New Metrics') + ' these metrics trivially pass because ' + reason)
print(spacing + ('\n' + spacing).join(set([metric.test for metric in new_metrics])))
# Inform of how to accept metric changes.
......@@ -369,14 +387,16 @@ else:
summary(t, sys.stdout, config.no_print_summary, True)
# Write perf stats if any exist or if a metrics file is specified.
stats = [stat for (_, stat) in t.metrics]
if config.metrics_file:
if hasMetricsFile:
print('Appending ' + str(len(stats)) + ' stats to file: ' + config.metrics_file)
with open(config.metrics_file, 'a') as file:
file.write("\n" + Perf.format_perf_stat(stats))
else:
elif canGitStatus and any(stats):
Perf.append_perf_stat(stats)
# Write summary
if config.summary_file:
with open(config.summary_file, 'w') as file:
summary(t, file)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment