Commit 5f06b61c authored by David Eichmann's avatar David Eichmann 🏋

CI: Always dump performance metrics.

parent 241921a0
Pipeline #11520 passed with stages
in 421 minutes and 24 seconds
......@@ -272,7 +272,7 @@ hadrian-ghc-in-ghci:
make binary-dist TAR_COMP_OPTS="-1"
- |
# Prepare to push git notes.
METRICS_FILE=$(mktemp)
METRICS_FILE=$CI_PROJECT_DIR/performance-metrics.tsv
git config user.email "ben+ghc-ci@smart-cactus.org"
git config user.name "GHC GitLab CI"
- |
......@@ -289,6 +289,7 @@ hadrian-ghc-in-ghci:
paths:
- ghc-*.tar.xz
- junit.xml
- performance-metrics.tsv
#################################
# x86_64-darwin
......
......@@ -43,8 +43,3 @@ GitHash = NewType("GitHash", str)
GitRef = NewType("GitRef", str)
TestEnv = NewType("TestEnv", str)
MetricName = NewType("MetricName", str)
\ No newline at end of file
MetricBaselineOracle = Callable[[WayName, GitHash], Optional[float]]
MetricDeviationOracle = Callable[[WayName, GitHash], Optional[float]]
MetricOracles = NamedTuple("MetricOracles", [("baseline", MetricBaselineOracle),
("deviation", MetricDeviationOracle)])
......@@ -81,6 +81,11 @@ AllowedPerfChange = NamedTuple('AllowedPerfChange',
('opts', Dict[str, str])
])
MetricBaselineOracle = Callable[[WayName, GitHash], Baseline]
MetricDeviationOracle = Callable[[WayName, GitHash], Optional[float]]
MetricOracles = NamedTuple("MetricOracles", [("baseline", MetricBaselineOracle),
("deviation", MetricDeviationOracle)])
def parse_perf_stat(stat_str: str) -> PerfStat:
field_vals = stat_str.strip('\t').split('\t')
return PerfStat(*field_vals) # type: ignore
......@@ -183,7 +188,7 @@ def parse_allowed_perf_changes(commitMsg: str
# Calculates a suggested string to append to the git commit in order to accept the
# given changes.
# changes: [(MetricChange, PerfStat)]
def allow_changes_string(changes: List[Tuple[MetricChange, PerfStat]]
def allow_changes_string(changes: List[Tuple[MetricChange, PerfStat, Any]]
) -> str:
Dec = MetricChange.Decrease
Inc = MetricChange.Increase
......@@ -193,7 +198,7 @@ def allow_changes_string(changes: List[Tuple[MetricChange, PerfStat]]
# Map tests to a map from change direction to metrics.
test_to_dir_to_metrics = {} # type: Dict[TestName, Dict[MetricChange, List[MetricName]]]
for (change, perf_stat) in changes:
for (change, perf_stat, _baseline) in changes:
change_dir_to_metrics = test_to_dir_to_metrics.setdefault(perf_stat.test, { Inc: [], Dec: [] })
change_dir_to_metrics[change].append(perf_stat.metric)
......@@ -246,12 +251,12 @@ def allow_changes_string(changes: List[Tuple[MetricChange, PerfStat]]
return '\n\n'.join(msgs)
# Formats a list of metrics into a string. Used e.g. to save metrics to a file or git note.
def format_perf_stat(stats: Union[PerfStat, List[PerfStat]]) -> str:
def format_perf_stat(stats: Union[PerfStat, List[PerfStat]], delimitor: str = "\t") -> str:
# If a single stat, convert to a singleton list.
if not isinstance(stats, list):
stats = [stats]
return "\n".join(["\t".join([str(stat_val) for stat_val in stat]) for stat in stats])
return "\n".join([delimitor.join([str(stat_val) for stat_val in stat]) for stat in stats])
# Appends a list of metrics to the git note of the given commit.
# Tries up to max_tries times to write to git notes should it fail for some reason.
......@@ -514,7 +519,8 @@ def get_commit_metric(gitNoteRef,
# force_print: Print stats even if the test stat was in the tolerance range.
# Returns a (MetricChange, pass/fail object) tuple. Passes if the stats are withing the expected value ranges.
def check_stats_change(actual: PerfStat,
baseline, tolerance_dev,
baseline: Baseline,
tolerance_dev,
allowed_perf_changes: Dict[TestName, List[AllowedPerfChange]] = {},
force_print = False
) -> Tuple[MetricChange, Any]:
......
......@@ -25,7 +25,7 @@ import subprocess
from testutil import getStdout, Watcher, str_warn, str_info
from testglobals import getConfig, ghc_env, getTestRun, TestConfig, TestOptions, brokens
from perf_notes import MetricChange, inside_git_repo, is_worktree_dirty
from perf_notes import MetricChange, inside_git_repo, is_worktree_dirty, format_perf_stat
from junit import junit
import cpu_features
......@@ -392,6 +392,22 @@ else:
# flush everything before we continue
sys.stdout.flush()
# Dump metrics data.
if any(t.metrics):
print("\nPerformance Metrics:\n")
for (change, stat, baseline) in t.metrics:
if baseline is None:
print("{stat} [{direction}]".format(
stat = format_perf_stat(stat),
direction = str(change)))
else:
print("{stat} [{direction} from ({baselineStat}) @ HEAD~{depth}]".format(
stat = format_perf_stat(stat),
baselineStat = format_perf_stat(baseline.perfStat, ", "),
direction = str(change),
depth = baseline.commitDepth))
print("")
# Warn if had to force skip perf tests (see Note force skip perf tests).
spacing = " "
if forceSkipPerfTests and not args.skip_perf_tests:
......@@ -401,7 +417,7 @@ else:
print(spacing + 'You can still run the tests without git by specifying an output file with --metrics-file FILE.')
# Warn of new metrics.
new_metrics = [metric for (change, metric) in t.metrics if change == MetricChange.NewMetric]
new_metrics = [metric for (change, metric, baseline) in t.metrics if change == MetricChange.NewMetric]
if any(new_metrics):
if inside_git_repo():
reason = 'a baseline (expected value) cannot be recovered from' + \
......@@ -441,7 +457,7 @@ else:
summary(t, sys.stdout, config.no_print_summary, config.supports_colors)
# Write perf stats if any exist or if a metrics file is specified.
stats = [stat for (_, stat) in t.metrics]
stats = [stat for (_, stat, __) in t.metrics]
if hasMetricsFile:
print('Appending ' + str(len(stats)) + ' stats to file: ' + config.metrics_file)
with open(config.metrics_file, 'a') as f:
......
......@@ -4,7 +4,7 @@
from my_typing import *
from pathlib import Path
from perf_notes import MetricChange, PerfStat
from perf_notes import MetricChange, PerfStat, Baseline, MetricOracles
from datetime import datetime
# -----------------------------------------------------------------------------
......@@ -243,7 +243,7 @@ class TestRun:
# [(change, PerfStat)] where change is one of the MetricChange
# constants: NewMetric, NoChange, Increase, Decrease.
# NewMetric happens when the previous git commit has no metric recorded.
self.metrics = [] # type: List[Tuple[MetricChange, PerfStat]]
self.metrics = [] # type: List[Tuple[MetricChange, PerfStat, Optional[Baseline]]]
global t
t = TestRun()
......
......@@ -26,7 +26,7 @@ from testutil import strip_quotes, lndir, link_or_copy_file, passed, \
import testutil
from cpu_features import have_cpu_feature
import perf_notes as Perf
from perf_notes import MetricChange
from perf_notes import MetricChange, PerfStat, MetricOracles
extra_src_files = {'T4198': ['exitminus1.c']} # TODO: See #12223
from my_typing import *
......@@ -1291,7 +1291,7 @@ def static_stats( name, way, stats_file ):
opts = getTestOpts()
return check_stats(name, way, in_statsdir(stats_file), opts.stats_range_fields)
def metric_dict(name, way, metric, value):
def metric_dict(name, way, metric, value) -> PerfStat:
return Perf.PerfStat(
test_env = config.test_env,
test = name,
......@@ -1354,7 +1354,7 @@ def check_stats(name: TestName,
tolerance_dev,
config.allowed_perf_changes,
config.verbose >= 4)
t.metrics.append((change, perf_stat))
t.metrics.append((change, perf_stat, baseline))
# If any metric fails then the test fails.
# Note, the remaining metrics are still run so that
......
......@@ -96,7 +96,7 @@ else:
os.symlink(str(src), str(dst))
class Watcher(object):
def __init__(self, count: int):
def __init__(self, count: int) -> None:
self.pool = count
self.evt = threading.Event()
self.sync_lock = threading.Lock()
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment