check method

List<MetricCheck> check(Map<String, num> measured, { required String ref, double slack = 0.3, })

Checks measured medians against the recorded goldens of ref: every metric is "lower is better", so the gate is one-sided — measured worse than the golden by more than the slack fails, measured better always passes. Missing goldens (a check before any record) print a warning and do not fail — the first recording on a reference establishes them.

Prints one line per metric and a summary. Returns one MetricCheck per measured metric — regressions are the rows where MetricCheck.regression is true; the number of regressions was the previous return value.

Implementation

List<MetricCheck> check(
  Map<String, num> measured, {
  required String ref,
  double slack = 0.3,
}) {
  final file = File(path);
  final dynamic doc = file.existsSync()
      ? jsonDecode(file.readAsStringSync())
      : <String, Object?>{'metrics': <String, Object?>{}};
  final metrics =
      (doc['metrics'] as Map<String, dynamic>).cast<String, Object?>();

  final rows = <MetricCheck>[];
  final keys = measured.keys.toList()..sort();
  for (final metric in keys) {
    final value = measured[metric]!;
    final metricDoc = metrics[metric] as Map<String, dynamic>?;
    final goldens =
        metricDoc?['goldens'] as Map<String, dynamic>? ?? const {};
    final golden = goldens[ref] ?? goldens['any'];
    final g = golden as num?;
    final limit = g == null
        ? null
        : (g * slack).abs().ceil().clamp(1, 1 << 62);
    rows.add(MetricCheck(
      metric: metric,
      ref: ref,
      measured: value,
      golden: g,
      limit: limit,
      slack: slack,
    ));
    if (g == null) {
      stdout.writeln('  $metric: no golden under ref "$ref" — run '
          '"--record" once on the reference to establish it');
      continue;
    }
    if (value > g + limit!) {
      stderr.writeln('  FAIL $metric: measured $value, golden $g '
          '(worse by more than ${slack * 100}%)');
    } else {
      stdout.writeln('  $metric: $value ≤ golden $g + ${slack * 100}% '
          '(regression gate)');
    }
  }
  final failures = rows.where((r) => r.regression).length;
  if (failures > 0) {
    stderr.writeln('REGRESSION: $failures metric(s) outside the golden '
        'envelope');
  } else {
    stdout.writeln('OK: all measured metrics within their goldens');
  }
  return rows;
}