compareBaselines function

BenchmarkComparison compareBaselines(Baseline baseline, Map<String, num> current, { num tolerancePercent = 10, Map<String, bool>? lowerIsBetterOverride, })

Compares current metrics against baseline (FR-010, AC-10, AC-11).

tolerancePercent is the band within which a change counts as stable. Direction semantics come from StandardMetrics.isLowerBetter: latency, memory and CPU regress upward, throughput regresses downward; custom metrics default to lower-is-better. Metrics missing from either side are skipped (reported through the absence from BenchmarkComparison.changes) rather than crashing the comparison.

Implementation

BenchmarkComparison compareBaselines(
  Baseline baseline,
  Map<String, num> current, {
  num tolerancePercent = 10,
  Map<String, bool>? lowerIsBetterOverride,
}) {
  final changes = <String, MetricChange>{};
  var sawRegression = false;
  var sawImprovement = false;

  for (final entry in baseline.metrics.entries) {
    final metric = entry.key;
    final baselineValue = entry.value;
    final currentValue = current[metric];
    if (currentValue == null) continue;

    final absoluteChange = currentValue - baselineValue;
    final percentChange = baselineValue == 0
        ? (currentValue == 0 ? 0 : 100)
        : absoluteChange / baselineValue * 100;
    final magnitude = percentChange.abs();

    final lowerIsBetter =
        lowerIsBetterOverride?[metric] ?? StandardMetrics.isLowerBetter(metric);
    // worsening direction: value moved the wrong way.
    final worsened = lowerIsBetter
        ? currentValue > baselineValue
        : currentValue < baselineValue;

    final isRegression = worsened && magnitude > tolerancePercent;
    final improved = !worsened && magnitude > tolerancePercent;

    MetricDirection direction;
    if (isRegression) {
      direction = MetricDirection.regressed;
    } else if (improved) {
      direction = MetricDirection.improved;
    } else {
      direction = MetricDirection.stable;
    }

    String? severity;
    if (isRegression) {
      severity = magnitude > tolerancePercent * 2 ? 'error' : 'warn';
    }

    changes[metric] = MetricChange(
      metric: metric,
      baselineValue: baselineValue,
      currentValue: currentValue,
      absoluteChange: absoluteChange,
      percentChange: percentChange,
      direction: direction,
      isRegression: isRegression,
      tolerance: tolerancePercent,
      severity: severity,
    );

    if (isRegression) sawRegression = true;
    if (improved) sawImprovement = true;
  }

  final overallStatus = sawRegression
      ? ComparisonStatus.regressed
      : sawImprovement
      ? ComparisonStatus.improved
      : ComparisonStatus.stable;

  return BenchmarkComparison(
    scenarioId: baseline.scenarioId,
    baseline: baseline,
    current: current,
    changes: changes,
    overallStatus: overallStatus,
  );
}