grade method

  1. @override
Future<Score> grade({
  1. required Trial trial,
  2. required Transcript transcript,
  3. required Outcome outcome,
  4. required EvalContext context,
  5. ReferenceSolution? referenceSolution,
})
override

Compute a score for one trial.

Implementation

@override
Future<Score> grade({
  required Trial trial,
  required Transcript transcript,
  required Outcome outcome,
  required EvalContext context,
  ReferenceSolution? referenceSolution,
}) async {
  final assertions = await computeAssertions(
    trial: trial,
    transcript: transcript,
    outcome: outcome,
    context: context,
    referenceSolution: referenceSolution,
  );
  final passedCount = assertions.where((a) => a.passed).length;
  final value = assertions.isEmpty ? 0.0 : passedCount / assertions.length;
  return Score(
    graderName: name,
    value: value,
    passed: value >= passThreshold,
    assertions: assertions,
    rationale: buildRationale(assertions),
  );
}