grade method

  1. @override
Future<Score> grade({
  1. required Trial trial,
  2. required Transcript transcript,
  3. required Outcome outcome,
  4. required EvalContext context,
  5. ReferenceSolution? referenceSolution,
})
override

Compute a score for one trial.

Implementation

@override
Future<Score> grade({
  required Trial trial,
  required Transcript transcript,
  required Outcome outcome,
  required EvalContext context,
  ReferenceSolution? referenceSolution,
}) async {
  await queue.enqueue(
    trial: trial,
    transcript: transcript,
    outcome: outcome,
    rubric: rubric,
  );
  final existing = await queue.fetchVerdict(trial);
  if (existing != null) return existing;
  return Score(
    graderName: name,
    value: null,
    passed: null,
    rationale: 'pending human review',
  );
}