stream method

@override Stream<String> stream(List<ChatMessage> messages, { String? system, int? maxTokens, })

Streams the response to messages as text deltas.

Implementation

@override
Stream<String> stream(
  List<ChatMessage> messages, {
  String? system,
  int? maxTokens,
}) async* {
  final key = _render(messages, system);
  final cached = await cache.lookup(key);
  if (cached != null) {
    yield cached;
    return;
  }
  final text = StringBuffer();
  var complete = false;
  try {
    await for (final delta in inner.stream(
      messages,
      system: system,
      maxTokens: maxTokens,
    )) {
      text.write(delta);
      yield delta;
    }
    complete = true;
  } finally {
    // Only a stream that ran to completion is worth replaying: a truncated
    // or failed one would be served to every similar prompt from now on.
    if (complete) await cache.put(key, text.toString());
  }
}