stream method

@override Stream<String> stream(List<ChatMessage> messages, { String? system, int? maxTokens, })

Streams the response to messages as text deltas.

Implementation

@override
Stream<String> stream(
  List<ChatMessage> messages, {
  String? system,
  int? maxTokens,
}) async* {
  var finished = false;
  final events = _http.postSse(
    joinUrl(baseUrl, 'chat/completions'),
    _auth(apiKey, headers),
    _body(messages, system, maxTokens, stream: true),
  );
  await for (final e in events) {
    if (e.data == '[DONE]') {
      finished = true;
      break;
    }
    final Object? json;
    try {
      json = jsonDecode(e.data);
    } on FormatException {
      continue;
    }
    if (json is! Map) continue;
    final error = json['error'];
    if (error is Map) {
      throw LlmHttpException(
        500,
        '${error['message'] ?? 'stream error'}',
        errorType: error['type'] as String?,
      );
    }
    final choices = json['choices'];
    if (choices is! List || choices.isEmpty) continue;
    final delta = (choices.first as Map)['delta'];
    if (delta is! Map) continue;
    final refusal = delta['refusal'];
    if (refusal is String && refusal.isNotEmpty) {
      throw LlmRefusalException(refusal);
    }
    // A finish reason is the other way servers end a stream; not every
    // OpenAI-compatible server sends `[DONE]`.
    if ((choices.first as Map)['finish_reason'] != null) finished = true;
    final text = delta['content'];
    if (text is String && text.isNotEmpty) yield text;
  }
  if (!finished) {
    throw const LlmException(
      'the response stream ended before the model finished; '
      'discard the partial text and retry',
    );
  }
}