generateChatResponseAsync method
Streams this turn's response token-by-token.
INVARIANT (critical for every consumer): each yielded FunctionCallResponse
/ ParallelFunctionCallResponse has ALREADY been committed to the
persistent chat history (_fullHistory + _modelHistory) at the moment it
is yielded — committed before the yield so a tool-response feeds in AFTER
it in history order. Because the call is committed on yield, any consumer
that collects these calls MUST answer every one (via addQueryChunk with a
Message.toolResponse) on EVERY exit path — normal completion,
cancellation, a tool throwing, AND a mid-stream stream error — or the
committed call is left dangling with no response and poisons the next turn
on this reused chat. generateChatResponseWithTools is the reference
implementation that balances all four paths; new consumers should prefer it
over re-driving this stream directly.
Implementation
Stream<ModelResponse> generateChatResponseAsync() async* {
gemmaLog('InferenceChat: Starting async stream generation');
final buffer = StringBuffer();
// Smart function handling mode - continuous scanning for JSON patterns
String funcBuffer = '';
gemmaLog('InferenceChat: Starting to iterate over native tokens...');
// Track if we emitted a function call (to record correct history and skip session clearing)
bool emittedFunctionCall = false;
// SDK-passthrough tool-call suppression (currently Gemma 4; keyed off the
// format, not the model — see FunctionCallParser.usesSdkPassthrough). The
// C++ runtime streams a tool-call turn as the raw
// `{"role":"assistant","tool_calls":[...]}` JSON (one or more concatenated
// objects) AND exposes it via lastRawResponse. The passthrough format reports
// "no function call in text", so the funcBuffer below never suppresses those
// tokens — without this they leak into the text channel (agent
// TextChunkEvent / voice synthesis). We classify the turn on its first
// non-whitespace char: a '{' means a tool-call JSON turn, which we swallow
// (the structured call is surfaced from lastRawResponse at end-of-stream);
// anything else is plain text and streams normally. Other formats are
// excluded by the guard below, so this first-char probe only ever runs for
// the passthrough format — it never misfires on a text-stream format's
// JSON-ish output. Invariant assumed: a passthrough turn is EITHER all
// tool-call JSON OR all text, never prose-then-JSON — Gemma 4's SDK emits a
// pure tool_calls object for a call, so the first-char classification holds.
final bool sdkPassthrough =
FunctionCallParser.usesSdkPassthrough(modelType) &&
tools.isNotEmpty &&
supportsFunctionCalls &&
toolChoice != ToolChoice.none &&
session is RawSdkResponseSession;
bool sdkClassified = false; // has this turn been classified yet?
bool sdkSwallow = false; // swallowing a tool-call JSON stream?
String sdkProbe = ''; // pre-classification token accumulator
final sdkSwallowed =
StringBuffer(); // swallowed JSON (fallback if unparsed)
final originalStream = session.getResponseAsync().map(
(token) => TextResponse(token),
);
// Apply thinking filter for models that may generate <think> tags.
// enable_thinking=false is passed via extraContext for .litertlm but is not
// reliable for all model bundles — keep filter as safety net.
final bool modelCanThink =
modelType == ModelType.deepSeek ||
modelType == ModelType.qwen ||
modelType == ModelType.qwen3 ||
modelType == ModelType.gemmaIt;
final Stream<ModelResponse> filteredStream = (isThinking || modelCanThink)
? ModelThinkingFilter.filterThinkingStream(
originalStream,
modelType: modelType,
)
: originalStream;
// If user didn't request thinking, discard ThinkingResponse events
final Stream<ModelResponse> thinkingHandledStream = isThinking
? filteredStream
: filteredStream.where((r) => r is! ThinkingResponse);
// Apply stop token filter for .litertlm on iOS (MediaPipe doesn't handle stop tokens)
final Stream<ModelResponse> stopFilteredStream =
StopTokenFilter.filterStopTokens(
thinkingHandledStream,
fileType: fileType,
);
await for (final response in stopFilteredStream) {
if (response is TextResponse) {
final token = response.token;
if (kDebugMode) {
gemmaLog(
'InferenceChat: Received filtered token: "$token"',
level: GemmaLogLevel.verbose,
);
}
// Gemma 4 SDK-passthrough classification (see `sdkPassthrough` above).
// Runs before the generic funcBuffer scanning, which is a no-op for the
// passthrough format anyway (it never detects a call in the text stream).
if (sdkPassthrough) {
if (sdkSwallow) {
// Inside a tool-call JSON stream — swallow every token.
sdkSwallowed.write(token);
continue;
}
if (!sdkClassified) {
sdkProbe += token;
final trimmed = sdkProbe.trimLeft();
if (trimmed.isEmpty) continue; // whitespace only — keep probing
sdkClassified = true;
if (trimmed.startsWith('{')) {
// Tool-call JSON turn — swallow this and all following tokens.
sdkSwallow = true;
sdkSwallowed.write(sdkProbe);
continue;
}
// Plain-text turn — emit the probed prefix, then stream the rest.
yield TextResponse(sdkProbe);
buffer.write(sdkProbe);
continue;
}
// Already classified as plain text — stream token-by-token.
yield response;
buffer.write(token);
continue;
}
// Track if this token should be added to buffer (default true)
bool shouldAddToBuffer = true;
// Continuous scanning for function calls in text - for models like DeepSeek
if (tools.isNotEmpty &&
supportsFunctionCalls &&
toolChoice != ToolChoice.none) {
// Check if we're currently buffering potential JSON
if (funcBuffer.isNotEmpty) {
// We're already buffering - add token and check for completion
funcBuffer += token;
if (kDebugMode) {
gemmaLog(
'InferenceChat: Buffering token: "$token", total: ${funcBuffer.length} chars',
level: GemmaLogLevel.verbose,
);
}
// Check if we now have a complete JSON
if (FunctionCallParser.isFunctionCallComplete(
funcBuffer,
modelType: modelType,
)) {
// First try to extract message from any JSON with message field
try {
final jsonData = jsonDecode(funcBuffer);
if (jsonData is Map<String, dynamic> &&
jsonData.containsKey('message')) {
// Found JSON with message field - extract and display the message
final message = jsonData['message'] as String;
if (kDebugMode) {
gemmaLog(
'InferenceChat: Extracted message from JSON: "$message"',
level: GemmaLogLevel.verbose,
);
}
yield TextResponse(message);
funcBuffer = '';
shouldAddToBuffer = false; // Don't add JSON tokens to buffer
continue;
}
} catch (e) {
gemmaLog(
'InferenceChat: Failed to parse JSON for message extraction: $e',
);
}
// If no message field found, try parsing as function call(s)
final allCalls = FunctionCallParser.parseAll(
funcBuffer,
modelType: modelType,
);
if (allCalls.isNotEmpty) {
gemmaLog(
'InferenceChat: Found ${allCalls.length} function call(s) in complete buffer!',
);
emittedFunctionCall = true;
// Add function call to history IMMEDIATELY (before yielding)
// so tool response from caller comes AFTER in history order
final toolCallMessage = Message.toolCall(text: funcBuffer);
_fullHistory.add(toolCallMessage);
_modelHistory.add(toolCallMessage);
gemmaLog(
'InferenceChat: Added function call to history before yielding',
);
if (allCalls.length == 1) {
yield allCalls.first;
} else {
yield ParallelFunctionCallResponse(calls: allCalls);
}
funcBuffer = '';
shouldAddToBuffer = false;
continue;
} else {
// Not a valid function call - emit as text and clear buffer
gemmaLog('InferenceChat: Invalid JSON, emitting as text');
yield TextResponse(funcBuffer);
funcBuffer = '';
shouldAddToBuffer = false;
continue;
}
}
// If buffer gets too long without completing, flush as text
if (funcBuffer.length > maxFunctionBufferLength) {
gemmaLog(
'InferenceChat: Buffer too long without completion, flushing as text',
);
yield TextResponse(funcBuffer);
funcBuffer = '';
shouldAddToBuffer = false;
continue;
}
// Still buffering, don't emit yet
shouldAddToBuffer = false;
} else {
// Not currently buffering - check if this token starts a function call
if (FunctionCallParser.isFunctionCallStart(
token,
modelType: modelType,
)) {
if (kDebugMode) {
gemmaLog(
'InferenceChat: Found potential function call start in token: "$token"',
level: GemmaLogLevel.verbose,
);
}
funcBuffer = token;
shouldAddToBuffer =
false; // Don't add to main buffer while we determine if it's JSON
} else if (modelType == ModelType.functionGemma &&
token.trim() == functionGemmaEndCall) {
// The call's closing brace already completed the buffer, so this
// trailing tag belongs to no call. It is markup, not text —
// emitting it would show `<end_function_call>` to the user and
// write it into chat history.
shouldAddToBuffer = false;
} else {
// Normal text token - emit immediately
if (kDebugMode) {
gemmaLog(
'InferenceChat: Emitting text token: "$token"',
level: GemmaLogLevel.verbose,
);
}
yield response;
shouldAddToBuffer = true; // Add to main buffer for history
}
}
} else {
// No function processing happening - emit token directly
if (kDebugMode) {
gemmaLog(
'InferenceChat: No function processing, emitting token as text: "$token"',
level: GemmaLogLevel.verbose,
);
}
yield response;
shouldAddToBuffer = true; // Add to main buffer for history
}
// Add token to buffer only if it should be included in final message
if (shouldAddToBuffer) {
buffer.write(token);
}
} else {
// For non-TextResponse (like ThinkingResponse), pass through
yield response;
}
}
gemmaLog('InferenceChat: Native token stream ended');
// The stream ended before we classified this Gemma 4 turn (e.g. a
// whitespace-only reply) — the probed tokens are plain text, flush them.
if (sdkPassthrough && !sdkClassified && sdkProbe.isNotEmpty) {
yield TextResponse(sdkProbe);
buffer.write(sdkProbe);
}
final response = buffer.toString();
gemmaLog(
'InferenceChat: Complete response accumulated: "$response"',
level: GemmaLogLevel.verbose,
);
// SDK-passthrough path (same guard that swallowed the tool-call JSON above):
// the structured tool calls live in `lastRawResponse`. Surface them here as
// the final ModelResponse(s).
if (sdkPassthrough) {
final raw = (session as RawSdkResponseSession).lastRawResponse;
if (raw != null) {
final allCalls = SdkResponseParser.extractToolCalls(raw);
if (allCalls.isNotEmpty) {
gemmaLog(
'InferenceChat: ${allCalls.length} SDK-parsed tool call(s) at end of stream',
);
emittedFunctionCall = true;
// Record the tool-call in history BEFORE yielding (mirrors the sync
// SDK path at ~L193 and the JSON paths) — otherwise the caller's
// tool-response has no matching call and is left orphaned, which
// corrupts the replayed history when the Gemma 4 session rotates.
// Strip the Gemma 4 escape tokens first, same as the sync path (#248).
final cleanRaw = SdkResponseParser.cleanRawForHistory(raw);
final toolCallMessage = Message.toolCall(text: cleanRaw);
_fullHistory.add(toolCallMessage);
_modelHistory.add(toolCallMessage);
if (allCalls.length == 1) {
yield allCalls.first;
} else {
yield ParallelFunctionCallResponse(calls: allCalls);
}
}
}
}
// Safety net: we suppressed a `{`-leading SDK-passthrough stream as a tool
// call, but lastRawResponse yielded no parseable call. Don't silently drop
// the model's output — surface the swallowed text (as it leaked pre-fix). It
// is deliberately NOT re-recorded to history: this only fires on a degenerate
// turn (a `{`-leading passthrough turn with no extractable tool_calls), where
// re-appending the JSON-shaped blob as an assistant turn would pollute the
// model's next-turn context more than omitting it.
if (sdkSwallow && !emittedFunctionCall && sdkSwallowed.isNotEmpty) {
gemmaLog(
'InferenceChat: SDK tool-call stream suppressed but no call parsed — '
'surfacing raw text (fallback)',
);
yield TextResponse(sdkSwallowed.toString());
}
// Handle end of stream - process any remaining buffer
if (funcBuffer.isNotEmpty) {
gemmaLog(
'InferenceChat: Processing remaining buffer at end of stream: ${funcBuffer.length} chars',
);
// For FunctionGemma, the function call spans response + funcBuffer
// (e.g., response="<start_function_call>call:fn", funcBuffer="{params}")
// For JSON models, funcBuffer contains the complete JSON
final contentToCheck = modelType == ModelType.functionGemma
? response + funcBuffer
: funcBuffer;
// First try to extract message from JSON if it has message field
if (FunctionCallParser.isFunctionCallComplete(
contentToCheck,
modelType: modelType,
)) {
try {
// For JSON parsing, use funcBuffer (the actual JSON part)
// For FunctionGemma parsing, use contentToCheck (full function call)
if (modelType != ModelType.functionGemma) {
final jsonData = jsonDecode(funcBuffer);
if (jsonData is Map<String, dynamic> &&
jsonData.containsKey('message')) {
final message = jsonData['message'] as String;
gemmaLog(
'InferenceChat: Extracted message from end-of-stream JSON: "$message"',
level: GemmaLogLevel.verbose,
);
yield TextResponse(message);
return;
}
}
// Try to parse as function call(s)
final allCalls = FunctionCallParser.parseAll(
contentToCheck,
modelType: modelType,
);
if (allCalls.isNotEmpty) {
gemmaLog(
'InferenceChat: ${allCalls.length} function call(s) found at end of stream',
);
emittedFunctionCall = true;
// Add function call to history IMMEDIATELY (before yielding)
final toolCallMessage = Message.toolCall(text: contentToCheck);
_fullHistory.add(toolCallMessage);
_modelHistory.add(toolCallMessage);
gemmaLog(
'InferenceChat: Added function call to history at end of stream',
);
if (allCalls.length == 1) {
yield allCalls.first;
} else {
yield ParallelFunctionCallResponse(calls: allCalls);
}
} else {
yield TextResponse(funcBuffer);
}
} catch (e) {
gemmaLog('InferenceChat: Failed to parse end-of-stream JSON: $e');
yield TextResponse(funcBuffer);
}
} else {
gemmaLog(
'InferenceChat: No complete JSON at end of stream, emitting remaining as text',
);
yield TextResponse(funcBuffer);
}
}
try {
gemmaLog('InferenceChat: Calculating response tokens...');
final responseTokens = await session.sizeInTokens(response);
gemmaLog('InferenceChat: Response tokens: $responseTokens');
_currentTokens += responseTokens;
gemmaLog('InferenceChat: Current total tokens: $_currentTokens');
if (_currentTokens >= (maxTokens - tokenBuffer)) {
gemmaLog('InferenceChat: Token limit reached, recreating session...');
await _recreateSessionWithReducedChunks();
gemmaLog('InferenceChat: Session recreated successfully');
}
} catch (e) {
gemmaLog('InferenceChat: Error during token calculation: $e');
}
try {
gemmaLog('InferenceChat: Adding message to history...');
// For function calls: already added to history when yielded (above)
// For text responses: add now since they weren't added during streaming.
// Skip an EMPTY response: a cancelled stream (stopGeneration before any
// text token) ends with response == '', and writing that empty assistant
// turn pollutes _modelHistory so later short replies come back empty
// (#325). "No text produced" is not a turn worth recording.
if (!emittedFunctionCall && response.isNotEmpty) {
final chatMessage = Message(text: response, isUser: false);
gemmaLog(
'InferenceChat: Created text message object: ${chatMessage.text}',
level: GemmaLogLevel.verbose,
);
_fullHistory.add(chatMessage);
gemmaLog('InferenceChat: Added to full history');
_modelHistory.add(chatMessage);
gemmaLog('InferenceChat: Added to model history');
} else {
gemmaLog(
'InferenceChat: Function call was already added to history when yielded',
);
}
gemmaLog('InferenceChat: Message added to history successfully');
// Clear model history for single-turn models (e.g., FunctionGemma)
// BUT only if this was NOT a function call - we need context for tool response
if (_isSingleTurnModel && !emittedFunctionCall) {
gemmaLog(
'InferenceChat: Single-turn model detected (text response), clearing model history...',
);
_modelHistory.clear();
_prefixes.clear();
_currentTokens = 0;
_toolsInstructionSent = false;
// Recreate session to clear native state
await session.close();
session = await sessionCreator!();
gemmaLog('InferenceChat: Model history cleared and session recreated');
} else if (_isSingleTurnModel && emittedFunctionCall) {
gemmaLog(
'InferenceChat: Single-turn model with function call - keeping history for tool response',
);
}
} catch (e) {
gemmaLog('InferenceChat: Error adding message to history: $e');
rethrow;
}
gemmaLog('InferenceChat: generateChatResponseAsync completed successfully');
}