completionWithTools method

Future<CompletionResult> completionWithTools(List<ChatMessage> messages, { int maxTokens = 256, double? temperature, int? topK, double? topP, List<String>? stopSequences, CactusTokenCallback? onToken, })

Implementation

Future<CompletionResult> completionWithTools(
  List<ChatMessage> messages, {
  int maxTokens = 256,
  double? temperature,
  int? topK,
  double? topP,
  List<String>? stopSequences,
  CactusTokenCallback? onToken,
}) async {
  if (_context == null) throw CactusException('CactusLM not initialized');

  if (_tools.isEmpty()) {
    final response = await completion(
      messages,
      maxTokens: maxTokens,
      temperature: temperature,
      topP: topP,
    );
    return CompletionResult(
      result: response.text,
      toolCalls: null,
    );
  }

  if (!isLoaded()) {
    return CompletionResult(
      result: "Model is not loaded",
      toolCalls: null,
    );
  }

  final toolSchemas = _tools.getSchemas();
  final toolsJson = jsonEncode(toolSchemas.map((e) => e.toJson()).toList());

  final userMessage = ChatMessage(role: 'user', content: messages.last.content);
  final formattedResult = await _context!.formatChatWithTools([userMessage], toolsJson);

  final params = CactusCompletionParams(
    messages: [ChatMessage(role: 'user', content: formattedResult.prompt)],
    maxPredictedTokens: maxTokens,
    temperature: temperature,
    topP: topP,
    chatTemplate: _chatTemplate,
    grammar: formattedResult.grammar,
  );

  final modelResponse = await _context!.completion(params);

  final toolResult = await parseAndExecuteTool(modelResponse.text, _tools);

  return CompletionResult(
    result: toolResult.toolCalled ? toolResult.toolOutput : modelResponse.text,
    toolCalls: toolResult.toolCalled ? [toolResult.toolName!] : null,
  );
}