chunkText method

List<TextChunk> chunkText(String text)

Splits text into chunks according to rules from AGENT_GUIDE.md

Implementation

List<TextChunk> chunkText(String text) {
  if (text.trim().isEmpty) {
    return [];
  }

  // If text is short - one chunk
  if (text.length <= maxSingleChunk) {
    return [
      TextChunk(
        content: text.trim(),
        chunkId: 0,
        section: 'full_doc',
        type: _detectType(text),
      ),
    ];
  }

  // Split text into sections: regular text and code blocks
  final sections = _splitIntoSections(text);
  final chunks = <TextChunk>[];
  int chunkId = 0;

  for (final section in sections) {
    if (section.isCode) {
      // Code block - separate chunk
      // IMPORTANT: For code blocks, only trim leading/trailing whitespace
      // but preserve all line breaks and formatting inside the code block
      final codeContent = section.content;
      // Remove only leading and trailing whitespace/newlines, but preserve internal formatting
      final trimmed = codeContent.trim();
      chunks.add(
        TextChunk(
          content: trimmed,
          chunkId: chunkId++,
          section: _detectSection(section.content),
          type: 'code',
        ),
      );
    } else {
      // Text sections - chunk according to rules
      final textChunks = _chunkTextSection(section.content);
      for (final chunk in textChunks) {
        chunks.add(
          TextChunk(
            content: chunk.trim(),
            chunkId: chunkId++,
            section: _detectSection(chunk),
            type: 'text',
          ),
        );
      }
    }
  }

  return chunks;
}