compress static method

Future<CompressedContext> compress({
  1. required List<ChunkSearchResult> chunks,
  2. CompressionLevel level = CompressionLevel.balanced,
  3. int maxTokens = 2000,
  4. String language = 'ko',
  5. int tokenCounter(
    1. String text
    )?,
})

Compress chunks for LLM context within a token budget.

chunks - Search result chunks to compress. level - Compression aggressiveness. maxTokens - Target maximum tokens. Without tokenCounter this is an estimate of 4 characters per token, which can be several times off for Korean, Thai or emoji. language - Language for stopword filtering ("ko" or "en"). tokenCounter - Counts tokens as your LLM does. When given, the text is cut at sentence boundaries until it has at most maxTokens tokens by this count, and CompressedContext.estimatedTokensSaved uses it too.

Implementation

static Future<CompressedContext> compress({
  required List<ChunkSearchResult> chunks,
  CompressionLevel level = CompressionLevel.balanced,
  int maxTokens = 2000,
  String language = 'ko',
  int Function(String text)? tokenCounter,
}) async {
  if (chunks.isEmpty) {
    return const CompressedContext(
      text: '',
      originalChars: 0,
      compressedChars: 0,
      ratio: 1.0,
      estimatedTokensSaved: 0,
      includedChunks: [],
    );
  }

  // Combine all chunk content
  final originalText = chunks.map((c) => c.content).join('\n\n');
  final originalChars = originalText.length;

  // Convert level to int for Rust
  final levelInt = level.index;

  // Create compression options
  final options = rust.CompressionOptions(
    removeStopwords: false, // Disabled - damages context
    removeDuplicates: true,
    language: language,
    level: levelInt,
  );

  final rust.CompressedText result;
  final int tokensSaved;
  if (tokenCounter == null) {
    // Calculate max chars from token budget (~4 chars per token)
    result = await rust.compressText(
      text: originalText,
      maxChars: maxTokens * 4,
      options: options,
    );
    // Calculate token savings (rough estimate)
    tokensSaved =
        (originalChars / 4).ceil() - (result.compressedChars / 4).ceil();
  } else {
    result = await _compressToTokens(
      originalText,
      maxTokens,
      tokenCounter,
      options,
    );
    tokensSaved = tokenCounter(originalText) - tokenCounter(result.text);
  }

  return CompressedContext(
    text: result.text,
    originalChars: result.originalChars,
    compressedChars: result.compressedChars,
    ratio: result.ratio,
    estimatedTokensSaved: tokensSaved,
    includedChunks: chunks,
  );
}