compress static method
Future<CompressedContext>
compress({
- required List<
ChunkSearchResult> chunks, - CompressionLevel level = CompressionLevel.balanced,
- int maxTokens = 2000,
- String language = 'ko',
- int tokenCounter(
- String text
Compress chunks for LLM context within a token budget.
chunks - Search result chunks to compress.
level - Compression aggressiveness.
maxTokens - Target maximum tokens. Without tokenCounter this is an
estimate of 4 characters per token, which can be several times off
for Korean, Thai or emoji.
language - Language for stopword filtering ("ko" or "en").
tokenCounter - Counts tokens as your LLM does. When given, the text is
cut at sentence boundaries until it has at most maxTokens tokens by
this count, and CompressedContext.estimatedTokensSaved uses it too.
Implementation
static Future<CompressedContext> compress({
required List<ChunkSearchResult> chunks,
CompressionLevel level = CompressionLevel.balanced,
int maxTokens = 2000,
String language = 'ko',
int Function(String text)? tokenCounter,
}) async {
if (chunks.isEmpty) {
return const CompressedContext(
text: '',
originalChars: 0,
compressedChars: 0,
ratio: 1.0,
estimatedTokensSaved: 0,
includedChunks: [],
);
}
// Combine all chunk content
final originalText = chunks.map((c) => c.content).join('\n\n');
final originalChars = originalText.length;
// Convert level to int for Rust
final levelInt = level.index;
// Create compression options
final options = rust.CompressionOptions(
removeStopwords: false, // Disabled - damages context
removeDuplicates: true,
language: language,
level: levelInt,
);
final rust.CompressedText result;
final int tokensSaved;
if (tokenCounter == null) {
// Calculate max chars from token budget (~4 chars per token)
result = await rust.compressText(
text: originalText,
maxChars: maxTokens * 4,
options: options,
);
// Calculate token savings (rough estimate)
tokensSaved =
(originalChars / 4).ceil() - (result.compressedChars / 4).ceil();
} else {
result = await _compressToTokens(
originalText,
maxTokens,
tokenCounter,
options,
);
tokensSaved = tokenCounter(originalText) - tokenCounter(result.text);
}
return CompressedContext(
text: result.text,
originalChars: result.originalChars,
compressedChars: result.compressedChars,
ratio: result.ratio,
estimatedTokensSaved: tokensSaved,
includedChunks: chunks,
);
}