fitInputs method
Fits complete effective inputs with the actual tokenizer, without overlap or truncation. Whitespace stays attached to nonblank text. Irreducible contextual failures retry the original chunk without derived context; irreducible body failures remain errors.
Implementation
Future<KnowledgeSnapshot> fitInputs({
required Future<int> Function(String) countTokens,
required int maxTokens,
required bool includeContext,
}) async {
if (maxTokens <= 0) throw ArgumentError.value(maxTokens, 'maxTokens');
final output = <Chunk>[];
final texts = <String, String>{};
final recoveries = {for (final r in _recoveries) (r.originalId, r.code): r};
final groups = <String, List<Chunk>>{};
for (final chunk in chunks) {
final okf = chunk.metadata['okf']! as Map;
final root = okf['parentChunkId'] as String? ?? chunk.id;
groups.putIfAbsent(root, () => []).add(chunk);
}
for (final entry in groups.entries) {
final group = entry.value;
final first = group.first;
final originalMetadata = first.metadata['okf']! as Map;
final originalStart =
originalMetadata['originalLineStart'] as int? ?? first.lineStart;
final originalEnd =
originalMetadata['originalLineEnd'] as int? ?? group.last.lineEnd;
void record(String code) {
recoveries[(entry.key, code)] = _InputRecovery(
entry.key,
code,
first.sourcePath,
originalStart,
originalEnd,
);
}
Future<({List<Chunk> chunks, Map<String, String> texts, bool split})?>
attempt(bool context, {bool restart = false}) async {
final staged = <Chunk>[];
final inputs = <String, String>{};
var segmentSplit = false;
// On context failure retry the complete original body, including when
// this snapshot was already fitted and reopened. Ordinary refits retain
// their existing fragment boundaries and IDs.
final inputsToFit = restart
? [
first.copyWith(
id: entry.key,
content: group.map((chunk) => chunk.content).join(),
lineStart: originalStart,
lineEnd: originalEnd,
metadata: {
...first.metadata,
'okf': Map<String, Object?>.from(originalMetadata)
..remove('parentChunkId')
..remove('characterStart')
..remove('characterEnd')
..remove('originalLineStart')
..remove('originalLineEnd'),
},
),
]
: group;
for (final chunk in inputsToFit) {
final contextual = context ? _contextTexts[chunk.id]! : '';
final prefix = context
? contextual.substring(
0,
contextual.length - chunk.content.length,
)
: '';
final ranges = <(int, int)>[];
Future<bool> fit(int start, int end) async {
final passage = chunk.content.substring(start, end);
final input = '$prefix$passage';
if (input.trim().isEmpty) return false;
if (await countTokens(input) <= maxTokens) {
ranges.add((start, end));
return true;
}
// Keep outer whitespace on a nonblank fragment: the encoder rejects
// blank inputs even when their token count fits. Only split inside
// the nonblank span; the full, untrimmed halves are measured again.
final textStart = end - passage.trimLeft().length;
final textEnd = start + passage.trimRight().length;
// Prefer a whitespace endpoint nearest the midpoint. No inference
// about other token counts is made from this input's count.
final middle = (textStart + textEnd) ~/ 2;
final lineBoundaries = RegExp(r'\n')
.allMatches(passage)
.map((m) => start + m.end)
.where((i) => i > textStart && i < textEnd)
.toList();
final boundaries =
(chunk.type == 'table' || chunk.type == 'code') &&
lineBoundaries.isNotEmpty
? lineBoundaries
: RegExp(r'\s+')
.allMatches(passage)
.map((m) => start + m.end)
.where((i) => i > textStart && i < textEnd);
int? cut;
for (final boundary in boundaries) {
if (cut == null ||
(boundary - middle).abs() < (cut - middle).abs()) {
cut = boundary;
}
}
if (cut == null) {
cut = middle;
if (cut > textStart &&
cut < textEnd &&
chunk.content.codeUnitAt(cut) >= 0xdc00 &&
chunk.content.codeUnitAt(cut) <= 0xdfff &&
chunk.content.codeUnitAt(cut - 1) >= 0xd800 &&
chunk.content.codeUnitAt(cut - 1) <= 0xdbff) {
cut--;
if (cut == textStart) cut += 2;
}
if (cut <= textStart || cut >= textEnd) return false;
segmentSplit = true;
}
return await fit(start, cut) && await fit(cut, end);
}
if (!await fit(0, chunk.content.length)) return null;
final okf = chunk.metadata['okf']! as Map<String, Object?>;
final offset = okf['characterStart'] as int? ?? 0;
for (final (start, end) in ranges) {
final passage = chunk.content.substring(start, end);
final unchanged = start == 0 && end == chunk.content.length;
final id = unchanged
? chunk.id
: sha256
.convert(
utf8.encode(
jsonEncode([entry.key, offset + start, offset + end]),
),
)
.toString();
final lineStart =
chunk.lineStart +
'\n'.allMatches(chunk.content.substring(0, start)).length;
staged.add(
unchanged
? chunk
: chunk.copyWith(
id: id,
content: passage,
lineStart: lineStart,
lineEnd:
lineStart +
'\n'.allMatches(passage.trimRight()).length,
metadata: {
...chunk.metadata,
'okf': {
...okf,
'parentChunkId': entry.key,
'characterStart': offset + start,
'characterEnd': offset + end,
'originalLineStart': originalStart,
'originalLineEnd': originalEnd,
},
},
),
);
inputs[id] = '$prefix$passage';
}
}
return (chunks: staged, texts: inputs, split: segmentSplit);
}
var result = await attempt(includeContext);
if (result == null && includeContext) {
result = await attempt(false, restart: true);
if (result != null) record('embedding_context_omitted');
}
if (result == null) {
throw StateError(
'Cannot fit body-only input within $maxTokens tokens: ${first.sourcePath}',
);
}
if (result.split) record('oversized_segment_split');
output.addAll(result.chunks);
texts.addAll(result.texts);
}
return KnowledgeSnapshot._(
bundleId,
List.unmodifiable(output),
graph,
fieldEdges,
_metadata,
Map.unmodifiable(texts),
sources,
assets,
List.unmodifiable(recoveries.values),
);
}