fitInputs method

Future<KnowledgeSnapshot> fitInputs({ required Future<int> countTokens(String), required int maxTokens, required bool includeContext, })

Splits oversized passages at whitespace using the actual tokenizer. Preserves text, character spans, and citation lines. Context counts against the budget; if context plus one word cannot fit, throws without truncation.

Implementation

Future<KnowledgeSnapshot> fitInputs({
  required Future<int> Function(String) countTokens,
  required int maxTokens,
  required bool includeContext,
}) async {
  if (maxTokens <= 0) throw ArgumentError.value(maxTokens, 'maxTokens');
  final output = <Chunk>[];
  final texts = <String, String>{};
  for (final chunk in chunks) {
    final full = textFor(chunk, includeContext: includeContext);
    final contextual = _contextTexts[chunk.id]!;
    final prefix = contextual.substring(
      0,
      contextual.length - chunk.content.length,
    );
    if (await countTokens(full) <= maxTokens) {
      output.add(chunk);
      texts[chunk.id] = contextual;
      continue;
    }
    final ends = RegExp(
      r'\S+\s*',
    ).allMatches(chunk.content).map((match) => match.end).toList();
    var start = 0;
    var first = 0;
    while (start < chunk.content.length) {
      var low = first;
      var high = ends.length - 1;
      int? chosen;
      while (low <= high) {
        final mid = (low + high) ~/ 2;
        final passage = chunk.content.substring(start, ends[mid]);
        final input = includeContext ? '$prefix$passage' : passage;
        if (await countTokens(input) <= maxTokens) {
          chosen = mid;
          low = mid + 1;
        } else {
          high = mid - 1;
        }
      }
      if (chosen == null) {
        throw StateError(
          'Cannot fit context and one word within $maxTokens tokens: ${chunk.sourcePath}',
        );
      }
      final end = ends[chosen];
      final passage = chunk.content.substring(start, end);
      final lineStart =
          chunk.lineStart +
          '\n'.allMatches(chunk.content.substring(0, start)).length;
      final id = sha256
          .convert(utf8.encode(jsonEncode([chunk.id, start, end])))
          .toString();
      final split = chunk.copyWith(
        id: id,
        content: passage,
        lineStart: lineStart,
        lineEnd: lineStart + '\n'.allMatches(passage.trimRight()).length,
        metadata: {
          'okf': {
            ...chunk.metadata['okf']! as Map<String, Object?>,
            'parentChunkId': chunk.id,
            'characterStart': start,
            'characterEnd': end,
          },
        },
      );
      output.add(split);
      texts[id] = '$prefix$passage';
      start = end;
      first = chosen + 1;
    }
  }
  return KnowledgeSnapshot._(
    bundleId,
    List.unmodifiable(output),
    graph,
    _metadata,
    Map.unmodifiable(texts),
    sources,
    assets,
  );
}