tokenize method

List<TextAndTokens> tokenize(String text, { int? maxTokens, })

Implementation

List<TextAndTokens> tokenize(String text, {int? maxTokens}) {
  final max = maxTokens ?? maxInputTokens;
  text = text.trim();
  if (text.isEmpty) {
    return [
      TextAndTokens(text: '', tokens: [startToken, endToken]),
    ];
  }

  final words = text.split(RegExp(r'\s+'));
  final allOutputTokens = <List<int>>[];
  final allOutputStrings = <String>[];

  var currentTokens = <int>[startToken];
  var currentStrings = <String>[];

  for (final word in words) {
    if (word.length > maxInputCharsPerWord) continue;

    // Get or compute normalized word
    final normalizedWord = _normalizedCache[word] ??= removeDiacritics(
      word.toLowerCase(),
    );
    final wordTokens = _tokenizeWord(normalizedWord);

    if (currentTokens.length + wordTokens.length >= max - 1) {
      // Finalize current chunk
      currentTokens.add(endToken);
      allOutputTokens.add(currentTokens);
      allOutputStrings.add(currentStrings.join(' '));

      // Start new chunk
      currentTokens = <int>[startToken, ...wordTokens];
      currentStrings = [word];
    } else {
      currentTokens.addAll(wordTokens);
      currentStrings.add(word);
    }
  }

  // Add final chunk
  currentTokens.add(endToken);
  allOutputTokens.add(currentTokens);
  allOutputStrings.add(currentStrings.join(' '));

  return List<TextAndTokens>.generate(
    allOutputStrings.length,
    (i) =>
        TextAndTokens(text: allOutputStrings[i], tokens: allOutputTokens[i]),
  );
}