count method

@override int count(String text)

Returns the number of tokens in text.

Implementation

@override
int count(String text) {
  if (text.isEmpty) return 0;
  final matches = _cl100kSplitPattern.allMatches(text);
  var total = 0;
  for (final m in matches) {
    total += _estimateChunkTokens(m.group(0)!);
  }
  return total == 0 ? _heuristicCount(text) : total;
}