tokenize method

List<LatexToken> tokenize(
  1. String text
)

Full initial tokenization

Implementation

List<LatexToken> tokenize(String text) {
  _textBuffer.clear();
  _textBuffer.write(text);
  final tokenizer = LatexTokenizer(text);
  final allTokens = tokenizer.tokenize();

  // Remove the trailing EOF token, as we manage it ourselves
  // Using in-place filtering instead of filterNot + toMutableList double copy
  _cachedTokens = <LatexToken>[];
  for (final token in allTokens) {
    if (token is! EOFToken) _cachedTokens.add(token);
  }

  _invalidateResultCache();
  return allTokens;
}