nextToken method

LatexToken nextToken()

Scans and returns the next token.

Unlike tokenize, this method does not add the token to the internal list, it just returns it. Used by IncrementalTokenizer.

Implementation

LatexToken nextToken() {
  if (_position >= input.length) {
    return EOFToken(range: SourceRange(_position, _position));
  }

  final savedSize = _tokens.length;
  final char = _peek();

  switch (char) {
    case '\\':
      _handleBackslash();
      break;
    case '%':
      _handleComment();
      break;
    case '~':
      final start = _position;
      _advance();
      _tokens.add(
        WhitespaceToken("\u00A0", range: SourceRange(start, _position)),
      );
      break;
    case '{':
      final start = _position;
      _advance();
      _tokens.add(LeftBraceToken(range: SourceRange(start, _position)));
      break;
    case '}':
      final start = _position;
      _advance();
      _tokens.add(RightBraceToken(range: SourceRange(start, _position)));
      break;
    case '[':
      final start = _position;
      _advance();
      _tokens.add(LeftBracketToken(range: SourceRange(start, _position)));
      break;
    case ']':
      final start = _position;
      _advance();
      _tokens.add(RightBracketToken(range: SourceRange(start, _position)));
      break;
    case '^':
      final start = _position;
      _advance();
      _tokens.add(SuperscriptToken(range: SourceRange(start, _position)));
      break;
    case '\'':
      final start = _position;
      _advance();
      _tokens.add(PrimeToken(range: SourceRange(start, _position)));
      break;
    case '_':
      final start = _position;
      _advance();
      _tokens.add(SubscriptToken(range: SourceRange(start, _position)));
      break;
    case '&':
      final start = _position;
      _advance();
      _tokens.add(AmpersandToken(range: SourceRange(start, _position)));
      break;
    case '\$':
      _handleMathShift();
      break;
    case '+':
    case '-':
    case '=':
    case '<':
    case '>':
    case ',':
    case ';':
    case ':':
    case '(':
    case ')':
    case '|':
    case '.':
      final start = _position;
      _tokens.add(TextToken(char!, range: SourceRange(start, start + 1)));
      _advance();
      break;
    case '\n':
    case '\r':
      _handleNewLine();
      break;
    case ' ':
    case '\t':
      _handleWhitespace();
      break;
    default:
      if (char != null) _handleText();
      break;
  }

  // handleComment doesn't generate a token, needs recursive evaluation
  if (_tokens.length > savedSize) {
    return _tokens.removeAt(_tokens.length - 1);
  } else {
    // Comment was skipped, grab the next actual token
    return nextToken();
  }
}