next method

Token next()

Returns the next token.

Implementation

Token next() {
  _skipComments();
  if (_pos >= input.length) return Token.eof(input.length + spanOffset);
  final start = _pos;
  final c = input.codeUnitAt(_pos);

  if (_isSpace(c)) {
    while (_pos < input.length) {
      final d = input.codeUnitAt(_pos);
      if (_isSpace(d)) {
        _pos++;
      } else if (d == 0x25 /* % */ ) {
        _skipComments();
      } else {
        break;
      }
    }
    return _token(TokenKind.space, ' ', start);
  }

  if (c == 0x5C /* \ */ ) {
    _pos++;
    if (_pos >= input.length) {
      return _token(TokenKind.controlWord, r'\', start);
    }
    final d = input.codeUnitAt(_pos);
    if (_isLetter(d)) {
      while (_pos < input.length && _isLetter(input.codeUnitAt(_pos))) {
        _pos++;
      }
      final name = input.substring(start, _pos);
      if (name == r'\verb') return _verb(start);
      final end = _pos;
      _skipSpacesAfterControlWord();
      return Token(
        TokenKind.controlWord,
        name,
        SourceSpan(start + spanOffset, end + spanOffset),
      );
    }
    final symbolEnd = _graphemeEnd(_pos);
    _pos = symbolEnd;
    return _token(
      TokenKind.controlSymbol,
      input.substring(start, _pos),
      start,
    );
  }

  _pos = _graphemeEnd(_pos);
  return _token(TokenKind.character, input.substring(start, _pos), start);
}