next method

Token next()

Returns the next token.

Implementation

Token next() {
  _skipComments();
  if (_pos >= input.length) return Token.eof(input.length + spanOffset);
  final start = _pos;
  final c = input.codeUnitAt(_pos);

  if (_isSpace(c)) {
    // An empty line: two line ends with only spaces between.
    var lineEnds = 0;
    while (_pos < input.length) {
      final d = input.codeUnitAt(_pos);
      if (_isSpace(d)) {
        if (d == 0x0A) lineEnds++;
        _pos++;
      } else if (d == 0x25 /* % */ ) {
        // A comment line is not empty.
        lineEnds = 0;
        _skipComments();
      } else {
        break;
      }
    }
    if (paragraphs && lineEnds >= 2) {
      return _token(TokenKind.controlWord, r'\par', start);
    }
    return _token(TokenKind.space, ' ', start);
  }

  if (c == 0x5C /* \ */ ) {
    _pos++;
    // `\\x` read as `\x`: the first backslash only doubles the second.
    if (undoubleBackslashes &&
        _pos < input.length &&
        input.codeUnitAt(_pos) == 0x5C) {
      _pos++;
    }
    final escape = _pos - 1;
    if (_pos >= input.length) {
      return _token(TokenKind.controlWord, r'\', start);
    }
    final d = input.codeUnitAt(_pos);
    if (_isLetter(d)) {
      while (_pos < input.length && _isLetter(input.codeUnitAt(_pos))) {
        _pos++;
      }
      final name = input.substring(escape, _pos);
      if (name == r'\verb') return _verb(start);
      final end = _pos;
      _skipSpacesAfterControlWord();
      return Token(
        TokenKind.controlWord,
        name,
        SourceSpan(start + spanOffset, end + spanOffset),
      );
    }
    // A doubled `\\` after the escape is one backslash: `\\\\` is `\\`.
    if (undoubleBackslashes &&
        d == 0x5C &&
        _pos + 1 < input.length &&
        input.codeUnitAt(_pos + 1) == 0x5C) {
      _pos += 2;
      return _token(TokenKind.controlSymbol, r'\\', start);
    }
    if (_isSpace(d)) {
      // A control space: `\` before a space, tab or line end, as TeX
      // reads it (plain TeX defines `\^^M` and `\^^I` as `\ `). The
      // blanks after it are skipped, over at most one line end.
      _skipBlanks();
      if (_pos < input.length && input.codeUnitAt(_pos) == 0x0A) _pos++;
      _skipBlanks();
      return _token(TokenKind.controlSymbol, r'\ ', start);
    }
    final symbolEnd = _graphemeEnd(_pos);
    _pos = symbolEnd;
    return _token(
      TokenKind.controlSymbol,
      input.substring(escape, _pos),
      start,
    );
  }

  _pos = _graphemeEnd(_pos);
  return _token(TokenKind.character, input.substring(start, _pos), start);
}