tokenize method

List<LatexToken> tokenize()

Executes lexical analysis

Implementation

List<LatexToken> tokenize() {
  // HLog.d(_tag, "开始词法分析，输入长度: ${input.length}");

  while (_position < input.length) {
    final char = _peek();
    if (char == null) break;

    switch (char) {
      case '\\':
        _handleBackslash();
        break;
      case '%':
        _handleComment();
        break;
      case '~':
        final start = _position;
        _advance();
        _tokens.add(
          WhitespaceToken("\u00A0", range: SourceRange(start, _position)),
        );
        break;
      case '{':
        final start = _position;
        _advance();
        _tokens.add(LeftBraceToken(range: SourceRange(start, _position)));
        break;
      case '}':
        final start = _position;
        _advance();
        _tokens.add(RightBraceToken(range: SourceRange(start, _position)));
        break;
      case '[':
        final start = _position;
        _advance();
        _tokens.add(LeftBracketToken(range: SourceRange(start, _position)));
        break;
      case ']':
        final start = _position;
        _advance();
        _tokens.add(RightBracketToken(range: SourceRange(start, _position)));
        break;
      case '^':
        final start = _position;
        _advance();
        _tokens.add(SuperscriptToken(range: SourceRange(start, _position)));
        break;
      case '\'':
        final start = _position;
        _advance();
        _tokens.add(PrimeToken(range: SourceRange(start, _position)));
        break;
      case '_':
        final start = _position;
        _advance();
        _tokens.add(SubscriptToken(range: SourceRange(start, _position)));
        break;
      case '&':
        final start = _position;
        _advance();
        _tokens.add(AmpersandToken(range: SourceRange(start, _position)));
        break;
      case '\$':
        _handleMathShift();
        break;
      // Dart 3 supports multi-matching for switch cases:
      case '+':
      case '-':
      case '=':
      case '<':
      case '>':
      case ',':
      case ';':
      case ':':
      case '(':
      case ')':
      case '|':
      case '.':
        final start = _position;
        _tokens.add(TextToken(char, range: SourceRange(start, start + 1)));
        _advance();
        break;
      case '\n':
      case '\r':
        _handleNewLine();
        break;
      case ' ':
      case '\t':
        _handleWhitespace();
        break;
      default:
        _handleText();
    }
  }

  _tokens.add(EOFToken(range: SourceRange(_position, _position)));
  // HLog.d(_tag, "词法分析完成，生成 ${_tokens.length} 个 token");
  return _tokens;
}