tokenize method

List<Token> tokenize()

Tokenizes the source string into a list of tokens.

Parses the input source and breaks it down into individual tokens (operators, numbers, identifiers, etc.) for further processing. Returns a list of Token objects representing the parsed input.

Implementation

List<Token> tokenize() {
  final tokens = <Token>[];
  while (index < endIndex) {
    eatWhitespace();
    if (index >= endIndex) break;

    final start = index;
    final char = source[index];

    if (char == '+') {
      tokens.add(Token(TokenType.plus, source, start, ++index));
    } else if (char == '-') {
      tokens.add(Token(TokenType.minus, source, start, ++index));
    } else if (char == '*') {
      tokens.add(Token(TokenType.multiply, source, start, ++index));
    } else if (char == '/') {
      tokens.add(Token(TokenType.divide, source, start, ++index));
    } else if (char == '(') {
      tokens.add(Token(TokenType.lparen, source, start, ++index));
    } else if (char == ')') {
      tokens.add(Token(TokenType.rparen, source, start, ++index));
    } else if (char == '%') {
      tokens.add(Token(TokenType.percent, source, start, ++index));
    } else if (char == ',') {
      tokens.add(Token(TokenType.comma, source, start, ++index));
    } else if (char == '#') {
      final range = eat(
        (c, i) => i == 0
            ? c == '#'
            : (c.codeUnitAt(0) >= 'a'.codeUnitAt(0) &&
                      c.codeUnitAt(0) <= 'z'.codeUnitAt(0)) ||
                  (c.codeUnitAt(0) >= 'A'.codeUnitAt(0) &&
                      c.codeUnitAt(0) <= 'Z'.codeUnitAt(0)) ||
                  (c.codeUnitAt(0) >= '0'.codeUnitAt(0) &&
                      c.codeUnitAt(0) <= '9'.codeUnitAt(0)) ||
                  c == '_',
        null,
      );
      tokens.add(
        Token(TokenType.symbol, source, range!.startIndex, range.endIndex),
      );
    } else if (char == "'") {
      index++;
      eat((c, i) => c != "'", null);
      tokens.add(Token(TokenType.string, source, start, index + 1));
      index++;
    } else if (char == '"') {
      index++;
      eat((c, i) => c != '"', null);
      tokens.add(Token(TokenType.string, source, start, index + 1));
      index++;
    } else if (char.codeUnitAt(0) >= '0'.codeUnitAt(0) &&
        char.codeUnitAt(0) <= '9'.codeUnitAt(0)) {
      eatNumber();
      tokens.add(Token(TokenType.number, source, start, index));
    } else if ((char.codeUnitAt(0) >= 'a'.codeUnitAt(0) &&
            char.codeUnitAt(0) <= 'z'.codeUnitAt(0)) ||
        (char.codeUnitAt(0) >= 'A'.codeUnitAt(0) &&
            char.codeUnitAt(0) <= 'Z'.codeUnitAt(0)) ||
        char == '_') {
      final range = eat(
        (c, i) =>
            (c.codeUnitAt(0) >= 'a'.codeUnitAt(0) &&
                c.codeUnitAt(0) <= 'z'.codeUnitAt(0)) ||
            (c.codeUnitAt(0) >= 'A'.codeUnitAt(0) &&
                c.codeUnitAt(0) <= 'Z'.codeUnitAt(0)) ||
            (c.codeUnitAt(0) >= '0'.codeUnitAt(0) &&
                c.codeUnitAt(0) <= '9'.codeUnitAt(0)) ||
            c == '_',
        null,
      );
      tokens.add(
        Token(
          TokenType.identifier,
          source,
          range!.startIndex,
          range.endIndex,
        ),
      );
    } else {
      throw 'Unexpected character: $char';
    }
  }
  tokens.add(Token(TokenType.eof, source, index, index));
  return tokens;
}