tokenize function

List<Token> tokenize(
  1. String src
)

lang-core tokenize.

Implementation

List<Token> tokenize(String src) {
  final tokens = <Token>[];
  var i = 0;
  final n = src.length;
  while (i < n) {
    while (i < n && (src[i] == ' ' || src[i] == '\t' || src[i] == '\r')) {
      i++;
    }
    if (i >= n) break;
    final c = src[i];
    final next = i + 1 < n ? src[i + 1] : '';
    final single = switch (c) {
      '\n' => T.newline,
      '(' => T.lParen,
      ')' => T.rParen,
      '[' => T.lBracket,
      ']' => T.rBracket,
      '{' => T.lBrace,
      '}' => T.rBrace,
      ',' => T.comma,
      ':' => T.colon,
      '.' => T.dot,
      '?' => T.question,
      '+' => T.plus,
      '*' => T.star,
      '/' => T.slash,
      '%' => T.percent,
      _ => null,
    };
    if (single != null) {
      tokens.add(Token(single));
      i++;
      continue;
    }
    if (c == '=' || c == '!' || c == '>' || c == '<') {
      final pair = next == '=';
      tokens.add(
        Token(switch (c) {
          '=' => pair ? T.eqEq : T.equals,
          '!' => pair ? T.notEq : T.not,
          '>' => pair ? T.greaterEq : T.greater,
          _ => pair ? T.lessEq : T.less,
        }),
      );
      i += pair ? 2 : 1;
      continue;
    }
    if (c == '&' || c == '|') {
      tokens.add(Token(c == '&' ? T.and : T.or));
      i += next == c ? 2 : 1;
      continue;
    }
    if (c == '"') {
      final start = i;
      i++;
      var closed = false;
      while (i < n) {
        if (src[i] == r'\') {
          i += 2;
        } else if (src[i] == '"') {
          i++;
          closed = true;
          break;
        } else {
          i++;
        }
      }
      if (i > n) i = n;
      final raw = src.substring(start, i);
      tokens.add(
        Token(T.str, _decodeDoubleQuoted(closed ? raw : '$raw"', raw)),
      );
      continue;
    }
    if (c == "'") {
      i++;
      final out = StringBuffer();
      while (i < n) {
        if (src[i] == r'\') {
          i++;
          if (i < n) {
            final esc = src[i];
            out.write(switch (esc) {
              'n' => '\n',
              't' => '\t',
              _ => esc,
            });
            i++;
          }
        } else if (src[i] == "'") {
          i++;
          break;
        } else {
          out.write(src[i]);
          i++;
        }
      }
      tokens.add(Token(T.str, out.toString()));
      continue;
    }
    if (c == '-') {
      final prev = tokens.isEmpty ? null : tokens.last;
      final negativeLiteral =
          !(prev != null && _valueEnds.contains(prev.t)) && _isDigit(next);
      if (!negativeLiteral) {
        tokens.add(const Token(T.minus));
        i++;
        continue;
      }
    }
    if (_isDigit(c) || (c == '-' && _isDigit(next))) {
      final start = i;
      if (src[i] == '-') i++;
      while (i < n && _isDigit(src[i])) {
        i++;
      }
      if (i + 1 < n && src[i] == '.' && _isDigit(src[i + 1])) {
        i++;
        while (i < n && _isDigit(src[i])) {
          i++;
        }
      }
      if (i < n && (src[i] == 'e' || src[i] == 'E')) {
        i++;
        if (i < n && (src[i] == '+' || src[i] == '-')) i++;
        while (i < n && _isDigit(src[i])) {
          i++;
        }
      }
      final text = src.substring(start, i);
      tokens.add(Token(T.num, _parseNumberLiteral(text)));
      continue;
    }
    if (c == r'$' && _isIdentStart(next)) {
      final start = i;
      i++;
      while (i < n && _isIdentPart(src[i])) {
        i++;
      }
      tokens.add(Token(T.stateVar, src.substring(start, i)));
      continue;
    }
    if (_isIdentStart(c)) {
      final start = i;
      while (i < n && _isIdentPart(src[i])) {
        i++;
      }
      final word = src.substring(start, i);
      tokens.add(switch (word) {
        'true' => const Token(T.trueKw),
        'false' => const Token(T.falseKw),
        'null' => const Token(T.nullKw),
        _ => Token(_isUpper(c) ? T.type : T.ident, word),
      });
      continue;
    }
    if (c == '@' && _isIdentStart(next)) {
      i++;
      final start = i;
      while (i < n && _isIdentPart(src[i])) {
        i++;
      }
      tokens.add(Token(T.builtinCall, src.substring(start, i)));
      continue;
    }
    i++;
  }
  tokens.add(const Token(T.eof));
  return tokens;
}