tokenize function
lang-core tokenize.
Implementation
List<Token> tokenize(String src) {
final tokens = <Token>[];
var i = 0;
final n = src.length;
while (i < n) {
while (i < n && (src[i] == ' ' || src[i] == '\t' || src[i] == '\r')) {
i++;
}
if (i >= n) break;
final c = src[i];
final next = i + 1 < n ? src[i + 1] : '';
final single = switch (c) {
'\n' => T.newline,
'(' => T.lParen,
')' => T.rParen,
'[' => T.lBracket,
']' => T.rBracket,
'{' => T.lBrace,
'}' => T.rBrace,
',' => T.comma,
':' => T.colon,
'.' => T.dot,
'?' => T.question,
'+' => T.plus,
'*' => T.star,
'/' => T.slash,
'%' => T.percent,
_ => null,
};
if (single != null) {
tokens.add(Token(single));
i++;
continue;
}
if (c == '=' || c == '!' || c == '>' || c == '<') {
final pair = next == '=';
tokens.add(
Token(switch (c) {
'=' => pair ? T.eqEq : T.equals,
'!' => pair ? T.notEq : T.not,
'>' => pair ? T.greaterEq : T.greater,
_ => pair ? T.lessEq : T.less,
}),
);
i += pair ? 2 : 1;
continue;
}
if (c == '&' || c == '|') {
tokens.add(Token(c == '&' ? T.and : T.or));
i += next == c ? 2 : 1;
continue;
}
if (c == '"') {
final start = i;
i++;
var closed = false;
while (i < n) {
if (src[i] == r'\') {
i += 2;
} else if (src[i] == '"') {
i++;
closed = true;
break;
} else {
i++;
}
}
if (i > n) i = n;
final raw = src.substring(start, i);
tokens.add(
Token(T.str, _decodeDoubleQuoted(closed ? raw : '$raw"', raw)),
);
continue;
}
if (c == "'") {
i++;
final out = StringBuffer();
while (i < n) {
if (src[i] == r'\') {
i++;
if (i < n) {
final esc = src[i];
out.write(switch (esc) {
'n' => '\n',
't' => '\t',
_ => esc,
});
i++;
}
} else if (src[i] == "'") {
i++;
break;
} else {
out.write(src[i]);
i++;
}
}
tokens.add(Token(T.str, out.toString()));
continue;
}
if (c == '-') {
final prev = tokens.isEmpty ? null : tokens.last;
final negativeLiteral =
!(prev != null && _valueEnds.contains(prev.t)) && _isDigit(next);
if (!negativeLiteral) {
tokens.add(const Token(T.minus));
i++;
continue;
}
}
if (_isDigit(c) || (c == '-' && _isDigit(next))) {
final start = i;
if (src[i] == '-') i++;
while (i < n && _isDigit(src[i])) {
i++;
}
if (i + 1 < n && src[i] == '.' && _isDigit(src[i + 1])) {
i++;
while (i < n && _isDigit(src[i])) {
i++;
}
}
if (i < n && (src[i] == 'e' || src[i] == 'E')) {
i++;
if (i < n && (src[i] == '+' || src[i] == '-')) i++;
while (i < n && _isDigit(src[i])) {
i++;
}
}
final text = src.substring(start, i);
tokens.add(Token(T.num, _parseNumberLiteral(text)));
continue;
}
if (c == r'$' && _isIdentStart(next)) {
final start = i;
i++;
while (i < n && _isIdentPart(src[i])) {
i++;
}
tokens.add(Token(T.stateVar, src.substring(start, i)));
continue;
}
if (_isIdentStart(c)) {
final start = i;
while (i < n && _isIdentPart(src[i])) {
i++;
}
final word = src.substring(start, i);
tokens.add(switch (word) {
'true' => const Token(T.trueKw),
'false' => const Token(T.falseKw),
'null' => const Token(T.nullKw),
_ => Token(_isUpper(c) ? T.type : T.ident, word),
});
continue;
}
if (c == '@' && _isIdentStart(next)) {
i++;
final start = i;
while (i < n && _isIdentPart(src[i])) {
i++;
}
tokens.add(Token(T.builtinCall, src.substring(start, i)));
continue;
}
i++;
}
tokens.add(const Token(T.eof));
return tokens;
}