scan function
Implementation
List<Token> scan(String text, {sourceUrl}) {
var out = <Token>[];
var scanner = SpanScanner(text, sourceUrl: sourceUrl);
while (!scanner.isDone) {
var potential = <Token>[];
if (scanner.scan(_comment) ||
scanner.scan(_whitespace) ||
scanner.scan(',') ||
scanner.scan('\ufeff')) continue;
for (var pattern in _patterns.keys) {
if (scanner.matches(pattern)) {
var pat = _patterns[pattern];
var scan = scanner.lastMatch?[0];
if (pat != null && scan != null) {
potential.add(Token(pat, scan, scanner.lastSpan));
}
}
}
if (potential.isEmpty) {
var ch = String.fromCharCode(scanner.readChar());
throw SyntaxError("Unexpected token '$ch'.", scanner.emptySpan);
} else {
// Choose longest token
potential.sort((a, b) => b.text.length.compareTo(a.text.length));
var chosen = potential.first;
out.add(chosen);
scanner.scan(chosen.text);
}
}
return out;
}