scan function

List<Token> scan(
  1. String text, {
  2. dynamic sourceUrl,
})

Implementation

List<Token> scan(String text, {sourceUrl}) {
  var out = <Token>[];
  var scanner = SpanScanner(text, sourceUrl: sourceUrl);

  while (!scanner.isDone) {
    var potential = <Token>[];

    if (scanner.scan(_comment) ||
        scanner.scan(_whitespace) ||
        scanner.scan(',') ||
        scanner.scan('\ufeff')) continue;

    for (var pattern in _patterns.keys) {
      if (scanner.matches(pattern)) {
        var pat = _patterns[pattern];
        var scan = scanner.lastMatch?[0];
        if (pat != null && scan != null) {
          potential.add(Token(pat, scan, scanner.lastSpan));
        }
      }
    }

    if (potential.isEmpty) {
      var ch = String.fromCharCode(scanner.readChar());
      throw SyntaxError("Unexpected token '$ch'.", scanner.emptySpan);
    } else {
      // Choose longest token
      potential.sort((a, b) => b.text.length.compareTo(a.text.length));
      var chosen = potential.first;
      out.add(chosen);
      scanner.scan(chosen.text);
    }
  }

  return out;
}