scanTokensAsync method

Future<List<Token>> scanTokensAsync(
  1. String source
)

Implementation

Future<List<Token>> scanTokensAsync(String source) async {
  this.source = source;
  List<Token> allTokens = [];

  int numChunks = (source.length / chunkSize).ceil();
  List<Future<List<Token>>> futures = [];

  for (int i = 0; i < numChunks; i++) {
    int start = i * chunkSize;
    int end = (i + 1) * chunkSize;
    if (end > source.length) end = source.length;

    futures.add(_processChunk(source.substring(start, end), start));
  }

  List<List<Token>> results = await Future.wait(futures);
  for (var tokens in results) {
    allTokens.addAll(tokens);
  }

  allTokens.sort((a, b) => a.line.compareTo(b.line));
  return allTokens;
}