tokenize method

bool tokenize({ Duration? budget, })

Continues an update that ran out of budget, for at most budget, and returns isComplete.

Implementation

bool tokenize({Duration? budget}) {
  final pending = _pending;
  if (pending == null) return true;
  final clock = budget == null ? null : (Stopwatch()..start());
  final lines = pending.newLines;
  final old = pending.old;
  final shift = old.length - lines.length;
  var tokenized = 0;
  while (pending.next < lines.length) {
    if (clock != null && tokenized > 0 && clock.elapsed >= budget!) {
      return false;
    }
    final i = pending.next;
    final oldIndex = i + shift;
    if (i >= pending.suffixStart &&
        old[oldIndex].startState == pending.state) {
      _lines.addAll(old.skip(oldIndex));
      break;
    }
    final line = DocumentLine._tokenize(_grammar, lines[i], pending.state);
    _lines.add(line);
    // Kept current, so a line that throws is retried, not skipped.
    pending
      ..state = line.endState
      ..next = i + 1;
    tokenized++;
  }
  _pending = null;
  return true;
}