update method

List<LatexToken> update(String newText, TextEdit edit)

Incremental update: Update the token list based on an edit operation

newText The complete text after the edit edit Edit description Returns the updated full token list (including EOF)

Implementation

List<LatexToken> update(String newText, TextEdit edit) {
  final oldTokens = _cachedTokens;

  // Empty edit (no change)
  if (edit.delta == 0 && edit.oldLength == 0) {
    _textBuffer.clear();
    _textBuffer.write(newText);
    return _buildResult(newText.length);
  }

  // Old tokens are empty or completely rewritten -> full tokenization
  if (oldTokens.isEmpty ||
      (edit.startOffset == 0 && edit.oldEndOffset >= _textBuffer.length)) {
    return tokenize(newText);
  }

  final delta = edit.delta;

  // === Step 1: Determine the dirty region range in the old token list ===
  // Find the first token index affected by the edit
  final dirtyStart = _findFirstAffectedToken(edit.startOffset);
  // Find the last token index affected by the edit (in the old token list)
  final dirtyEnd = _findLastAffectedToken(edit.oldEndOffset);

  // === Step 2: Retain tokens before the dirty region (offsets unchanged) ===
  // Do not copy prefix, operate directly on oldTokens by index

  // === Step 3: Determine the text range for re-tokenization ===
  // Re-tokenization start point = start position of the first dirty token (in the new text)
  final retokenizeStart = dirtyStart < oldTokens.length
      ? oldTokens[dirtyStart].range.start
      : edit.startOffset;

  // Suffix tokens (old tokens after the edit region, to be reused after shifting)
  final suffixStartIdx = dirtyEnd + 1;
  // Using subList view rather than toList() copy (read-only usage)
  final suffixTokens = suffixStartIdx < oldTokens.length
      ? oldTokens.sublist(suffixStartIdx, oldTokens.length)
      : <LatexToken>[];

  // Expected start position of suffix tokens in the new text (used for convergence detection)
  final suffixExpectedStart = suffixTokens.isNotEmpty
      ? suffixTokens.first.range.start + delta
      : newText.length;

  // === Step 4: Re-tokenize the new text from retokenizeStart until convergence ===
  final newMiddleTokens = _retokenizeRegion(
    newText,
    retokenizeStart,
    suffixExpectedStart,
    suffixTokens,
    delta,
  );

  // === Step 5: Concatenate results ===
  // If re-tokenization exceeds the expected suffix start, the suffix must be trimmed
  final adjustedSuffixTokens = _adjustSuffix(
    suffixTokens,
    newMiddleTokens,
    delta,
    suffixExpectedStart,
  );

  // In-place construction: prefix(0..dirtyStart) + middle + adjustedSuffix
  final newTokens = <LatexToken>[];
  for (int i = 0; i < dirtyStart; i++) {
    newTokens.add(oldTokens[i]);
  }
  newTokens.addAll(newMiddleTokens);
  newTokens.addAll(adjustedSuffixTokens);
  _cachedTokens = newTokens;

  _textBuffer.clear();
  _textBuffer.write(newText);
  _invalidateResultCache();

  return _buildResult(newText.length);
}