consumeTextAtom method

TextToken? consumeTextAtom()

Consumes one Unicode character of the current text token

TeX's unbraced sub/superscripts only consume the following single token; normal text is merged during tokenization, so when parsing sub/superscripts, we need to extract just the first character from the merged text token

Implementation

TextToken? consumeTextAtom() {
  final token = peek();
  if (token is! TextToken) return null;

  // Check for UTF-16 surrogate pairs (high and low surrogates)
  int atomLength = 1;
  if (token.content.length > 1) {
    final c1 = token.content.codeUnitAt(0);
    final c2 = token.content.codeUnitAt(1);
    final isHighSurrogate = c1 >= 0xD800 && c1 <= 0xDBFF;
    final isLowSurrogate = c2 >= 0xDC00 && c2 <= 0xDFFF;

    if (isHighSurrogate && isLowSurrogate) {
      atomLength = 2;
    }
  }

  final atomEnd = token.range.start + atomLength;
  final atom = TextToken(
    token.content.substring(0, atomLength),
    range: SourceRange(token.range.start, atomEnd),
  );

  if (atomLength < token.content.length) {
    _mutableTokens ??= List<LatexToken>.of(initialTokens);
    _mutableTokens![_position] = atom;
    _mutableTokens!.insert(
      _position + 1,
      TextToken(
        token.content.substring(atomLength),
        range: SourceRange(atomEnd, token.range.end),
      ),
    );
  }

  advance();
  return atom;
}