scan method
bool
scan(
- TypstExternalScannerState state,
- TreeSitterExternalLexer lexer,
- List<
bool> validSymbols
override
Implementation
@override
bool scan(
TypstExternalScannerState state,
TreeSitterExternalLexer lexer,
List<bool> validSymbols,
) {
bool valid(TypstExternalToken token) =>
token.index < validSymbols.length && validSymbols[token.index];
bool result(TypstExternalToken token) {
lexer.setResultSymbol(symbols[token.index]);
return true;
}
bool accept(TypstExternalToken token) {
result(token);
lexer.markEnd();
return true;
}
lexer.markEnd();
if (valid(TypstExternalToken.recovery)) {
return result(TypstExternalToken.recovery);
}
if (valid(TypstExternalToken.immediateSet)) {
state.immediate = true;
return result(TypstExternalToken.immediateSet);
}
if (valid(TypstExternalToken.identifier) &&
(_isIdStart(lexer.lookahead) || lexer.lookahead == 0x5f)) {
_advance(lexer);
while (_isIdContinue(lexer.lookahead) || lexer.lookahead == 0x2d) {
_advance(lexer);
}
state.immediate = true;
return accept(TypstExternalToken.identifier);
}
if (valid(TypstExternalToken.label) &&
(_isIdContinue(lexer.lookahead) || lexer.lookahead == 0x2d)) {
_advance(lexer);
lexer.markEnd();
while (_isIdContinue(lexer.lookahead) ||
lexer.lookahead == 0x2d ||
lexer.lookahead == 0x2e ||
lexer.lookahead == 0x3a) {
final upTo = lexer.lookahead != 0x2e && lexer.lookahead != 0x3a;
_advance(lexer);
if (upTo) lexer.markEnd();
}
state.immediate = true;
return result(TypstExternalToken.label);
}
if (valid(TypstExternalToken.rawSpanBlob)) {
while (lexer.lookahead != 0x60 && !lexer.isEof) {
_advance(lexer);
}
return accept(TypstExternalToken.rawSpanBlob);
}
if (valid(TypstExternalToken.rawLang) &&
(lexer.lookahead == 0x5f || _isIdStart(lexer.lookahead))) {
_advance(lexer);
while (lexer.lookahead == 0x2d || _isIdContinue(lexer.lookahead)) {
_advance(lexer);
}
return accept(TypstExternalToken.rawLang);
}
if (valid(TypstExternalToken.rawBlckBlob)) {
while (!lexer.isEof) {
if (lexer.lookahead == 0x60) {
_advance(lexer);
if (lexer.lookahead == 0x60) {
_advance(lexer);
if (lexer.lookahead == 0x60) {
_advance(lexer);
return result(TypstExternalToken.rawBlckBlob);
}
}
}
_advance(lexer);
lexer.markEnd();
}
return false;
}
if (valid(TypstExternalToken.url)) {
state.worker.clear();
const brack = 0;
const paren = 1;
while (true) {
final codePoint = lexer.lookahead;
final length = state.worker.length;
if (_isUrlPlain(codePoint)) {
// Advance at the bottom, matching the source for-loop.
} else if (codePoint == 0x5b) {
state.worker.add(brack);
} else if (codePoint == 0x28) {
state.worker.add(paren);
} else if (codePoint == 0x5d &&
length > 0 &&
state.worker[length - 1] == brack) {
state.worker.removeLast();
} else if (codePoint == 0x29 &&
length > 0 &&
state.worker[length - 1] == paren) {
state.worker.removeLast();
} else {
return accept(TypstExternalToken.url);
}
_advance(lexer);
}
}
if (valid(TypstExternalToken.termination)) {
final termination = _termination(state, lexer, 0);
if (termination != _TypstTermination.none) {
if (termination == _TypstTermination.inclusive) {
_advance(lexer);
lexer.markEnd();
if (_containerAt(state, 0) != _containerBracket) {
_dedent(state);
}
}
_containerPop(state);
return result(TypstExternalToken.termination);
}
}
if (valid(TypstExternalToken.stringBlob) &&
!lexer.isEof &&
lexer.lookahead != 0x5c &&
lexer.lookahead != 0x22) {
_advance(lexer);
return accept(TypstExternalToken.stringBlob);
}
if (valid(TypstExternalToken.unit) && state.immediate) {
if (lexer.lookahead == 0x25) {
_advance(lexer);
return accept(TypstExternalToken.unit);
}
if (_isAsciiLower(lexer.lookahead)) {
_advance(lexer);
while (_isAsciiLower(lexer.lookahead)) {
_advance(lexer);
}
return accept(TypstExternalToken.unit);
}
}
if (valid(TypstExternalToken.inlinedItemEnd)) {
if (state.immediate) {
if (valid(TypstExternalToken.immediateBrack) &&
lexer.lookahead == 0x5b) {
return accept(TypstExternalToken.immediateBrack);
}
if (valid(TypstExternalToken.immediateParen) &&
lexer.lookahead == 0x28) {
return accept(TypstExternalToken.immediateParen);
}
}
if (valid(TypstExternalToken.elseToken)) {
if (_parseSpace(state, lexer)) return true;
if (_parseComment(state, lexer)) return true;
if (lexer.lookahead != 0x65) {
return result(TypstExternalToken.inlinedItemEnd);
}
_advance(lexer);
if (lexer.lookahead != 0x6c) {
return result(TypstExternalToken.inlinedItemEnd);
}
_advance(lexer);
if (lexer.lookahead != 0x73) {
return result(TypstExternalToken.inlinedItemEnd);
}
_advance(lexer);
if (lexer.lookahead != 0x65) {
return result(TypstExternalToken.inlinedItemEnd);
}
_advance(lexer);
if (!_isIdContinue(lexer.lookahead) && lexer.lookahead != 0x2d) {
return accept(TypstExternalToken.elseToken);
}
return result(TypstExternalToken.inlinedItemEnd);
}
if (lexer.lookahead == 0x2e) {
_advance(lexer);
if (lexer.lookahead == 0x5f) {
_advance(lexer);
if (_isIdContinue(lexer.lookahead) || lexer.lookahead == 0x2d) {
return false;
}
} else if (_isIdStart(lexer.lookahead)) {
return false;
}
return result(TypstExternalToken.inlinedItemEnd);
}
if (lexer.lookahead == 0x3b) {
_advance(lexer);
lexer.markEnd();
}
return result(TypstExternalToken.inlinedItemEnd);
}
if (_parseSpace(state, lexer)) return true;
if (lexer.lookahead == 0x2f) {
final column = lexer.column;
if (_parseComment(state, lexer)) return true;
if (valid(TypstExternalToken.mathFrac)) {
return accept(TypstExternalToken.mathFrac);
}
if (_isSp(lexer.lookahead) || _isLb(lexer.lookahead)) {
if (valid(TypstExternalToken.lineStartCheck)) {
state.lineStart = true;
return result(TypstExternalToken.lineStartCheck);
}
if (valid(TypstExternalToken.term) && state.lineStart) {
state.lineStart = false;
_redent(state, column);
return accept(TypstExternalToken.term);
}
}
}
if (valid(TypstExternalToken.lineStartCheck)) {
if (lexer.lookahead == 0x3d) {
while (lexer.lookahead == 0x3d) {
_advance(lexer);
}
if (_isSp(lexer.lookahead) || _isLb(lexer.lookahead) || lexer.isEof) {
state.lineStart = true;
}
} else if (lexer.lookahead == 0x2d || lexer.lookahead == 0x2b) {
_advance(lexer);
if (_isSp(lexer.lookahead) || _isLb(lexer.lookahead) || lexer.isEof) {
state.lineStart = true;
}
} else if (_isAsciiDigit(lexer.lookahead)) {
while (_isAsciiDigit(lexer.lookahead)) {
_advance(lexer);
}
if (lexer.lookahead == 0x2e) {
_advance(lexer);
if (_isSp(lexer.lookahead) || _isLb(lexer.lookahead) || lexer.isEof) {
state.lineStart = true;
}
}
}
return result(TypstExternalToken.lineStartCheck);
}
if (state.immediate) {
if (valid(TypstExternalToken.immediateBrack) && lexer.lookahead == 0x5b) {
return accept(TypstExternalToken.immediateBrack);
}
if (valid(TypstExternalToken.immediateParen) && lexer.lookahead == 0x28) {
return accept(TypstExternalToken.immediateParen);
}
if (valid(TypstExternalToken.immediateIdent) &&
(_isIdStart(lexer.lookahead) || lexer.lookahead == 0x5f)) {
return accept(TypstExternalToken.immediateIdent);
}
if (valid(TypstExternalToken.immediateMathCall) &&
lexer.lookahead == 0x28) {
return accept(TypstExternalToken.immediateMathCall);
}
if (valid(TypstExternalToken.immediateMathApply) &&
(lexer.lookahead == 0x28 ||
lexer.lookahead == 0x5b ||
lexer.lookahead == 0x7b)) {
return accept(TypstExternalToken.immediateMathApply);
}
if (valid(TypstExternalToken.immediateMathPrime) &&
lexer.lookahead == 0x27) {
_advance(lexer);
while (lexer.lookahead == 0x27) {
_advance(lexer);
}
return accept(TypstExternalToken.immediateMathPrime);
}
if (valid(TypstExternalToken.immediateMathField) &&
lexer.lookahead == 0x2e) {
_advance(lexer);
if (_isIdStart(lexer.lookahead)) {
return accept(TypstExternalToken.immediateMathField);
}
return false;
}
}
if (valid(TypstExternalToken.section)) {
state.containers.add(_containerSection + state.headingLevel);
return accept(TypstExternalToken.section);
}
if (valid(TypstExternalToken.barrier)) {
state.containers.add(_containerBarrier);
return accept(TypstExternalToken.barrier);
}
if (valid(TypstExternalToken.content) && lexer.lookahead == 0x5b) {
_advance(lexer);
state.indentation.add(0);
state.containers.add(_containerContent);
return accept(TypstExternalToken.content);
}
if (valid(TypstExternalToken.strong) && lexer.lookahead == 0x2a) {
_advance(lexer);
state.indentation.add(0);
state.containers.add(_containerStrong);
return accept(TypstExternalToken.strong);
}
if (valid(TypstExternalToken.emph) && lexer.lookahead == 0x5f) {
_advance(lexer);
state.indentation.add(0);
state.containers.add(_containerEmph);
return accept(TypstExternalToken.emph);
}
if (valid(TypstExternalToken.bracket) && lexer.lookahead == 0x5b) {
_advance(lexer);
state.containers.add(_containerBracket);
return accept(TypstExternalToken.bracket);
}
if (state.lineStart &&
valid(TypstExternalToken.head1) &&
lexer.lookahead == 0x3d) {
_advance(lexer);
var count = 1;
while (lexer.lookahead == 0x3d) {
_advance(lexer);
count++;
}
if (_isSp(lexer.lookahead) || _isLb(lexer.lookahead) || lexer.isEof) {
if (state.containers.isNotEmpty &&
_containerAt(state, 0) >= _containerSection) {
final level = _containerAt(state, 0) - _containerSection;
if (count <= level) {
_containerPop(state);
return result(TypstExternalToken.termination);
}
}
state.headingLevel = count & 0xff;
state.lineStart = false;
final token = switch (count) {
1 => TypstExternalToken.head1,
2 => TypstExternalToken.head2,
3 => TypstExternalToken.head3,
4 => TypstExternalToken.head4,
5 => TypstExternalToken.head5,
_ => TypstExternalToken.headP,
};
return accept(token);
}
return false;
}
if (state.lineStart && valid(TypstExternalToken.item)) {
if (lexer.lookahead == 0x2d || lexer.lookahead == 0x2b) {
final column = lexer.column;
_advance(lexer);
if (_isSp(lexer.lookahead) || _isLb(lexer.lookahead) || lexer.isEof) {
state.lineStart = false;
_redent(state, column);
return accept(TypstExternalToken.item);
}
return false;
}
if (_isAsciiDigit(lexer.lookahead)) {
final column = lexer.column;
_advance(lexer);
while (_isAsciiDigit(lexer.lookahead)) {
_advance(lexer);
}
if (lexer.lookahead == 0x2e) {
_advance(lexer);
if (_isSp(lexer.lookahead) || _isLb(lexer.lookahead) || lexer.isEof) {
state.lineStart = false;
_redent(state, column);
return accept(TypstExternalToken.item);
}
}
return false;
}
}
if (valid(TypstExternalToken.antiMarkup) && _isWordPart(lexer.lookahead)) {
_advance(lexer);
if (lexer.lookahead != 0x5f && lexer.lookahead != 0x2a) return false;
_advance(lexer);
if (!_isWordPart(lexer.lookahead)) return false;
_advance(lexer);
return accept(TypstExternalToken.antiMarkup);
}
if (valid(TypstExternalToken.mathIdent) && _isIdStart(lexer.lookahead)) {
_advance(lexer);
if (valid(TypstExternalToken.mathLetter) &&
(lexer.lookahead == 0x5f || !_isIdContinue(lexer.lookahead))) {
state.immediate = true;
return accept(TypstExternalToken.mathLetter);
}
while (lexer.lookahead != 0x5f && _isIdContinue(lexer.lookahead)) {
_advance(lexer);
}
state.immediate = true;
return accept(TypstExternalToken.mathIdent);
}
if (valid(TypstExternalToken.mathGroupEnd)) {
if (lexer.lookahead == 0x29 ||
lexer.lookahead == 0x5d ||
lexer.lookahead == 0x7d) {
_advance(lexer);
return accept(TypstExternalToken.mathGroupEnd);
}
if (lexer.lookahead == 0x24) {
return accept(TypstExternalToken.mathGroupEnd);
}
if (lexer.lookahead == 0x7c) {
_advance(lexer);
// The pinned C scanner uses `lex_advance_if` here. Despite its name,
// that macro fails the entire scan unless `|` is followed by `]`.
// A bare math `|` must fall through to the internal lexer as
// `_math_token_orph`, not terminate the surrounding group.
if (lexer.lookahead != 0x5d) return false;
_advance(lexer);
return accept(TypstExternalToken.mathGroupEnd);
}
}
if (valid(TypstExternalToken.elseToken)) {
if (valid(TypstExternalToken.blockedExprEnd)) {
lexer.setResultSymbol(symbols[TypstExternalToken.blockedExprEnd.index]);
while (_isSp(lexer.lookahead) || _isLb(lexer.lookahead)) {
_advance(lexer);
}
if (!_advanceIf(lexer, 0x65) ||
!_advanceIf(lexer, 0x6c) ||
!_advanceIf(lexer, 0x73) ||
!_advanceIf(lexer, 0x65)) {
return true;
}
if (lexer.lookahead == 0x5b ||
lexer.lookahead == 0x7b ||
lexer.lookahead == 0x2f ||
_isSp(lexer.lookahead)) {
return accept(TypstExternalToken.elseToken);
}
return true;
}
if (!_advanceIf(lexer, 0x65) ||
!_advanceIf(lexer, 0x6c) ||
!_advanceIf(lexer, 0x73) ||
!_advanceIf(lexer, 0x65)) {
return false;
}
if (lexer.lookahead == 0x5b ||
lexer.lookahead == 0x7b ||
lexer.lookahead == 0x2f ||
_isSp(lexer.lookahead)) {
return accept(TypstExternalToken.elseToken);
}
return false;
}
if (valid(TypstExternalToken.blockedExprEnd)) {
if (lexer.lookahead == 0x7d) {
return result(TypstExternalToken.blockedExprEnd);
}
if (lexer.lookahead == 0x3b) {
_advance(lexer);
return accept(TypstExternalToken.blockedExprEnd);
}
if (_isLb(lexer.lookahead)) {
lexer.markEnd();
_advance(lexer);
while (_isLb(lexer.lookahead) || _isSp(lexer.lookahead)) {
_advance(lexer);
}
if (lexer.lookahead == 0x2e) return false;
return result(TypstExternalToken.blockedExprEnd);
}
return false;
}
if (valid(TypstExternalToken.inlinedStmtEnd)) {
while (_isSp(lexer.lookahead)) {
_advance(lexer);
}
if (lexer.lookahead == 0x3b) {
_advance(lexer);
lexer.markEnd();
return result(TypstExternalToken.inlinedStmtEnd);
}
if (lexer.isEof || lexer.lookahead == 0x5d || _isLb(lexer.lookahead)) {
return result(TypstExternalToken.inlinedStmtEnd);
}
return false;
}
if (valid(TypstExternalToken.indent) || valid(TypstExternalToken.dedent)) {
while (_isLb(lexer.lookahead) || _isSp(lexer.lookahead)) {
_advance(lexer);
}
if (_termination(state, lexer, 0) != _TypstTermination.none) {
if (valid(TypstExternalToken.dedent)) {
_dedent(state);
return result(TypstExternalToken.dedent);
}
return false;
}
if (state.indentation.isEmpty) return false;
if (lexer.lookahead == 0x5d) {
if (valid(TypstExternalToken.dedent)) {
_dedent(state);
return result(TypstExternalToken.dedent);
}
return false;
}
// Tree-sitter 0.20's TSLexer.get_column rewinds to the line start and
// returns zero without recounting when the lexer is already at EOF.
// Preserve that quirk so a nonempty final line does not synthesize an
// indentation token after its last character.
final column = lexer.isEof ? 0 : lexer.column;
final indentation = state.indentation.last;
if (column > indentation) {
if (valid(TypstExternalToken.indent)) {
state.indentation.add(column);
return result(TypstExternalToken.indent);
}
return false;
}
if (column < indentation) {
if (state.indentation.length > 1 &&
valid(TypstExternalToken.redent) &&
column > state.indentation[state.indentation.length - 2]) {
_redent(state, column);
return result(TypstExternalToken.redent);
}
if (valid(TypstExternalToken.dedent)) {
_dedent(state);
return result(TypstExternalToken.dedent);
}
}
}
return false;
}