tokenizeExpression function

List<ExpressionToken> tokenizeExpression(String source, { ExpressionSyntax syntax = ExpressionSyntax.standard, })

source as tokens, ending with one of kind end.

Throws ExpressionSyntaxException for a character syntax does not take.

Implementation

List<ExpressionToken> tokenizeExpression(
  String source, {
  ExpressionSyntax syntax = ExpressionSyntax.standard,
}) {
  final tokens = <ExpressionToken>[];
  var line = 1;
  var column = 1;
  var at = 0;
  final single = '(),.+-*/${syntax.punctuation}';
  const pairs = <String>['<=', '>=', '==', '!=', '&&', '||'];
  const logical = '<>!?:';

  void advance(int count) {
    for (var i = 0; i < count; i++) {
      if (source[at + i] == '\n') {
        line++;
        column = 1;
      } else {
        column++;
      }
    }
    at += count;
  }

  while (at < source.length) {
    final ch = source[at];
    if (ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n') {
      advance(1);
      continue;
    }
    // `//` to the end of the line. A file is a thing an author comments,
    // and a language with no comments is one whose files grow a README
    // beside them.
    if (ch == '/' && at + 1 < source.length && source[at + 1] == '/') {
      while (at < source.length && source[at] != '\n') {
        advance(1);
      }
      continue;
    }

    final startLine = line;
    final startColumn = column;
    if (_isDigit(ch) ||
        (ch == '.' && at + 1 < source.length && _isDigit(source[at + 1]))) {
      var end = at;
      while (end < source.length &&
          (_isDigit(source[end]) || source[end] == '.')) {
        end++;
      }
      if (syntax.exponents &&
          end < source.length &&
          (source[end] == 'e' || source[end] == 'E')) {
        var digits = end + 1;
        if (digits < source.length &&
            (source[digits] == '+' || source[digits] == '-')) {
          digits++;
        }
        if (digits < source.length && _isDigit(source[digits])) {
          end = digits;
          while (end < source.length && _isDigit(source[end])) {
            end++;
          }
        }
      }
      final text = source.substring(at, end);
      final value = double.tryParse(text);
      if (value == null) {
        throw ExpressionSyntaxException(
          '"$text" is not a number.',
          startLine,
          startColumn,
        );
      }
      advance(end - at);
      tokens.add(
        ExpressionToken('number', text, startLine, startColumn, number: value),
      );
      continue;
    }
    if (_isNameStart(ch)) {
      var end = at;
      while (end < source.length && _isNamePart(source[end])) {
        end++;
      }
      final text = source.substring(at, end);
      advance(end - at);
      tokens.add(ExpressionToken('name', text, startLine, startColumn));
      continue;
    }
    if (syntax.units && ch == '[') {
      final close = source.indexOf(']', at);
      final newline = source.indexOf('\n', at);
      if (close < 0 || (newline >= 0 && newline < close)) {
        throw ExpressionSyntaxException(
          'a unit in "[" is never closed with "]".',
          startLine,
          startColumn,
        );
      }
      final text = source.substring(at + 1, close).trim();
      advance(close + 1 - at);
      tokens.add(ExpressionToken('unit', text, startLine, startColumn));
      continue;
    }
    if (syntax.logic && at + 1 < source.length) {
      final pair = source.substring(at, at + 2);
      if (pairs.contains(pair)) {
        advance(2);
        tokens.add(ExpressionToken(pair, pair, startLine, startColumn));
        continue;
      }
    }
    if (single.contains(ch) || (syntax.logic && logical.contains(ch))) {
      advance(1);
      tokens.add(ExpressionToken(ch, ch, startLine, startColumn));
      continue;
    }
    throw ExpressionSyntaxException(
      '"$ch" is not part of this language.',
      startLine,
      startColumn,
    );
  }
  tokens.add(ExpressionToken('end', '', line, column));
  return tokens;
}