lexIntoTokens method

List<Node> lexIntoTokens()

Implementation

List<Node> lexIntoTokens() {
  final tokens = <Node>[];
  var isString = true;
  // Index specifying where to match from
  var startIndex = 0;

  // At every iteration, we should be able to match a new token until we
  // reach the end of the string. If for some reason we don't match a
  // token in any iteration of the loop, throw an error.
  while (startIndex < messageString.length) {
    Match? match;
    if (isString) {
      // TODO(thkim1011): Uncomment this when we add escaping as an option.
      // See https://github.com/flutter/flutter/issues/113455.
      // match = escapedString.matchAsPrefix(message, startIndex);
      // if (match != null) {
      //   final String string = match.group(0)!;
      //   tokens.add(Node.string(startIndex, string == "''" ? "'" : string.substring(1, string.length - 1)));
      //   startIndex = match.end;
      //   continue;
      // }
      match = unescapedString.matchAsPrefix(messageString, startIndex);
      if (match != null) {
        tokens.add(Node.string(startIndex, match.group(0)!));
        startIndex = match.end;
        continue;
      }
      match = brace.matchAsPrefix(messageString, startIndex);
      if (match != null) {
        tokens.add(Node.brace(startIndex, match.group(0)!));
        isString = false;
        startIndex = match.end;
        continue;
      }
      // Theoretically, we only reach this point because of unmatched single quotes because
      // 1. If it begins with single quotes, then we match the longest string contained in single quotes.
      // 2. If it begins with braces, then we match those braces.
      // 3. Else the first character is neither single quote or brace so it is matched by RegExp "unescapedString"
      throw L10nParserException(
        'ICU Lexing Error: Unmatched single quotes.',
        filename,
        messageId,
        messageString,
        startIndex,
      );
    } else {
      RegExp matcher;
      ST? matchedType;

      // Try to match tokens until we succeed
      for (matchedType in matchers.keys) {
        matcher = matchers[matchedType]!;
        match = matcher.matchAsPrefix(messageString, startIndex);
        if (match != null) {
          break;
        }
      }

      if (match == null) {
        match = brace.matchAsPrefix(messageString, startIndex);
        if (match != null) {
          tokens.add(Node.brace(startIndex, match.group(0)!));
          isString = true;
          startIndex = match.end;
          continue;
        }
        // This should only happen when there are special characters we are unable to match.
        throw L10nParserException('ICU Lexing Error: Unexpected character.',
            filename, messageId, messageString, startIndex);
      } else if (matchedType == ST.empty) {
        // Do not add whitespace as a token.
        startIndex = match.end;
        continue;
      } else {
        tokens.add(Node(matchedType!, startIndex, value: match.group(0)));
        startIndex = match.end;
        continue;
      }
    }
  }
  return tokens;
}