tokenize function

List<Token> tokenize(String input, Settings settings)

Implementation

List<Token> tokenize(String input, Settings settings) {
  var line = 1;
  var column = 1;
  var index = 0;
  var tokens = <Token>[];

  while (index < input.length) {
    final whitespace = parseWhitespace(input, index, line, column);
    if (whitespace != null) {
      index = whitespace.index;
      line = whitespace.line;
      column = whitespace.column;
      continue;
    }

    final token = _parseToken(input, index, line, column);

    if (token != null) {
      token.loc = Location.create(
        line,
        column,
        index,
        token.line,
        token.column,
        token.index,
        settings.source,
      );
      tokens.add(token);
      index = token.index;
      line = token.line;
      column = token.column;
    } else {
      final msg = unexpectedSymbol(
        substring(input, index, index + 1),
        settings.source,
        line,
        column,
      );
      throw JSONASTException(msg, input, settings.source, line, column);
    }
  }
  return tokens;
}