tokenize function

List<Token> tokenize(String input, Settings settings)

Implementation

List<Token> tokenize(String input, Settings settings) {
  int line = 1;
  int column = 1;
  int index = 0;
  List<Token> tokens = <Token>[];

  while (index < input.length) {
    final whitespace = parseWhitespace(input, index, line, column);
    if (whitespace != null) {
      index = whitespace.index;
      line = whitespace.line;
      column = whitespace.column;
      continue;
    }

    final token = _parseToken(input, index, line, column);

    if (token != null) {
      final src = settings.source ?? "";
      token.loc = Location.create(
          line, column, index, token.line, token.column, token.index, src);
      tokens.add(token);
      index = token.index;
      line = token.line;
      column = token.column;
    } else {
      final src = settings.source ?? "";
      final msg = unexpectedSymbol(
          substring(input, index, index + 1), src, line, column);
      throw new JSONASTException(msg, input, src, line, column);
    }
  }
  return tokens;
}