tokenize function

List<Token> tokenize(String input, Settings settings)

Implementation

List<Token> tokenize(String input, Settings settings) {
  var line = 1;
  var column = 1;
  var index = 0;
  var tokens = <Token>[];

  while (index < input.length) {
    final whitespace = parseWhitespace(input, index, line, column);
    if (whitespace != null) {
      index = whitespace.index;
      line = whitespace.line;
      column = whitespace.column;
      continue;
    }

    final token = _parseToken(input, index, line, column);

    if (token != null) {
      token.loc = Location.create(line, column, index, token.line, token.column,
          token.index, settings.source);
      tokens.add(token);
      index = token.index;
      line = token.line;
      column = token.column;
    } else {
      final msg = unexpectedSymbol(
          substring(input, index, index + 1), settings.source, line, column);
      throw JSONASTException(msg, input, settings.source, line, column);
    }
  }
  return tokens;
}