tokenize function

List<Token> tokenize(
  1. StringStream input
)

Implementation

List<Token> tokenize(StringStream input) {
  final res = <Token>[];

  while (input.isNotEmpty) {
    final char = input.head();

    if (char == ' ') {
      input.eatChar();
      continue;
    } else if (_isAlpha(char) || char == '_') {
      res.add(_lexId(input));
    } else {
      res.add(_lexSymbol(input));
    }
  }

  return res;
}