tokenize function

List<Token> tokenize(
  1. String tagStr
)

Implementation

List<Token> tokenize(String tagStr) {
  int pos = 0;
  final tokens = <Token>[];
  while (pos < tagStr.length) {
    final (token, size) = nextToken(tagStr, pos);
    if (token.$1 != "SPACE") {
      tokens.add(token);
    }
    pos += size;
  }
  return tokens;
}