tokenize method

Iterable<Token> tokenize(
  1. String? chunk
)

Implementation

Iterable<Token> tokenize(String? chunk) sync* {
  if (chunk != null) {
    String _sequence = '';

    for (int i = 0; i < chunk.length; i++) {
      final char = chunk[i];

      if (delimiters.contains(char)) {
        if (_sequence.isNotEmpty) yield Token(_sequence);
        if (yieldTokens) yield Token(char);
        _sequence = '';
      } else {
        _sequence += char;
      }
    }

    if (_sequence.isNotEmpty) yield Token(_sequence);
  }
}