tokenizer 0.0.1

Converts a stream of strings into stream of tokens splitted by delimiters

example/lib/main.dart

import 'dart:async';

import 'package:tokenizer/tokenizer.dart';

void main(List<String> args) async {
  final string = 'Hello, world';
  final tokenizer = Tokenizer({','});
  final c = StreamController<String>();

  c.add(string);
  c.close();

  final tokens = await c.stream.transform(tokenizer.streamTransformer).toList();
  print(tokens); // [Token("Hello"), Token(","), Token(" "), Token("world")]
}