tokenizer_parser 0.1.3

A lightweight tokenizer and parser-composition toolkit for Dart, designed to build custom language grammars and structured token streams.

example/main.dart

import 'package:tokenizer_parser/tokenizer_parser.dart';

import 'ql_lang.dart';

void main() {
  const sample = 'query User { user(id: 1) { name } }';
  final (tokens, remaining) =
      Tokenizer.tokenize(sample, QlLang.lang, QlLang.ignore);

  print('tokens: ${tokens.length}');
  print('remaining segments: ${remaining.length}');
}