rumil_tokens 0.1.0

Lossless source code tokenizer built on Rumil parser combinators. Classified token spans for syntax highlighting. Built-in grammars for Dart, Scala, YAML, JSON, and shell.

example/example.dart

import 'package:rumil_tokens/rumil_tokens.dart';

void main() {
  const source = '''
void main() {
  final x = 42;
  // greeting
  print("hello \$x");
}
''';

  final tokens = tokenize(source, dart);

  for (final token in tokens) {
    final kind = switch (token) {
      Keyword() => 'keyword',
      TypeName() => 'type',
      StringLit() => 'string',
      NumberLit() => 'number',
      Comment() => 'comment',
      Annotation() => 'annotation',
      Punctuation() => 'punct',
      Operator() => 'op',
      Variable() => 'var',
      Identifier() => 'ident',
      Whitespace() => 'ws',
      Plain() => 'plain',
    };
    if (token is! Whitespace) {
      print('$kind: ${token.text}');
    }
  }
}