bert_tokenizer 1.2.0

Pure Dart WordPiece tokenizer for BERT NLP models. Completely independent of Flutter.

example/bert_tokenizer_example.dart

import 'dart:io';
import 'package:bert_tokenizer/bert_tokenizer.dart';

void main() {
  // Read vocab file.
  final vocabFile = File('vocab.txt');

  if (!vocabFile.existsSync()) {
    print('Missing vocab file.');
    return;
  }

  // Read string content.
  final vocabContent = vocabFile.readAsStringSync();

  // Initialize the tokenizer.
  final tokenizer = BertTokenizer.fromStringContent(vocabContent);

  // Define input text.
  final text = "Dart is awesome!";

  /// First use case.
  // Get string tokens.
  final tokens = tokenizer.tokenize(text);

  // Print string tokens.
  print('Tokens: $tokens');

  /// Second use case.
  // Define max length.
  final maxLength = 10;

  // Prepare model inputs.
  final input = tokenizer.prepareNerInput(text, maxLength);

  // Print input IDs.
  print('IDs: ${input.inputIds}');

  // Print input masks.
  print('Masks: ${input.inputMask}');

  // Print segment IDs.
  print('Segments: ${input.segmentIds}');

  /// Third use case.
  // Convert IDs back.
  final reversed = tokenizer.convertIdsToTokens(input.inputIds);

  // Print reversed tokens.
  print('Reversed: $reversed');
}