encode method

@override TokenizedInput encode(String prefix, String text)

Tokenizes (prefix + text) per this tokenizer's convention (BOS/EOS wrap for SentencePiece, CLS/.../SEP for WordPiece).

Implementation

@override
TokenizedInput encode(String prefix, String text) {
  final normalized = _normalize(prefix + text);
  final words = _preTokenize(normalized);

  final ids = <int>[clsId];
  for (final word in words) {
    ids.addAll(_wordPieceEncode(word));
  }
  ids.add(sepId);
  return TokenizedInput(
    ids: ids,
    attentionMask: List<int>.filled(ids.length, 1),
    tokenTypeIds: List<int>.filled(ids.length, 0),
  );
}