detokenize method

String detokenize(
  1. List<int> tokens
)

Detokenizes the tokens.

Implementation

String detokenize(List<int> tokens) {
  if (tokens.isEmpty) return "";
  if (tokens[0] == bosTokenId) tokens = tokens.sublist(1);
  if (tokens.isNotEmpty && tokens.last == eosTokenId) {
    tokens = tokens.sublist(0, tokens.length - 1);
  }

  try {
    int paddingIndex = tokens.indexWhere((element) => element == padTokenId);
    if (paddingIndex != -1) tokens = tokens.sublist(0, paddingIndex);
  } catch (e) {
    // Ignore errors if pad token not found
  }

  List<String> detokenizedTokens =
      tokens.map((token) => i2s[token]!).toList();
  return tokensToText(detokenizedTokens);
}