Tiktoken constructor
Tiktoken({})
Implementation
Tiktoken({
required this.name,
required this.patStr,
required this.mergeableRanks,
required this.specialTokens,
this.explicitNVocab,
}) {
maxTokenValue = max(
mergeableRanks.values.reduce(max),
specialTokens.values.reduce(max),
);
if (explicitNVocab != null) {
assert(mergeableRanks.length + specialTokens.length == explicitNVocab);
assert(maxTokenValue == explicitNVocab! - 1);
}
specialTokensSet = specialTokens.keys.toSet();
_coreBPE = CoreBPE.create(mergeableRanks, specialTokens, patStr);
}