encodingForModel function
Returns the encoding used by a model
Implementation
Tiktoken encodingForModel(String modelName) {
String? encodingName;
if (_MODEL_TO_ENCODING.containsKey(modelName)) {
encodingName = _MODEL_TO_ENCODING[modelName]!;
} else {
for (var item in _MODEL_PREFIX_TO_ENCODING.entries) {
final modelPrefix = item.key;
final modelEncodingName = item.value;
if (modelName.startsWith(modelPrefix)) {
return getEncoding(modelEncodingName);
}
}
}
if (encodingName == null) {
throw TiktokenError(
"Could not automatically map $modelName to a tokeniser. "
"Please use `getEncoding` to explicitly get the tokeniser you expect.");
}
return getEncoding(encodingName);
}