encodingForModel function

Tiktoken encodingForModel(
  1. String modelName
)

Returns the encoding used by a model

Implementation

Tiktoken encodingForModel(String modelName) {
  String? encodingName;

  if (_MODEL_TO_ENCODING.containsKey(modelName)) {
    encodingName = _MODEL_TO_ENCODING[modelName]!;
  } else {
    for (var item in _MODEL_PREFIX_TO_ENCODING.entries) {
      final modelPrefix = item.key;
      final modelEncodingName = item.value;

      if (modelName.startsWith(modelPrefix)) {
        return getEncoding(modelEncodingName);
      }
    }
  }

  if (encodingName == null) {
    throw TiktokenError(
        "Could not automatically map $modelName to a tokeniser. "
        "Please use `getEncoding` to explicitly get the tokeniser you expect.");
  }

  return getEncoding(encodingName);
}