getEncoding function

Tiktoken getEncoding(String encodingName)

Implementation

Tiktoken getEncoding(String encodingName) {
  if (_ENCODINGS.containsKey(encodingName)) {
    return _ENCODINGS[encodingName]!;
  }

  late CoreBPEConstructor constructor;

  switch (encodingName) {
    case "cl100k_base":
      constructor = CoreBPEConstructor.cl100kBase();
      break;
    case "gpt2":
      constructor = CoreBPEConstructor.gpt2();
      break;
    case "r50k_base":
      constructor = CoreBPEConstructor.r50kBase();
      break;
    case "p50k_base":
      constructor = CoreBPEConstructor.p50kBase();
      break;
    case "p50k_edit":
      constructor = CoreBPEConstructor.p50kEdit();
      break;
    default:
      throw TiktokenError("Unknown encoding $encodingName");
  }

  final enc = Tiktoken(
    name: constructor.name,
    patStr: constructor.patStr,
    mergeableRanks: constructor.mergeableRanks,
    explicitNVocab: constructor.explicitNVocab,
    specialTokens: constructor.specialTokens,
  );

  _ENCODINGS[encodingName] = enc;

  return enc;
}