getEncoding function

Tiktoken getEncoding(String encodingName)

Implementation

Tiktoken getEncoding(String encodingName) {
  if (_ENCODINGS.containsKey(encodingName)) {
    return _ENCODINGS[encodingName]!;
  }

  late CoreBPEConstructor constructor;

  switch (encodingName) {
    case 'cl100k_base':
      constructor = CoreBPEConstructor.cl100kBase();
      break;
    // case 'gpt2':
    //   constructor = CoreBPEConstructor.gpt2();
    //   break;
    // case 'r50k_base':
    //   constructor = CoreBPEConstructor.r50kBase();
    //   break;
    // case 'p50k_base':
    //   constructor = CoreBPEConstructor.p50kBase();
    //   break;
    // case 'p50k_edit':
    //   constructor = CoreBPEConstructor.p50kEdit();
    //   break;
    default:
      throw TiktokenError("Unknown encoding $encodingName");
  }

  final enc = Tiktoken(
    name: constructor.name,
    patStr: constructor.patStr,
    mergeableRanks: constructor.mergeableRanks,
    explicitNVocab: constructor.explicitNVocab,
    specialTokens: constructor.specialTokens,
  );

  _ENCODINGS[encodingName] = enc;

  return enc;
}