getEncoder static method

TiktokenEncoder getEncoder(
  1. TiktokenEncodingType encodingType
)

Returns the tiktoken encoding for the given encodingType.

Implementation

static TiktokenEncoder getEncoder(TiktokenEncodingType encodingType) {
  if (_CACHE.containsKey(encodingType)) {
    return _CACHE[encodingType]!;
  }

  CoreBPEConstructor constructor = switch (encodingType) {
    TiktokenEncodingType.cl100k_base => CoreBPEConstructor.cl100kBase(),
    TiktokenEncodingType.o200k_base => CoreBPEConstructor.o200kBase(),
  };

  final enc = TiktokenEncoder(
    name: constructor.name,
    patternStr: constructor.patternStr,
    mergeableRanks: constructor.mergeableRanks,
    explicitNVocab: constructor.explicitNVocab,
    specialTokens: constructor.specialTokens,
  );

  _CACHE[encodingType] = enc;

  return enc;
}