tiktoken/src/core_bpe_constructor library

Classes

CoreBPEConstructor

Constants

END_OF_PROMPT → const String

END_OF_TEXT → const String

FIM_MIDDLE → const String

FIM_PREFIX → const String

FIM_SUFFIX → const String