encodeSingleToken method

int encodeSingleToken(
  1. ByteArray bytes
)

Implementation

int encodeSingleToken(ByteArray bytes) {
  final token = encoder[bytes];
  if (token != null) return token;
  final specialToken = specialTokensEncoder[bytes.asString()];
  if (specialToken != null) return specialToken;

  throw TiktokenError("Error encoding single token: ${bytes.bytes}");
}