Tiktoken constructor

Tiktoken({
  1. required String name,
  2. required String patStr,
  3. required Map<ByteArray, int> mergeableRanks,
  4. required Map<String, int> specialTokens,
  5. int? explicitNVocab,
})

Implementation

Tiktoken({
  required this.name,
  required this.patStr,
  required this.mergeableRanks,
  required this.specialTokens,
  this.explicitNVocab,
}) {
  maxTokenValue = max(
    mergeableRanks.values.reduce(max),
    specialTokens.values.reduce(max),
  );

  if (explicitNVocab != null) {
    assert(mergeableRanks.length + specialTokens.length == explicitNVocab);
    assert(maxTokenValue == explicitNVocab! - 1);
  }

  specialTokensSet = specialTokens.keys.toSet();

  _coreBPE = CoreBPE.create(mergeableRanks, specialTokens, patStr);
}