encode method

List<int> encode(String text)

Encode a string into a list of token ids. Applies merges in the order learned during training (lowest rank first).

Implementation

List<int> encode(String text) {
  var seq = List<int>.of(utf8.encode(text));
  if (seq.isEmpty) return seq;
  // Greedy pairwise merging using the merge-rank map.
  // Each pass finds the merge with the lowest rank among all
  // current adjacent pairs and applies it. Terminates when no
  // adjacent pair is in the vocab.
  while (seq.length >= 2) {
    int bestRank = 1 << 30;
    int bestI = -1;
    for (int i = 0; i + 1 < seq.length; i++) {
      final r = _mergeRank[_pairKey(seq[i], seq[i + 1])];
      if (r != null && r < bestRank) {
        bestRank = r;
        bestI = i;
      }
    }
    if (bestI == -1) break;
    final newId = 256 + bestRank;
    seq = [...seq.sublist(0, bestI), newId, ...seq.sublist(bestI + 2)];
  }
  return seq;
}