encode method

List<int> encode(
  1. String text
)

The IDs of text, without special tokens at the ends.

Implementation

List<int> encode(String text) {
  final out = <int>[];
  var start = 0;
  var i = 0;
  while (i < text.length) {
    if (_addedFirstUnits.contains(text.codeUnitAt(i))) {
      final match = _addedAt(text, i);
      if (match != null) {
        _encodePart(text.substring(start, i), out);
        out.add(match.$1);
        i += match.$2;
        start = i;
        continue;
      }
    }
    i++;
  }
  _encodePart(text.substring(start), out);
  return out;
}