processOtherUnknownWords method

void processOtherUnknownWords(
  1. String other,
  2. List<String> tokens
)

Implementation

void processOtherUnknownWords(String other, List<String> tokens) {
  var mat = CharacterUtil.reSkip.allMatches(other);
  int offset = 0;
  for (var match in mat) {
    if (match.start > offset) {
      tokens.add(other.substring(offset, match.start));
    }
    tokens.add(match.group(0)!);
    offset = match.end;
  }
  if (offset < other.length) {
    tokens.add(other.substring(offset));
  }
}