processOtherUnknownWords method
Implementation
void processOtherUnknownWords(String other, List<String> tokens) {
var mat = CharacterUtil.reSkip.allMatches(other);
int offset = 0;
for (var match in mat) {
if (match.start > offset) {
tokens.add(other.substring(offset, match.start));
}
tokens.add(match.group(0)!);
offset = match.end;
}
if (offset < other.length) {
tokens.add(other.substring(offset));
}
}