tokenize method

List<String> tokenize(String line)

Returns a list of words from line that is separated by space or game defined separators.

Implementation

List<String> tokenize(String line) {
  final tokens = <String>[];
  final s = StringBuffer();

  for (int i = 0; i < line.length; i++) {
    final c = line.substring(i, i + 1);
//      if (i == line.length - 1){
//        s.add(c);
//        tokens.add(s.toString().trim());
//        s = StringBuffer();
//      }else
    if (c == ' ' && s.length > 0) {
      tokens.add(s.toString().trim());
      s.clear();
    } else if (Z.engine.mem.dictionary._separators.contains(c)) {
      if (s.length > 0) {
        tokens.add(s.toString().trim());
        s.clear();
      }
      tokens.add(c.trim());
    } else {
      s.write(c);
    }
  }

  if (s.length > 0) {
    tokens.add(s.toString().trim());
  }

  log.fine("Got tokens: $tokens");

  return tokens;
}