cut method

void cut(
  1. String sentence,
  2. List<String> tokens
)

Implementation

void cut(String sentence, List<String> tokens) {
  StringBuffer chinese = StringBuffer();
  StringBuffer other = StringBuffer();
  for (int i = 0; i < sentence.length; ++i) {
    String ch = sentence[i];
    if (CharacterUtil.isChineseLetter(ch)) {
      if (other.isNotEmpty) {
        processOtherUnknownWords(other.toString(), tokens);
        other.clear();
      }
      chinese.write(ch);
    } else {
      if (chinese.isNotEmpty) {
        viterbi(chinese.toString(), tokens);
        chinese.clear();
      }
      other.write(ch);
    }
  }
  if (chinese.isNotEmpty) {
    viterbi(chinese.toString(), tokens);
  } else {
    processOtherUnknownWords(other.toString(), tokens);
  }
}