textToTokens method

@override List<String> textToTokens(String text, { bool boxPosition = false, })

Converts text to tokens.

Implementation

@override
List<String> textToTokens(String text, {bool boxPosition = false}) {
  // Preprocessing text
  text = text.replaceAllMapped(
      RegExp(r'([MLBR])'), (match) => ' ${match.group(0)}'); // Add spaces
  text = text.replaceAll(RegExp(r'\bA\w*\b'), ''); // Remove sign prefix
  text = text.replaceAll(RegExp(r' +'), ' '); // Remove consecutive spaces
  text = text.trim();

  List<Sign> signs = [
    for (String f in text.split(" ")) fswToSign(f)
  ]; // Splitting text into FSW signs and converting them to Sign objects
  List<String> tokens = [];

  // Tokenizing each sign
  for (Sign sign in signs) {
    tokens.addAll(tokenizeSymbol(sign.box,
        boxPosition: boxPosition)); // Tokenizing box symbol
    for (SignSymbol symbol in sign.symbols) {
      tokens.addAll(tokenizeSymbol(symbol)); // Tokenizing other symbols
    }
  }
  return tokens;
}