preprocess method

EnglishPreprocessResult preprocess(String input)

Removes inline syntax from input and records each valid control.

Implementation

EnglishPreprocessResult preprocess(String input) {
  final text = _pythonLstrip(input);
  final result = StringBuffer();
  final sourceWords = <String>[];
  final controls = <int, EnglishInlineControl>{};
  var lastEnd = 0;

  for (final match in _linkPattern.allMatches(text)) {
    final preceding = text.substring(lastEnd, match.start);
    result.write(preceding);
    sourceWords.addAll(_pythonWhitespaceSplit(preceding));

    final source = match.group(1)!;
    final destination = match.group(2)!;
    final control = _parseControl(destination);
    if (control != null) {
      controls[sourceWords.length] = control;
    }
    result.write(source);
    sourceWords.add(source);
    lastEnd = match.end;
  }

  if (lastEnd < text.length) {
    final trailing = text.substring(lastEnd);
    result.write(trailing);
    sourceWords.addAll(_pythonWhitespaceSplit(trailing));
  }

  return EnglishPreprocessResult(
    text: result.toString(),
    sourceWords: sourceWords,
    controls: controls,
  );
}