splitChunks method

List<ChunkWithIndex> splitChunks(
  1. String text,
  2. int maxLength,
  3. bool overlap,
  4. bool useHeuristic,
  5. bool useQuotesBracketsProcessing,
  6. int maxRecoverStep,
  7. int maxRecoverLength,
)

Implementation

List<ChunkWithIndex> splitChunks(
    String text, int maxLength, bool overlap, bool useHeuristic, bool useQuotesBracketsProcessing, int maxRecoverStep, int maxRecoverLength) {
  List<SentenceIndex> span = [];
  List<ChunkWithIndex> chunks = [];

  List<SentenceIndex> indices = splitSentencesIndex(text, useHeuristic, useQuotesBracketsProcessing, maxRecoverStep, maxRecoverLength);

  for (SentenceIndex index in indices) {
    if (span.length > 0) {
      if (index.getEnd() - span.elementAt(0).getStart() > maxLength) {
        chunks.add(getChunkWithIndex(span, text));
        if (overlap) {
          double halfSpanSize = span.length / 2.0;
          span = List.from(span.getRange((halfSpanSize - (halfSpanSize % 1)).toInt(), span.length));
        } else {
          span = [];
        }
      }
    }
    span.add(index);
  }

  chunks.add(getChunkWithIndex(span, text));
  return chunks;
}