chunkTextFile static method

Stream<IChunk> chunkTextFile({
  1. required File file,
  2. required int maxChunkSize,
  3. required int maxPostOverlap,
})

Implementation

static Stream<IChunk> chunkTextFile({
  required File file,
  required int maxChunkSize,
  required int maxPostOverlap,
}) async* {
  _checkSizing(maxChunkSize, maxPostOverlap);
  yield* file
      .openRead()
      .transform(utf8.decoder)
      .transform(const LineSplitter())
      .map((v) => "$v\n")
      .transform(TXPieceSplitter(min(maxChunkSize, maxPostOverlap)))
      .transform(
        TXChunkMerger(maxSize: maxChunkSize, maxPost: maxPostOverlap),
      );
}