dvChunkImportRows function

List<DVImportChunkPlan> dvChunkImportRows(String content, { required int chunkSize, required bool hasHeader, })

Splits content into chunks of at most chunkSize rows.

With hasHeader, the first non-blank line is the header: it is not imported as a row, and it is attached to every chunk.

Implementation

List<DVImportChunkPlan> dvChunkImportRows(
  String content, {
  required int chunkSize,
  required bool hasHeader,
}) {
  if (chunkSize < 1) {
    // Zero would loop forever and negative would throw somewhere less obvious
    // than here.
    throw ArgumentError.value(
      chunkSize,
      'chunkSize',
      'chunkSize must be positive',
    );
  }

  // Blank lines are dropped rather than imported as empty records. The
  // trailing \r matters too: left on the end of the last field it silently
  // becomes part of the value, so "Ada\r" is imported and never matches
  // "Ada".
  final List<String> lines = const LineSplitter()
      .convert(content)
      .map((String line) => line.endsWith('\r')
          ? line.substring(0, line.length - 1)
          : line)
      .where((String line) => line.trim().isNotEmpty)
      .toList(growable: false);

  if (lines.isEmpty) return const <DVImportChunkPlan>[];

  final String? header = hasHeader ? lines.first : null;
  final int firstDataIndex = hasHeader ? 1 : 0;
  if (firstDataIndex >= lines.length) {
    // A file with only a header. No chunks rather than one empty chunk: a
    // worker handed nothing to do is a job dispatched for no reason.
    return const <DVImportChunkPlan>[];
  }

  final List<DVImportChunkPlan> chunks = <DVImportChunkPlan>[];
  for (int index = firstDataIndex; index < lines.length; index += chunkSize) {
    final int end = math.min(index + chunkSize, lines.length);
    chunks.add(DVImportChunkPlan(
      // 1-based over the source file, header included, so the number points
      // at the line someone would open the file to.
      startRow: index + 1,
      rows: lines.sublist(index, end),
      header: header,
    ));
  }
  return chunks;
}