splitSessionParseBatches function

List<SessionParseBatch> splitSessionParseBatches(List<String> lines, { required String filePath, required int firstLineNumber, bool shallowGiantCustoms = false, })

Splits lines into bounded transfer batches. Issue #199 E5: an oversize line becomes its own batch, so one huge record can never stall a batch beyond its own parse.

Implementation

List<SessionParseBatch> splitSessionParseBatches(
  List<String> lines, {
  required String filePath,
  required int firstLineNumber,
  bool shallowGiantCustoms = false,
}) {
  final batches = <SessionParseBatch>[];
  var start = 0;
  var weight = 0;
  for (var i = 0; i < lines.length; i++) {
    final lineWeight = lines[i].length;
    if (i > start &&
        (i - start >= sessionParseBatchMaxLines ||
            weight + lineWeight > sessionParseBatchMaxBytes)) {
      batches.add(
        _batch(
          lines,
          start,
          i,
          filePath,
          firstLineNumber,
          shallowGiantCustoms: shallowGiantCustoms,
        ),
      );
      start = i;
      weight = 0;
    }
    weight += lineWeight;
  }
  if (start < lines.length) {
    batches.add(
      _batch(
        lines,
        start,
        lines.length,
        filePath,
        firstLineNumber,
        shallowGiantCustoms: shallowGiantCustoms,
      ),
    );
  }
  return batches;
}