readJSONLFile<T> function

Future<List<T>> readJSONLFile<T>(String filePath)

Read and parse a JSONL file.

Implementation

Future<List<T>> readJSONLFile<T>(String filePath) async {
  final file = File(filePath);
  final stat = await file.stat();
  if (stat.size <= _maxJsonlReadBytes) {
    final content = await file.readAsString();
    return parseJSONL<T>(content);
  }

  // For large files, read the tail
  final raf = await file.open(mode: FileMode.read);
  try {
    final offset = stat.size - _maxJsonlReadBytes;
    await raf.setPosition(offset);
    final bytes = await raf.read(_maxJsonlReadBytes);
    final content = utf8.decode(bytes, allowMalformed: true);

    // Skip the first partial line
    final newlineIndex = content.indexOf('\n');
    if (newlineIndex != -1 && newlineIndex < content.length - 1) {
      return parseJSONL<T>(content.substring(newlineIndex + 1));
    }
    return parseJSONL<T>(content);
  } finally {
    await raf.close();
  }
}