fast_csv 0.1.10

Classic non-configurable CSV parser suitable for most use cases. Pretty fast parsing.

fast_csv

Classic non-configurable CSV parser suitable for most use cases. Pretty fast parsing.

Version: 0.1.10

Also demonstrates an example of creating a parser using parser_builder.
Creating a fast parser is very easy.
It may be a little slower, a little faster, or have the same performance as a handwritten one, but the time it takes to create it can be reduced by several times using parser_builder.

Example of the parser usage

import 'package:fast_csv/fast_csv.dart' as _fast_csv;

void main(List<String> args) {
  final result = _fast_csv.parse(_csv);
  print(result.join('\n'));
  for (final row in result) {
    final car = row[1];
    final price = num.parse(row[4]);
    print('$car $price');
  }
}

const _csv = '''
1997,Ford,E350,"ac, ""abs"", moon",3000.00
1999,Chevy,"Venture В«Extended EditionВ»","",4900.00
1996,Jeep,Grand Cherokee,"MUST SELL! air, moon roof, loaded",4799.00
''';

An example of using a configurable parser

This parser is slightly slower than the non-configurable parser.

import 'package:fast_csv/fast_csv_ex.dart' as _fast_csv_ex;

void main(List<String> args) {
  final result = _fast_csv_ex.parse(_csv, separator: ';');
  print(result.join('\n'));
  for (final row in result) {
    final car = row[1];
    final price = num.parse(row[4]);
    print('$car $price');
  }
}

const _csv = '''
1997;Ford;E350;"ac, ""abs"", moon";3000.00
1999;Chevy;"Venture В«Extended EditionВ»";"";4900.00
1996;Jeep;Grand Cherokee;"MUST SELL! air, moon roof, loaded";4799.00
''';

CSV parser declaration

tool/build_csv_parser.dart

import 'package:parser_builder/branch.dart';
import 'package:parser_builder/bytes.dart';
import 'package:parser_builder/character.dart';
import 'package:parser_builder/combinator.dart';
import 'package:parser_builder/fast_build.dart';
import 'package:parser_builder/multi.dart';
import 'package:parser_builder/parser_builder.dart';
import 'package:parser_builder/sequence.dart';
import 'package:parser_builder/transformers.dart';

void main(List<String> args) async {
  final context = Context();
  context.optimizeForSize = false;
  final filename = 'lib/fast_csv.dart';
  await fastBuild(context, [_parse], filename, header: __header);
}

const __header = r'''
// This code was generated by a tool.
// https://github.com/mezoni/parser_builder

// ignore_for_file: unused_local_variable

import 'package:source_span/source_span.dart';

/// Parses the CSV data and returns the result as a `List<List<String>>`.
/// - Will not parse numbers
/// - The character `,` is used as a field separator
/// - Line endings are `\n`, `\r\n` or '\r'
/// - The start and end of strings is the character `"`
/// - Escaping a character `"` in a string is parsed via sequence `""`
/// - Exception `FormatException` will be thrown if parsing fails
List<List<String>> parse(String source) {
  final state = StringState(source);
  final result = _parse(state);
  if (!state.ok) {
    final errors = Err.errorReport(state.error);
    final message = _errorMessage(source, errors);
    throw FormatException('\n$message');
  }

  return result!;
}

''';

const _chars = Named(
    '_chars',
    Many0(Alt([
      Satisfy(NotCharClass('["]')),
      Value(0x22, Tag('""')),
    ])));

const _closeQuote = Named('_closeQuote', Skip<String>([Tag('"'), _ws]));

const _eol = Named('_eol', Alt([LineEnding(), Tag('\r')]));

const _field = Named('_field', Alt([_string, _text]));

const _openQuote = Named('_openQuote', Skip<String>([_ws, Tag('"')]));

const _parse = Named('_parse', Terminated(_rows, Eof<String>()));

const _row = Named('_row', SeparatedList1(_field, Tag(',')));

const _rows = Named(
    '_rows',
    Terminated(
        SeparatedList1(_row, Skip<String>([_eol, Not(Eof())])), Opt(_eol)));

const _string = Named(
    '_string', Delimited(_openQuote, Map$(_chars, _toString), _closeQuote));

const _text = Named('_text', TakeWhile(NotCharClass('[,"] | #xA | #xD')));

const _toString = TX<List<int>, String>('=> String.fromCharCodes(x);');

const _ws = Named('_ws', SkipWhile(CharClass('#x20 | #x9')));

Performance tests

The files from the resource listed below were used to measure performance (excluding files with format violation).

https://people.sc.fsu.edu/~jburkardt/data/csv/csv.html

Source code for testing procedures.

void _test1(int count) {
  for (var i = 0; i < count; i++) {
    for (var k = 0; k < _tables.length; k++) {
      final table = _tables[k];
      final res = CsvToListConverter(
              allowInvalid: false, eol: '\n', shouldParseNumbers: false)
          .convert(table);
    }
  }
}

void _test2(int count) {
  for (var i = 0; i < count; i++) {
    for (var k = 0; k < _tables.length; k++) {
      final table = _tables[k];
      final res = _fast_csv_ex.parse(table);
    }
  }
}

void _test3(int count) {
  for (var i = 0; i < count; i++) {
    for (var k = 0; k < _tables.length; k++) {
      final table = _tables[k];
      final res = _fast_csv.parse(table);
    }
  }
}

Results:

List of files:
---------------
test_csv\addresses.csv
test_csv\airtravel.csv
test_csv\biostats.csv
test_csv\cities.csv
test_csv\crash_catalonia.csv
test_csv\deniro.csv
test_csv\example.csv
test_csv\faithful.csv
test_csv\ford_escort.csv
test_csv\freshman_kgs.csv
test_csv\freshman_lbs.csv
test_csv\grades.csv
test_csv\homes.csv
test_csv\hooke.csv
test_csv\hurricanes.csv
test_csv\hw_25000.csv
test_csv\lead_shot.csv
test_csv\letter_frequency.csv
test_csv\news_decline.csv
test_csv\nile.csv
test_csv\oscar_age_female.csv
test_csv\snakes_count_10.csv
test_csv\snakes_count_100.csv
test_csv\snakes_count_1000.csv
test_csv\snakes_count_10000.csv
test_csv\tally_cab.csv
test_csv\taxables.csv
test_csv\trees.csv
test_csv\zillow.csv
---------------
Parse in loop by 5 times:
Results:
Time passed: 0.000, Test 'csv': 3223.142 ms
Time passed: 3.225, Test 'fast_csv_ex': 1105.707 ms
Time passed: 4.331, Test 'fast_csv': 884.672 ms
Time passed: 5.215, Test 'csv': 3222.903 ms
Time passed: 8.438, Test 'fast_csv_ex': 1151.602 ms
Time passed: 9.590, Test 'fast_csv': 847.008 ms
Time passed: 10.437, Test 'csv': 3211.529 ms
Time passed: 13.649, Test 'fast_csv_ex': 1063.984 ms
Time passed: 14.713, Test 'fast_csv': 860.238 ms
Time passed: 15.573, Test 'csv': 3191.367 ms
Time passed: 18.764, Test 'fast_csv_ex': 1049.525 ms
Time passed: 19.814, Test 'fast_csv': 882.388 ms
Time passed: 20.696, Test 'csv': 3197.08 ms
Time passed: 23.894, Test 'fast_csv_ex': 1049.68 ms
Time passed: 24.943, Test 'fast_csv': 858.907 ms