add method

Future<void> add(WarcRecord record)

Implementation

Future<void> add(WarcRecord record) async {
  await _createBaseDirIfNeeded();
  await _updateOutputFile();
  final position = await _warcWriter!.add(record);

  if (_cdxjWriter != null && record.header.targetUri != null) {
    String? mime;
    String? digest;
    WarcHttpBlock? httpBlock;
    final ct = (record.header['Content-Type'] ?? '').split(';').first.trim();
    if (ct == 'application/http') {
      httpBlock = WarcHttpBlock.fromBlock(record.block);
      if (record.header.type == WarcTypes.response) {
        mime = httpBlock.payloadContentType?.split(';').first.trim();
      }
      digest = hex.encode(sha1.convert(httpBlock.payloadBytes).bytes);
    }

    final cdxj = CdxjRecord(
      url: record.header.targetUri!,
      timestamp: record.header.date,
      mime: mime ?? 'warc/${record.header.type}',
      filename: _currentFileName!,
      offset: position.compressed.offset,
      length: position.compressed.length,
      status: httpBlock?.statusCode,
      digest: digest ?? hex.encode(sha1.convert(record.block.bytes).bytes),
    );
    final storeCdxj = _storeCdxjFn == null ? true : await _storeCdxjFn!(cdxj);
    if (storeCdxj) {
      _cdxjWriter!.add(cdxj);
    }
  }
}