SegmentReader constructor

SegmentReader(Uint8List buffer, { int offset = 0, int? length, bool verify = true, bool verifyBlocksOnRead = true, })

Parses a segment starting at offset.

With verifyBlocksOnRead every document is preceded by a one-time check of its block's CRC: the first read that touches a block pays one checksum pass over that block, later reads only a bitmap lookup. Without it a single flipped bit inside a document body would be returned as data.

Implementation

factory SegmentReader(
  Uint8List buffer, {
  int offset = 0,
  int? length,
  bool verify = true,
  bool verifyBlocksOnRead = true,
}) {
  final end = length == null ? buffer.length : offset + length;
  if (end - offset < segmentHeaderSize + segmentFooterSize) {
    throw VxsFormatException('segment too short', offset: offset);
  }
  if (buffer[offset] != segmentMagic) {
    throw VxsFormatException('bad segment magic', offset: offset);
  }
  if (buffer[offset + 1] != segmentVersion) {
    throw VxsFormatException('unsupported segment version', offset: offset);
  }
  final docCount = VxsLe.u32(buffer, offset + 4);
  final blockCount = VxsLe.u32(buffer, offset + 8);
  final footerStart = end - segmentFooterSize;
  if (VxsLe.u32(buffer, footerStart) != segmentFooterMagic) {
    throw VxsFormatException('bad segment footer magic', offset: footerStart);
  }
  final tableOffset = VxsLe.u32(buffer, footerStart + 4);
  final indexOffset = VxsLe.u32(buffer, footerStart + 8);
  final indexLength = VxsLe.u32(buffer, footerStart + 12);
  final bloomOffset = VxsLe.u32(buffer, footerStart + 16);
  final bloomLength = VxsLe.u32(buffer, footerStart + 20);
  final storedCrc = VxsLe.u32(buffer, footerStart + 24);
  if (tableOffset < segmentHeaderSize ||
      indexOffset + indexLength > footerStart ||
      bloomOffset + bloomLength > footerStart ||
      tableOffset + blockCount * 4 > footerStart) {
    throw VxsFormatException(
      'segment metadata out of bounds',
      offset: offset,
    );
  }
  // The metadata checksum covers the block table, index and bloom filter but
  // not the segment header, where blockCount lives. Requiring the three
  // regions to tile exactly is what keeps a flipped blockCount from steering
  // the block table into the index and bypassing the on-read block checksum.
  if (tableOffset + blockCount * 4 != indexOffset ||
      indexOffset + indexLength != bloomOffset ||
      bloomOffset + bloomLength != footerStart) {
    throw VxsFormatException(
      'segment metadata regions do not tile the file',
      offset: offset,
    );
  }
  if (verify) {
    final actual = VxsCrc32c.compute(buffer, tableOffset, footerStart);
    if (actual != storedCrc) {
      throw VxsFormatException(
        'segment metadata checksum mismatch: stored '
        '0x${storedCrc.toRadixString(16)}, computed '
        '0x${actual.toRadixString(16)}',
        offset: footerStart,
      );
    }
  }
  if (indexLength < 4) {
    throw VxsFormatException('empty segment index', offset: indexOffset);
  }
  final indexCount = VxsLe.u32(buffer, indexOffset);
  if (4 + (indexCount + 1) * 4 > indexLength) {
    throw VxsFormatException(
      'segment index is truncated',
      offset: indexOffset,
    );
  }
  final entryOffsetsStart = indexOffset + 4;
  final entriesStart = entryOffsetsStart + (indexCount + 1) * 4;
  final bloom = BloomFilter.parse(buffer, bloomOffset).filter;
  return SegmentReader._(
    buffer,
    offset,
    end,
    docCount,
    blockCount,
    tableOffset,
    indexOffset,
    indexLength,
    bloomOffset,
    bloomLength,
    indexCount,
    entryOffsetsStart,
    entriesStart,
    bloom,
    verifyBlocksOnRead ? Uint8List((blockCount + 7) >> 3) : null,
  );
}