toBytes method

Uint8List toBytes()

Serializes the store to the rag_kit binary format.

The format is compact and self-contained (all integers and floats are little-endian):

bytes 0-3    magic "RGK1"
bytes 4-7    embedding dimension, uint32
bytes 8-11   document count, uint32
per document:
  uint32 byte length + UTF-8 bytes   id
  uint32 byte length + UTF-8 bytes   text
  uint32 byte length + UTF-8 bytes   metadata as JSON
  dimension x 4 bytes                embedding as float32

Document metadata must be JSON-encodable; otherwise a JsonUnsupportedObjectError is thrown.

Implementation

Uint8List toBytes() {
  final builder = BytesBuilder(copy: false);
  final header = ByteData(12)
    ..setUint8(0, _magic0)
    ..setUint8(1, _magic1)
    ..setUint8(2, _magic2)
    ..setUint8(3, _magic3)
    ..setUint32(4, _dimension ?? 0, Endian.little)
    ..setUint32(8, _entries.length, Endian.little);
  builder.add(header.buffer.asUint8List());
  for (final entry in _entries.values) {
    _writeString(builder, entry.document.id);
    _writeString(builder, entry.document.text);
    _writeString(builder, jsonEncode(entry.document.metadata));
    final vector = entry.vector;
    final vectorData = ByteData(vector.length * 4);
    for (var i = 0; i < vector.length; i++) {
      vectorData.setFloat32(i * 4, vector[i], Endian.little);
    }
    builder.add(vectorData.buffer.asUint8List());
  }
  return builder.toBytes();
}