Tokenizer.fromBytes constructor

Tokenizer.fromBytes(
  1. Uint8List json
)

Loads a tokenizer from the raw bytes of a tokenizer.json.

Throws FormatException if the bytes are not a valid tokenizer.

Implementation

factory Tokenizer.fromBytes(Uint8List json) {
  final buffer = calloc<Uint8>(json.length);
  buffer.asTypedList(json.length).setAll(0, json);
  final handle = tkFromBytes(buffer, json.length);
  calloc.free(buffer);
  if (handle == nullptr) {
    throw const FormatException('Not a valid tokenizer.json');
  }
  return Tokenizer._(handle);
}