WolfDefenderTokenizer.fromBytes constructor

WolfDefenderTokenizer.fromBytes(Uint8List bytes)

Implementation

factory WolfDefenderTokenizer.fromBytes(Uint8List bytes) {
  final data = ByteData.sublistView(bytes);
  var offset = 0;
  int u32() {
    final value = data.getUint32(offset, Endian.little);
    offset += 4;
    return value;
  }

  if (bytes.length < 8 ||
      u32() != wolfDefenderTokenizerMagic ||
      u32() != wolfDefenderTokenizerVersion) {
    throw const FormatException('Not a Wolf Defender tokenizer file');
  }
  final count = u32();
  if (count > _pairBase) {
    throw FormatException('$count tokens; at most $_pairBase');
  }
  final tokens = List<String>.filled(count, '');
  final ids = <String, int>{};
  for (var id = 0; id < count; id++) {
    final length = data.getUint16(offset, Endian.little);
    offset += 2;
    final token = _decodeKeepingBom(
      Uint8List.sublistView(bytes, offset, offset + length),
    );
    offset += length;
    tokens[id] = token;
    ids[token] = id;
  }
  final mergeCount = u32();
  final mergeRanks = <int, int>{};
  final mergeResults = Int32List(mergeCount);
  for (var rank = 0; rank < mergeCount; rank++) {
    final left = u32(), right = u32();
    if (left >= count || right >= count) {
      throw FormatException('Merge $rank has an unknown ID');
    }
    final result = ids[tokens[left] + tokens[right]];
    if (result == null) {
      throw FormatException('Merge $rank has no result token');
    }
    // The first (lowest) rank of a pair wins, as in `tokenizers`.
    mergeRanks.putIfAbsent(_pair(left, right), () => rank);
    mergeResults[rank] = result;
  }
  final addedCount = u32();
  final added = <String, int>{};
  final firstUnits = <int>{};
  var maxLength = 0;
  for (var i = 0; i < addedCount; i++) {
    final id = u32();
    if (id >= count) throw FormatException('Added token $id is unknown');
    final token = tokens[id];
    if (token.isEmpty) continue;
    added[token] = id;
    firstUnits.add(token.codeUnitAt(0));
    if (token.length > maxLength) maxLength = token.length;
  }
  if (offset != bytes.length) {
    throw FormatException('${bytes.length - offset} bytes after the end');
  }
  final unk = ids['<unk>'];
  if (unk == null) throw const FormatException('No <unk> token');
  final byteTokens = Int32List(256);
  for (var b = 0; b < 256; b++) {
    byteTokens[b] =
        ids['<0x${b.toRadixString(16).toUpperCase().padLeft(2, '0')}>'] ?? -1;
  }
  return WolfDefenderTokenizer._(
    tokens,
    ids,
    mergeRanks,
    mergeResults,
    added,
    firstUnits,
    maxLength,
    byteTokens,
    unk,
  );
}