fromJsonString static method

WordPieceEmbeddingTokenizer fromJsonString(String content)

Parse a WordPiece tokenizer from the raw tokenizer.json content.

Implementation

static WordPieceEmbeddingTokenizer fromJsonString(String content) {
  final json = jsonDecode(content) as Map<String, dynamic>;
  final model = json['model'] as Map<String, dynamic>?;
  if (model == null || model['type'] != 'WordPiece') {
    throw FormatException(
      'tokenizer.json is not a WordPiece model '
      '(model.type=${model?['type']}); use the Gemma SentencePiece adapter.',
    );
  }

  final rawVocab = model['vocab'] as Map<String, dynamic>;
  final vocab = <String, int>{
    for (final entry in rawVocab.entries) entry.key: entry.value as int,
  };

  final unkToken = (model['unk_token'] as String?) ?? '[UNK]';
  final continuingSubwordPrefix =
      (model['continuing_subword_prefix'] as String?) ?? '##';
  final maxInputCharsPerWord =
      (model['max_input_chars_per_word'] as int?) ?? 100;

  // Normalizer flags (BertNormalizer). HuggingFace uses `strip_accents: null`
  // to mean "follow lowercase", so a null value resolves to [lowercase].
  final normalizer = json['normalizer'] as Map<String, dynamic>?;
  final lowercase = (normalizer?['lowercase'] as bool?) ?? true;
  final stripAccentsRaw = normalizer?['strip_accents'];
  final stripAccents = stripAccentsRaw is bool ? stripAccentsRaw : lowercase;
  final handleChineseChars =
      (normalizer?['handle_chinese_chars'] as bool?) ?? true;
  final cleanText = (normalizer?['clean_text'] as bool?) ?? true;

  // Special-token ids come from the post-processor's TemplateProcessing block
  // when present; otherwise fall back to the canonical BERT ids.
  int specialId(String token, int fallback) => vocab[token] ?? fallback;
  final clsId = specialId('[CLS]', 101);
  final sepId = specialId('[SEP]', 102);
  final unkId = specialId(unkToken, 100);

  return WordPieceEmbeddingTokenizer._(
    vocab: vocab,
    clsId: clsId,
    sepId: sepId,
    unkId: unkId,
    continuingSubwordPrefix: continuingSubwordPrefix,
    maxInputCharsPerWord: maxInputCharsPerWord,
    lowercase: lowercase,
    stripAccents: stripAccents,
    handleChineseChars: handleChineseChars,
    cleanText: cleanText,
  );
}