encode method

List<int> encode(String text, { bool addSpecial = true, bool parseSpecial = true, })

Tokenize text.

addSpecial adds BOS/special prefix tokens when the vocab requests it. parseSpecial interprets <...> style strings as special tokens.

Implementation

List<int> encode(
  String text, {
  bool addSpecial = true,
  bool parseSpecial = true,
}) {
  if (text.isEmpty) return const <int>[];
  final b = LlamaLibrary.bindings;
  final bytes = utf8.encode(text);

  final textPtr = calloc<Char>(bytes.length);
  try {
    textPtr.cast<Uint8>().asTypedList(bytes.length).setAll(0, bytes);

    // Probe with a 0-sized buffer; result is `-needed` if too small.
    final probe = b.llama_tokenize(
      vocab.pointer,
      textPtr,
      bytes.length,
      nullptr,
      0,
      addSpecial,
      parseSpecial,
    );

    if (probe >= 0) {
      // Empty input was caught above; an exact 0-fit here is unexpected
      // but harmless. Anything else indicates a misuse.
      if (probe == 0) return const <int>[];
      throw LlamaTokenizeException(
        'tokenize probe returned positive count: $probe',
      );
    }

    final needed = -probe;
    final out = calloc<llama_token>(needed);
    try {
      final n = b.llama_tokenize(
        vocab.pointer,
        textPtr,
        bytes.length,
        out,
        needed,
        addSpecial,
        parseSpecial,
      );
      if (n < 0) {
        throw LlamaTokenizeException('tokenize failed: $n');
      }
      return List<int>.generate(n, (i) => out[i], growable: false);
    } finally {
      calloc.free(out);
    }
  } finally {
    calloc.free(textPtr);
  }
}