llama_tokenize method

int llama_tokenize(
  1. Pointer<llama_model> model,
  2. Pointer<Char> text,
  3. int text_len,
  4. Pointer<llama_token> tokens,
  5. int n_max_tokens,
  6. bool add_bos,
  7. bool special,
)

@details Convert the provided text into tokens. @param tokens The tokens pointer must be large enough to hold the resulting tokens. @return Returns the number of tokens on success, no more than n_max_tokens @return Returns a negative number on failure - the number of tokens that would have been returned @param special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated as plaintext. Does not insert a leading space.

Implementation

int llama_tokenize(
  ffi.Pointer<llama_model> model,
  ffi.Pointer<ffi.Char> text,
  int text_len,
  ffi.Pointer<llama_token> tokens,
  int n_max_tokens,
  bool add_bos,
  bool special,
) {
  return _llama_tokenize(
    model,
    text,
    text_len,
    tokens,
    n_max_tokens,
    add_bos,
    special,
  );
}