LlamaCppBindings class

Generated bindings for llama.cpp.

Constructors

LlamaCppBindings(DynamicLibrary dynamicLibrary): The symbols are looked up in dynamicLibrary.
LlamaCppBindings.fromLookup(Pointer<T> lookup<T extends NativeType>(String symbolName)): The symbols are looked up with lookup.

Properties

hashCode → int: The hash code for this object.
no setterinherited
runtimeType → Type: A representation of the runtime type of the object.
no setterinherited

Methods

lib_llama_cpp_abi_version() → int
lib_llama_cpp_chat_parse_json(Pointer<Char> parser_request_json, Pointer<Pointer<Char>> error_json) → Pointer<Char>
lib_llama_cpp_chat_templates_apply_json(Pointer<lib_llama_cpp_chat_templates> handle, Pointer<Char> request_json, Pointer<Pointer<Char>> error_json) → Pointer<Char>
lib_llama_cpp_chat_templates_free(Pointer<lib_llama_cpp_chat_templates> handle) → void
lib_llama_cpp_chat_templates_init(Pointer<llama_model> model, Pointer<Char> chat_template_override, Pointer<Char> bos_token_override, Pointer<Char> eos_token_override, Pointer<Pointer<Char>> error_json) → Pointer<lib_llama_cpp_chat_templates>
lib_llama_cpp_media_blob_free(Pointer<lib_llama_cpp_media_blob> handle) → void
lib_llama_cpp_media_blob_from_encoded_bytes(Pointer<lib_llama_cpp_media_context> handle, Pointer<UnsignedChar> bytes, int byte_count, Pointer<Char> id, Pointer<Pointer<Char>> error_json) → Pointer<lib_llama_cpp_media_blob>
lib_llama_cpp_media_blob_from_file(Pointer<lib_llama_cpp_media_context> handle, Pointer<Char> path, Pointer<Char> id, Pointer<Pointer<Char>> error_json) → Pointer<lib_llama_cpp_media_blob>
lib_llama_cpp_media_eval_prompt(Pointer<lib_llama_cpp_media_context> media, Pointer<llama_context> llama, Pointer<Char> prompt, Pointer<Pointer<lib_llama_cpp_media_blob>> blobs, int blob_count, int n_past, int seq_id, int n_batch, bool logits_last, bool add_special, bool parse_special, Pointer<Int32> new_n_past, Pointer<Pointer<Char>> error_json) → int
lib_llama_cpp_media_free(Pointer<lib_llama_cpp_media_context> handle) → void
lib_llama_cpp_media_init(Pointer<Char> mmproj_path, Pointer<llama_model> model, Pointer<Char> options_json, Pointer<Pointer<Char>> error_json) → Pointer<lib_llama_cpp_media_context>
lib_llama_cpp_media_supports_audio(Pointer<lib_llama_cpp_media_context> handle) → bool
lib_llama_cpp_media_supports_vision(Pointer<lib_llama_cpp_media_context> handle) → bool
lib_llama_cpp_string_free(Pointer<Char> value) → void
llama_adapter_get_alora_invocation_tokens(Pointer<llama_adapter_lora> adapter) → Pointer<Int32>
llama_adapter_get_alora_n_invocation_tokens(Pointer<llama_adapter_lora> adapter) → int: Get the invocation tokens if the current lora is an alora
llama_adapter_lora_free(Pointer<llama_adapter_lora> adapter) → void: Manually free a LoRA adapter NOTE: loaded adapters that are not manually freed will be freed when the associated model is deleted
llama_adapter_lora_init(Pointer<llama_model> model, Pointer<Char> path_lora) → Pointer<llama_adapter_lora>: Load a LoRA adapter from file The adapter is valid as long as the associated model is not freed
llama_adapter_meta_count(Pointer<llama_adapter_lora> adapter) → int: Get the number of metadata key/value pairs
llama_adapter_meta_key_by_index(Pointer<llama_adapter_lora> adapter, int i, Pointer<Char> buf, int buf_size) → int: Get metadata key name by index
llama_adapter_meta_val_str(Pointer<llama_adapter_lora> adapter, Pointer<Char> key, Pointer<Char> buf, int buf_size) → int: Get metadata value as a string by key name
llama_adapter_meta_val_str_by_index(Pointer<llama_adapter_lora> adapter, int i, Pointer<Char> buf, int buf_size) → int: Get metadata value as a string by index
llama_add_bos_token(Pointer<llama_vocab> vocab) → bool
llama_add_eos_token(Pointer<llama_vocab> vocab) → bool
llama_attach_threadpool(Pointer<llama_context> ctx, ggml_threadpool_t threadpool, ggml_threadpool_t threadpool_batch) → void: Optional: an auto threadpool gets created in ggml if not passed explicitly
llama_backend_free() → void: Call once at the end of the program - currently only used for MPI
llama_backend_init() → void: Initialize the llama + ggml backend If numa is true, use NUMA optimizations Call once at the start of the program
llama_batch_free(llama_batch batch) → void: Frees a batch of tokens allocated with llama_batch_init()
llama_batch_get_one(Pointer<Int32> tokens, int n_tokens) → llama_batch: Return batch for single sequence of tokens The sequence ID will be fixed to 0 The position of the tokens will be tracked automatically by llama_decode
llama_batch_init(int n_tokens, int embd, int n_seq_max) → llama_batch: Allocates a batch of tokens on the heap that can hold a maximum of n_tokens Each token can be assigned up to n_seq_max sequence ids The batch has to be freed with llama_batch_free() If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd * sizeof(float) Otherwise, llama_batch.token will be allocated to store n_tokens llama_token The rest of the llama_batch members are allocated with size n_tokens All members are left uninitialized
llama_chat_apply_template(Pointer<Char> tmpl, Pointer<llama_chat_message> chat, int n_msg, bool add_ass, Pointer<Char> buf, int length) → int: Apply chat template. Inspired by hf apply_chat_template() on python.
llama_chat_builtin_templates(Pointer<Pointer<Char>> output, int len) → int: Get list of built-in chat templates
llama_context_default_params() → llama_context_params
llama_copy_state_data(Pointer<llama_context> ctx, Pointer<Uint8> dst) → int
llama_decode(Pointer<llama_context> ctx, llama_batch batch) → int: Process a batch of tokens. Requires the context to have a memory. For encode-decoder contexts, processes the batch using the decoder. Positive return values does not mean a fatal error, but rather a warning. Upon fatal-error or abort, the ubatches that managed to be been processed will remain in the memory state of the context To handle this correctly, query the memory state using llama_memory_seq_pos_min() and llama_memory_seq_pos_max() Upon other return values, the memory state is restored to the state before this call 0 - success 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context) 2 - aborted (processed ubatches will remain in the context's memory) -1 - invalid input batch < -1 - fatal error (processed ubatches will remain in the context's memory)
llama_detach_threadpool(Pointer<llama_context> ctx) → void
llama_detokenize(Pointer<llama_vocab> vocab, Pointer<Int32> tokens, int n_tokens, Pointer<Char> text, int text_len_max, bool remove_special, bool unparse_special) → int: @details Convert the provided tokens into text (inverse of llama_tokenize()). @param text The char pointer must be large enough to hold the resulting text. @return Returns the number of chars/bytes on success, no more than text_len_max. @return Returns a negative number on failure - the number of chars/bytes that would have been returned. @param remove_special Allow to remove BOS and EOS tokens if model is configured to do so. @param unparse_special If true, special tokens are rendered in the output.
llama_encode(Pointer<llama_context> ctx, llama_batch batch) → int: Process a batch of tokens. In contrast to llama_decode() - this call does not use KV cache. For encode-decoder contexts, processes the batch using the encoder. Can store the encoder output internally for later use by the decoder's cross-attention layers. 0 - success < 0 - error. the memory state is restored to the state before this call
llama_flash_attn_type_name(llama_flash_attn_type flash_attn_type) → Pointer<Char>
llama_free(Pointer<llama_context> ctx) → void: Frees all allocated memory
llama_free_model(Pointer<llama_model> model) → void
llama_get_embeddings(Pointer<llama_context> ctx) → Pointer<Float>: Get all output token embeddings. when pooling_type == LLAMA_POOLING_TYPE_NONE or when using a generative model, the embeddings for which llama_batch.logitsi != 0 are stored contiguously in the order they have appeared in the batch. shape: n_outputs*n_embd Otherwise, returns NULL. TODO: deprecate in favor of llama_get_embeddings_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522)
llama_get_embeddings_ith(Pointer<llama_context> ctx, int i) → Pointer<Float>: Get the embeddings for the ith token. For positive indices, Equivalent to: llama_get_embeddings(ctx) + ctx->output_idsi*n_embd Negative indices can be used to access embeddings in reverse order, -1 is the last embedding. shape: n_embd (1-dimensional) returns NULL for invalid ids.
llama_get_embeddings_seq(Pointer<llama_context> ctx, int seq_id) → Pointer<Float>: Get the embeddings for a sequence id Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE when pooling_type == LLAMA_POOLING_TYPE_RANK, returns floatn_cls_out with the rank(s) of the sequence otherwise: floatn_embd (1-dimensional)
llama_get_logits(Pointer<llama_context> ctx) → Pointer<Float>: Token logits obtained from the last call to llama_decode() The logits for which llama_batch.logitsi != 0 are stored contiguously in the order they have appeared in the batch. Rows: number of tokens for which llama_batch.logitsi != 0 Cols: n_vocab TODO: deprecate in favor of llama_get_logits_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522)
llama_get_logits_ith(Pointer<llama_context> ctx, int i) → Pointer<Float>: Logits for the ith token. For positive indices, Equivalent to: llama_get_logits(ctx) + ctx->output_idsi*n_vocab Negative indices can be used to access logits in reverse order, -1 is the last logit. returns NULL for invalid ids.
llama_get_memory(Pointer<llama_context> ctx) → llama_memory_t
llama_get_model(Pointer<llama_context> ctx) → Pointer<llama_model>
llama_get_sampled_candidates_count_ith(Pointer<llama_context> ctx, int i) → int
llama_get_sampled_candidates_ith(Pointer<llama_context> ctx, int i) → Pointer<Int32>: Get the backend sampled candidates (token ids) for the ith token These are needed to map probability/logit indices to vocab token ids. Returns NULL if no candidates were sampled.
llama_get_sampled_logits_count_ith(Pointer<llama_context> ctx, int i) → int
llama_get_sampled_logits_ith(Pointer<llama_context> ctx, int i) → Pointer<Float>: Get the backend sampled logits for the ith token Returns NULL if no logits were sampled.
llama_get_sampled_probs_count_ith(Pointer<llama_context> ctx, int i) → int
llama_get_sampled_probs_ith(Pointer<llama_context> ctx, int i) → Pointer<Float>: Get the backend sampled probabilities for the ith token The index matches llama_get_sampled_token_ith(). Returns NULL if no probabilities were generated.
llama_get_sampled_token_ith(Pointer<llama_context> ctx, int i) → int: Get the backend sampled token for the ith token. Returns LLAMA_TOKEN_NULL if no token was sampled.
llama_get_state_size(Pointer<llama_context> ctx) → int
llama_init_from_model(Pointer<llama_model> model, llama_context_params params) → Pointer<llama_context>
llama_load_model_from_file(Pointer<Char> path_model, llama_model_params params) → Pointer<llama_model>
llama_load_session_file(Pointer<llama_context> ctx, Pointer<Char> path_session, Pointer<Int32> tokens_out, int n_token_capacity, Pointer<Size> n_token_count_out) → bool
llama_log_get(Pointer<ggml_log_callback> log_callback, Pointer<Pointer<Void>> user_data) → void: Set callback for all future logging events. If this is not called, or NULL is supplied, everything is output on stderr. The logger state is global so these functions are NOT thread safe.
llama_log_set(ggml_log_callback log_callback, Pointer<Void> user_data) → void
llama_max_devices() → int
llama_max_parallel_sequences() → int
llama_max_tensor_buft_overrides() → int
llama_memory_can_shift(llama_memory_t mem) → bool: Check if the memory supports shifting
llama_memory_clear(llama_memory_t mem, bool data) → void: Clear the memory contents If data == true, the data buffers will also be cleared together with the metadata
llama_memory_seq_add(llama_memory_t mem, int seq_id, int p0, int p1, int delta) → void: Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1) p0 < 0 : 0, p1 p1 < 0 : [p0, inf)
llama_memory_seq_cp(llama_memory_t mem, int seq_id_src, int seq_id_dst, int p0, int p1) → void: Copy all tokens that belong to the specified sequence to another sequence p0 < 0 : 0, p1 p1 < 0 : [p0, inf)
llama_memory_seq_div(llama_memory_t mem, int seq_id, int p0, int p1, int d) → void: Integer division of the positions by factor of d > 1 p0 < 0 : 0, p1 p1 < 0 : [p0, inf)
llama_memory_seq_keep(llama_memory_t mem, int seq_id) → void: Removes all tokens that do not belong to the specified sequence
llama_memory_seq_pos_max(llama_memory_t mem, int seq_id) → int: Returns the largest position present in the memory for the specified sequence Note that all positions in the range pos_min, pos_max are guaranteed to be present in the memory Return -1 if the sequence is empty
llama_memory_seq_pos_min(llama_memory_t mem, int seq_id) → int: Returns the smallest position present in the memory for the specified sequence This is typically non-zero only for SWA caches Note that all positions in the range pos_min, pos_max are guaranteed to be present in the memory Return -1 if the sequence is empty
llama_memory_seq_rm(llama_memory_t mem, int seq_id, int p0, int p1) → bool: Removes all tokens that belong to the specified sequence and have positions in [p0, p1) Returns false if a partial sequence cannot be removed. Removing a whole sequence never fails seq_id < 0 : match any sequence p0 < 0 : 0, p1 p1 < 0 : [p0, inf)
llama_model_chat_template(Pointer<llama_model> model, Pointer<Char> name) → Pointer<Char>: Get the default chat template. Returns nullptr if not available If name is NULL, returns the default chat template
llama_model_cls_label(Pointer<llama_model> model, int i) → Pointer<Char>: Returns label of classifier output by index (<n_cls_out). Returns nullptr if no label provided
llama_model_decoder_start_token(Pointer<llama_model> model) → int: For encoder-decoder models, this function returns id of the token that must be provided to the decoder to start generating output sequence. For other models, it returns -1.
llama_model_default_params() → llama_model_params: Helpers for getting default parameters TODO: update API to start accepting pointers to params structs (https://github.com/ggml-org/llama.cpp/discussions/9172)
llama_model_desc(Pointer<llama_model> model, Pointer<Char> buf, int buf_size) → int: Get a string describing the model type
llama_model_free(Pointer<llama_model> model) → void
llama_model_get_vocab(Pointer<llama_model> model) → Pointer<llama_vocab>
llama_model_has_decoder(Pointer<llama_model> model) → bool: Returns true if the model contains a decoder that requires llama_decode() call
llama_model_has_encoder(Pointer<llama_model> model) → bool: Returns true if the model contains an encoder that requires llama_encode() call
llama_model_init_from_user(Pointer<gguf_context> metadata, llama_model_set_tensor_data_t set_tensor_data, Pointer<Void> set_tensor_data_ud, llama_model_params params) → Pointer<llama_model>: Create a new model from GGUF metadata as well as a function to set the tensor data
llama_model_is_diffusion(Pointer<llama_model> model) → bool: Returns true if the model is diffusion-based (like LLaDA, Dream, etc.)
llama_model_is_hybrid(Pointer<llama_model> model) → bool: Returns true if the model is hybrid (like Jamba, Granite, etc.)
llama_model_is_recurrent(Pointer<llama_model> model) → bool: Returns true if the model is recurrent (like Mamba, RWKV, etc.)
llama_model_load_from_file(Pointer<Char> path_model, llama_model_params params) → Pointer<llama_model>: Load a model from a file If the file is split into multiple parts, the file name must follow this pattern:
llama_model_load_from_file_ptr(Pointer<__sFILE> file, llama_model_params params) → Pointer<llama_model>: Load a model from an open FILE pointer
llama_model_load_from_splits(Pointer<Pointer<Char>> paths, int n_paths, llama_model_params params) → Pointer<llama_model>: Load a model from multiple splits (support custom naming scheme) The paths must be in the correct order
llama_model_meta_count(Pointer<llama_model> model) → int: Get the number of metadata key/value pairs
llama_model_meta_key_by_index(Pointer<llama_model> model, int i, Pointer<Char> buf, int buf_size) → int: Get metadata key name by index
llama_model_meta_key_str(llama_model_meta_key key) → Pointer<Char>: Get sampling metadata key name. Returns nullptr if the key is invalid
llama_model_meta_val_str(Pointer<llama_model> model, Pointer<Char> key, Pointer<Char> buf, int buf_size) → int: Get metadata value as a string by key name
llama_model_meta_val_str_by_index(Pointer<llama_model> model, int i, Pointer<Char> buf, int buf_size) → int: Get metadata value as a string by index
llama_model_n_cls_out(Pointer<llama_model> model) → int: Returns the number of classifier outputs (only valid for classifier models) Undefined behavior for non-classifier models
llama_model_n_ctx_train(Pointer<llama_model> model) → int
llama_model_n_embd(Pointer<llama_model> model) → int
llama_model_n_embd_inp(Pointer<llama_model> model) → int
llama_model_n_embd_out(Pointer<llama_model> model) → int
llama_model_n_head(Pointer<llama_model> model) → int
llama_model_n_head_kv(Pointer<llama_model> model) → int
llama_model_n_layer(Pointer<llama_model> model) → int
llama_model_n_params(Pointer<llama_model> model) → int: Returns the total number of parameters in the model
llama_model_n_swa(Pointer<llama_model> model) → int
llama_model_quantize(Pointer<Char> fname_inp, Pointer<Char> fname_out, Pointer<llama_model_quantize_params> params) → int: Returns 0 on success
llama_model_quantize_default_params() → llama_model_quantize_params
llama_model_rope_freq_scale_train(Pointer<llama_model> model) → double: Get the model's RoPE frequency scaling factor
llama_model_rope_type(Pointer<llama_model> model) → llama_rope_type
llama_model_save_to_file(Pointer<llama_model> model, Pointer<Char> path_model) → void
llama_model_size(Pointer<llama_model> model) → int: Returns the total size of all the tensors in the model in bytes
llama_n_batch(Pointer<llama_context> ctx) → int
llama_n_ctx(Pointer<llama_context> ctx) → int: NOTE: After creating a llama_context, it is recommended to query the actual values using these functions In some cases the requested values via llama_context_params may differ from the actual values used by the context ref: https://github.com/ggml-org/llama.cpp/pull/17046#discussion_r2503085732
llama_n_ctx_seq(Pointer<llama_context> ctx) → int
llama_n_ctx_train(Pointer<llama_model> model) → int
llama_n_embd(Pointer<llama_model> model) → int
llama_n_head(Pointer<llama_model> model) → int
llama_n_layer(Pointer<llama_model> model) → int
llama_n_rs_seq(Pointer<llama_context> ctx) → int
llama_n_seq_max(Pointer<llama_context> ctx) → int
llama_n_threads(Pointer<llama_context> ctx) → int: Get the number of threads used for generation of a single token.
llama_n_threads_batch(Pointer<llama_context> ctx) → int: Get the number of threads used for prompt and batch processing (multiple token).
llama_n_ubatch(Pointer<llama_context> ctx) → int
llama_n_vocab(Pointer<llama_vocab> vocab) → int
llama_new_context_with_model(Pointer<llama_model> model, llama_context_params params) → Pointer<llama_context>
llama_numa_init(ggml_numa_strategy numa) → void: optional:
llama_opt_epoch(Pointer<llama_context> lctx, ggml_opt_dataset_t dataset, ggml_opt_result_t result_train, ggml_opt_result_t result_eval, int idata_split, ggml_opt_epoch_callback callback_train, ggml_opt_epoch_callback callback_eval) → void
llama_opt_init(Pointer<llama_context> lctx, Pointer<llama_model> model, llama_opt_params lopt_params) → void
llama_opt_param_filter_all(Pointer<ggml_tensor> tensor, Pointer<Void> userdata) → bool: always returns true
llama_perf_context(Pointer<llama_context> ctx) → llama_perf_context_data
llama_perf_context_print(Pointer<llama_context> ctx) → void
llama_perf_context_reset(Pointer<llama_context> ctx) → void
llama_perf_sampler(Pointer<llama_sampler> chain) → llama_perf_sampler_data: NOTE: the following work only with samplers constructed via llama_sampler_chain_init
llama_perf_sampler_print(Pointer<llama_sampler> chain) → void
llama_perf_sampler_reset(Pointer<llama_sampler> chain) → void
llama_pooling_type$1(Pointer<llama_context> ctx) → llama_pooling_type
llama_print_system_info() → Pointer<Char>: Print system information
llama_sampler_accept(Pointer<llama_sampler> smpl, int token) → void
llama_sampler_apply(Pointer<llama_sampler> smpl, Pointer<llama_token_data_array> cur_p) → void
llama_sampler_chain_add(Pointer<llama_sampler> chain, Pointer<llama_sampler> smpl) → void: important: takes ownership of the sampler object and will free it when llama_sampler_free is called
llama_sampler_chain_default_params() → llama_sampler_chain_params
llama_sampler_chain_get(Pointer<llama_sampler> chain, int i) → Pointer<llama_sampler>: return NULL if:
llama_sampler_chain_init(llama_sampler_chain_params params) → Pointer<llama_sampler>: llama_sampler_chain a type of llama_sampler that can chain multiple samplers one after another
llama_sampler_chain_n(Pointer<llama_sampler> chain) → int: the total number of samplers in the chain
llama_sampler_chain_remove(Pointer<llama_sampler> chain, int i) → Pointer<llama_sampler>: after removing a sampler, the chain will no longer own it, and it will not be freed when the chain is freed
llama_sampler_clone(Pointer<llama_sampler> smpl) → Pointer<llama_sampler>
llama_sampler_free(Pointer<llama_sampler> smpl) → void: important: do not free if the sampler has been added to a llama_sampler_chain (via llama_sampler_chain_add)
llama_sampler_get_seed(Pointer<llama_sampler> smpl) → int: Returns the seed used by the sampler if applicable, LLAMA_DEFAULT_SEED otherwise
llama_sampler_init(Pointer<llama_sampler_i> iface, llama_sampler_context_t ctx) → Pointer<llama_sampler>: mirror of llama_sampler_i:
llama_sampler_init_adaptive_p(double target, double decay, int seed) → Pointer<llama_sampler>: adaptive-p: select tokens near a configurable target probability over time.
llama_sampler_init_dist(int seed) → Pointer<llama_sampler>: seed == LLAMA_DEFAULT_SEED to use a random seed.
llama_sampler_init_dry(Pointer<llama_vocab> vocab, int n_ctx_train, double dry_multiplier, double dry_base, int dry_allowed_length, int dry_penalty_last_n, Pointer<Pointer<Char>> seq_breakers, int num_breakers) → Pointer<llama_sampler>: @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982
llama_sampler_init_grammar(Pointer<llama_vocab> vocab, Pointer<Char> grammar_str, Pointer<Char> grammar_root) → Pointer<llama_sampler>: @details Initializes a GBNF grammar, see grammars/README.md for details. @param vocab The vocabulary that this grammar will be used with. @param grammar_str The production rules for the grammar, encoded as a string. Returns an empty grammar if empty. Returns NULL if parsing of grammar_str fails. @param grammar_root The name of the start symbol for the grammar.
llama_sampler_init_grammar_lazy(Pointer<llama_vocab> vocab, Pointer<Char> grammar_str, Pointer<Char> grammar_root, Pointer<Pointer<Char>> trigger_words, int num_trigger_words, Pointer<Int32> trigger_tokens, int num_trigger_tokens) → Pointer<llama_sampler>
llama_sampler_init_grammar_lazy_patterns(Pointer<llama_vocab> vocab, Pointer<Char> grammar_str, Pointer<Char> grammar_root, Pointer<Pointer<Char>> trigger_patterns, int num_trigger_patterns, Pointer<Int32> trigger_tokens, int num_trigger_tokens) → Pointer<llama_sampler>: @details Lazy grammar sampler, introduced in https://github.com/ggml-org/llama.cpp/pull/9639 @param trigger_patterns A list of patterns that will trigger the grammar sampler. Pattern will be matched from the start of the generation output, and grammar sampler will be fed content starting from its first match group. @param trigger_tokens A list of tokens that will trigger the grammar sampler. Grammar sampler will be fed content starting from the trigger token included.
llama_sampler_init_greedy() → Pointer<llama_sampler>: available samplers:
llama_sampler_init_infill(Pointer<llama_vocab> vocab) → Pointer<llama_sampler>: this sampler is meant to be used for fill-in-the-middle infilling it's supposed to be used after top_k + top_p sampling
llama_sampler_init_logit_bias(int n_vocab, int n_logit_bias, Pointer<llama_logit_bias> logit_bias) → Pointer<llama_sampler>
llama_sampler_init_min_p(double p, int min_keep) → Pointer<llama_sampler>: @details Minimum P sampling as described in https://github.com/ggml-org/llama.cpp/pull/3841
llama_sampler_init_mirostat(int n_vocab, int seed, double tau, double eta, int m) → Pointer<llama_sampler>: @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. @param candidates A vector of llama_token_data containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. @param eta The learning rate used to update mu based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause mu to be updated more quickly, while a smaller learning rate will result in slower updates. @param m The number of tokens considered in the estimation of s_hat. This is an arbitrary value that is used to calculate s_hat, which in turn helps to calculate the value of k. In the paper, they use m = 100, but you can experiment with different values to see how it affects the performance of the algorithm. @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (2 * tau) and is updated in the algorithm based on the error between the target and observed surprisal.
llama_sampler_init_mirostat_v2(int seed, double tau, double eta) → Pointer<llama_sampler>: @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. @param candidates A vector of llama_token_data containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. @param eta The learning rate used to update mu based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause mu to be updated more quickly, while a smaller learning rate will result in slower updates. @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (2 * tau) and is updated in the algorithm based on the error between the target and observed surprisal.
llama_sampler_init_penalties(int penalty_last_n, double penalty_repeat, double penalty_freq, double penalty_present) → Pointer<llama_sampler>: NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first.
llama_sampler_init_temp(double t) → Pointer<llama_sampler>: #details Updates the logits l_i` = l_i/t. When t <= 0.0f, the maximum logit is kept at it's original value, the rest are set to -inf
llama_sampler_init_temp_ext(double t, double delta, double exponent) → Pointer<llama_sampler>: @details Dynamic temperature implementation (a.k.a. entropy) described in the paper https://arxiv.org/abs/2309.02772.
llama_sampler_init_top_k(int k) → Pointer<llama_sampler>: @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 Setting k <= 0 makes this a noop
llama_sampler_init_top_n_sigma(double n) → Pointer<llama_sampler>: @details Top n sigma sampling as described in academic paper "Top-nσ: Not All Logits Are You Need" https://arxiv.org/pdf/2411.07641
llama_sampler_init_top_p(double p, int min_keep) → Pointer<llama_sampler>: @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
llama_sampler_init_typical(double p, int min_keep) → Pointer<llama_sampler>: @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.
llama_sampler_init_xtc(double p, double t, int min_keep, int seed) → Pointer<llama_sampler>: @details XTC sampler as described in https://github.com/oobabooga/text-generation-webui/pull/6335
llama_sampler_name(Pointer<llama_sampler> smpl) → Pointer<Char>
llama_sampler_reset(Pointer<llama_sampler> smpl) → void
llama_sampler_sample(Pointer<llama_sampler> smpl, Pointer<llama_context> ctx, int idx) → int: @details Sample and accept a token from the idx-th output of the last evaluation
llama_save_session_file(Pointer<llama_context> ctx, Pointer<Char> path_session, Pointer<Int32> tokens, int n_token_count) → bool
llama_set_abort_callback(Pointer<llama_context> ctx, ggml_abort_callback abort_callback, Pointer<Void> abort_callback_data) → void: Set abort callback
llama_set_adapter_cvec(Pointer<llama_context> ctx, Pointer<Float> data, int len, int n_embd, int il_start, int il_end) → int: Apply a loaded control vector to a llama_context, or if data is NULL, clear the currently loaded vector. n_embd should be the size of a single layer's control, and data should point to an n_embd x n_layers buffer starting from layer 1. il_start and il_end are the layer range the vector should apply to (both inclusive) See llama_control_vector_load in common to load a control vector.
llama_set_adapters_lora(Pointer<llama_context> ctx, Pointer<Pointer<llama_adapter_lora>> adapters, int n_adapters, Pointer<Float> scales) → int: Set LoRa adapters on the context. Will only modify if the adapters currently in context are different.
llama_set_causal_attn(Pointer<llama_context> ctx, bool causal_attn) → void: Set whether to use causal attention or not If set to true, the model will only attend to the past tokens
llama_set_embeddings(Pointer<llama_context> ctx, bool embeddings) → void: Set whether the context outputs embeddings or not TODO: rename to avoid confusion with llama_get_embeddings()
llama_set_n_threads(Pointer<llama_context> ctx, int n_threads, int n_threads_batch) → void: Set the number of threads used for decoding n_threads is the number of threads used for generation (single token) n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens)
llama_set_sampler(Pointer<llama_context> ctx, int seq_id, Pointer<llama_sampler> smpl) → bool: EXPERIMENTAL attach a sampler to the context note: prefer initializing the context with llama_context_params.samplers when possible
llama_set_state_data(Pointer<llama_context> ctx, Pointer<Uint8> src) → int
llama_set_warmup(Pointer<llama_context> ctx, bool warmup) → void: Set whether the model is in warmup mode or not If true, all model tensors are activated during llama_decode() to load and cache their weights.
llama_split_path(Pointer<Char> split_path, int maxlen, Pointer<Char> path_prefix, int split_no, int split_count) → int: @details Build a split GGUF final path for this chunk. llama_split_path(split_path, sizeof(split_path), "/models/ggml-model-q4_0", 2, 4) => split_path = "/models/ggml-model-q4_0-00002-of-00004.gguf" Returns the split_path length.
llama_split_prefix(Pointer<Char> split_prefix, int maxlen, Pointer<Char> split_path, int split_no, int split_count) → int: @details Extract the path prefix from the split_path if and only if the split_no and split_count match. llama_split_prefix(split_prefix, 64, "/models/ggml-model-q4_0-00002-of-00004.gguf", 2, 4) => split_prefix = "/models/ggml-model-q4_0" Returns the split_prefix length.
llama_state_get_data(Pointer<llama_context> ctx, Pointer<Uint8> dst, int size) → int: Copies the state to the specified destination address. Destination needs to have allocated enough memory. Returns the number of bytes copied
llama_state_get_size(Pointer<llama_context> ctx) → int: Returns the actual size in bytes of the state (logits, embedding and memory) Only use when saving the state, not when restoring it, otherwise the size may be too small.
llama_state_load_file(Pointer<llama_context> ctx, Pointer<Char> path_session, Pointer<Int32> tokens_out, int n_token_capacity, Pointer<Size> n_token_count_out) → bool: Save/load session file
llama_state_save_file(Pointer<llama_context> ctx, Pointer<Char> path_session, Pointer<Int32> tokens, int n_token_count) → bool
llama_state_seq_get_data(Pointer<llama_context> ctx, Pointer<Uint8> dst, int size, int seq_id) → int: Copy the state of a single sequence into the specified buffer
llama_state_seq_get_data_ext(Pointer<llama_context> ctx, Pointer<Uint8> dst, int size, int seq_id, int flags) → int
llama_state_seq_get_size(Pointer<llama_context> ctx, int seq_id) → int: Get the exact size needed to copy the state of a single sequence
llama_state_seq_get_size_ext(Pointer<llama_context> ctx, int seq_id, int flags) → int
llama_state_seq_load_file(Pointer<llama_context> ctx, Pointer<Char> filepath, int dest_seq_id, Pointer<Int32> tokens_out, int n_token_capacity, Pointer<Size> n_token_count_out) → int
llama_state_seq_save_file(Pointer<llama_context> ctx, Pointer<Char> filepath, int seq_id, Pointer<Int32> tokens, int n_token_count) → int
llama_state_seq_set_data(Pointer<llama_context> ctx, Pointer<Uint8> src, int size, int dest_seq_id) → int: Copy the sequence data (originally copied with llama_state_seq_get_data) into the specified sequence Returns:
llama_state_seq_set_data_ext(Pointer<llama_context> ctx, Pointer<Uint8> src, int size, int dest_seq_id, int flags) → int
llama_state_set_data(Pointer<llama_context> ctx, Pointer<Uint8> src, int size) → int: Set the state reading from the specified address Returns the number of bytes read
llama_supports_gpu_offload() → bool
llama_supports_mlock() → bool
llama_supports_mmap() → bool
llama_supports_rpc() → bool
llama_synchronize(Pointer<llama_context> ctx) → void: Wait until all computations are finished This is automatically done when using one of the functions below to obtain the computation results and is not necessary to call it explicitly in most cases
llama_time_us() → int
llama_token_bos(Pointer<llama_vocab> vocab) → int
llama_token_cls(Pointer<llama_vocab> vocab) → int
llama_token_eos(Pointer<llama_vocab> vocab) → int
llama_token_eot(Pointer<llama_vocab> vocab) → int
llama_token_fim_mid(Pointer<llama_vocab> vocab) → int
llama_token_fim_pad(Pointer<llama_vocab> vocab) → int
llama_token_fim_pre(Pointer<llama_vocab> vocab) → int
llama_token_fim_rep(Pointer<llama_vocab> vocab) → int
llama_token_fim_sep(Pointer<llama_vocab> vocab) → int
llama_token_fim_suf(Pointer<llama_vocab> vocab) → int
llama_token_get_attr(Pointer<llama_vocab> vocab, Dartllama_token token) → llama_token_attr
llama_token_get_score(Pointer<llama_vocab> vocab, int token) → double
llama_token_get_text(Pointer<llama_vocab> vocab, int token) → Pointer<Char>
llama_token_is_control(Pointer<llama_vocab> vocab, int token) → bool
llama_token_is_eog(Pointer<llama_vocab> vocab, int token) → bool
llama_token_nl(Pointer<llama_vocab> vocab) → int
llama_token_pad(Pointer<llama_vocab> vocab) → int
llama_token_sep(Pointer<llama_vocab> vocab) → int
llama_token_to_piece(Pointer<llama_vocab> vocab, int token, Pointer<Char> buf, int length, int lstrip, bool special) → int: Token Id -> Piece. Uses the vocabulary in the provided context. Does not write null terminator to the buffer. User can skip up to 'lstrip' leading spaces before copying (useful when encoding/decoding multiple tokens with 'add_space_prefix') @param special If true, special tokens are rendered in the output.
llama_tokenize(Pointer<llama_vocab> vocab, Pointer<Char> text, int text_len, Pointer<Int32> tokens, int n_tokens_max, bool add_special, bool parse_special) → int: @details Convert the provided text into tokens. @param tokens The tokens pointer must be large enough to hold the resulting tokens. @return Returns the number of tokens on success, no more than n_tokens_max @return Returns a negative number on failure - the number of tokens that would have been returned @return Returns INT32_MIN on overflow (e.g., tokenization result size exceeds int32_t limit) @param add_special Allow to add BOS and EOS tokens if model is configured to do so. @param parse_special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated as plaintext. Does not insert a leading space.
llama_vocab_bos(Pointer<llama_vocab> vocab) → int: Special tokens
llama_vocab_cls(Pointer<llama_vocab> vocab) → int: CLS is equivalent to BOS
llama_vocab_eos(Pointer<llama_vocab> vocab) → int
llama_vocab_eot(Pointer<llama_vocab> vocab) → int
llama_vocab_fim_mid(Pointer<llama_vocab> vocab) → int
llama_vocab_fim_pad(Pointer<llama_vocab> vocab) → int
llama_vocab_fim_pre(Pointer<llama_vocab> vocab) → int
llama_vocab_fim_rep(Pointer<llama_vocab> vocab) → int
llama_vocab_fim_sep(Pointer<llama_vocab> vocab) → int
llama_vocab_fim_suf(Pointer<llama_vocab> vocab) → int
llama_vocab_get_add_bos(Pointer<llama_vocab> vocab) → bool
llama_vocab_get_add_eos(Pointer<llama_vocab> vocab) → bool
llama_vocab_get_add_sep(Pointer<llama_vocab> vocab) → bool
llama_vocab_get_attr(Pointer<llama_vocab> vocab, Dartllama_token token) → llama_token_attr
llama_vocab_get_score(Pointer<llama_vocab> vocab, int token) → double
llama_vocab_get_text(Pointer<llama_vocab> vocab, int token) → Pointer<Char>: Vocab
llama_vocab_is_control(Pointer<llama_vocab> vocab, int token) → bool: Identify if Token Id is a control token or a render-able token
llama_vocab_is_eog(Pointer<llama_vocab> vocab, int token) → bool: Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.)
llama_vocab_mask(Pointer<llama_vocab> vocab) → int
llama_vocab_n_tokens(Pointer<llama_vocab> vocab) → int
llama_vocab_nl(Pointer<llama_vocab> vocab) → int
llama_vocab_pad(Pointer<llama_vocab> vocab) → int
llama_vocab_sep(Pointer<llama_vocab> vocab) → int
llama_vocab_type$1(Pointer<llama_vocab> vocab) → llama_vocab_type
llcs_engine_cancel(Pointer<llcs_engine> engine, int task_id) → void: Cancel a running task. Idempotent; safe to call after the task has already finished.
llcs_engine_caps(Pointer<llcs_engine> engine) → Pointer<Char>: Return a JSON object describing engine capabilities after model load: { "chat_template": "...", "supports_tools": bool, "supports_parallel_tool_calls": bool, "supports_reasoning": bool, "supports_vision": bool, "supports_audio": bool }
llcs_engine_create(Pointer<Char> params_json, Pointer<Pointer<Char>> error_out) → Pointer<llcs_engine>: Create an engine from a JSON params blob.
llcs_engine_destroy(Pointer<llcs_engine> engine) → void: Destroy the engine and release all resources. Safe to call with NULL.
llcs_engine_poll(Pointer<llcs_engine> engine, int task_id, int timeout_ms) → Pointer<Char>: Block up to timeout_ms for the next event for the given task.
llcs_engine_submit(Pointer<llcs_engine> engine, Pointer<Char> oai_request_json, Pointer<Pointer<Char>> error_out) → int: Submit an OAI-shaped chat completion request.
llcs_string_free(Pointer<Char> str) → void: Free a string returned by any llcs_engine_* function. Safe to call with NULL.
mtmd_bitmap_free(Pointer<mtmd_bitmap> bitmap) → void
mtmd_bitmap_get_data(Pointer<mtmd_bitmap> bitmap) → Pointer<UnsignedChar>
mtmd_bitmap_get_id(Pointer<mtmd_bitmap> bitmap) → Pointer<Char>: bitmap ID is optional, but useful for KV cache tracking these getters/setters are dedicated functions, so you can for example calculate the hash of the image based on mtmd_bitmap_get_data()
mtmd_bitmap_get_n_bytes(Pointer<mtmd_bitmap> bitmap) → int
mtmd_bitmap_get_nx(Pointer<mtmd_bitmap> bitmap) → int
mtmd_bitmap_get_ny(Pointer<mtmd_bitmap> bitmap) → int
mtmd_bitmap_init(int nx, int ny, Pointer<UnsignedChar> data) → Pointer<mtmd_bitmap>: mtmd_bitmap
mtmd_bitmap_init_from_audio(int n_samples, Pointer<Float> data) → Pointer<mtmd_bitmap>
mtmd_bitmap_is_audio(Pointer<mtmd_bitmap> bitmap) → bool
mtmd_bitmap_set_id(Pointer<mtmd_bitmap> bitmap, Pointer<Char> id) → void
mtmd_context_params_default() → mtmd_context_params
mtmd_decode_use_mrope(Pointer<mtmd_context> ctx) → bool: whether the current model use M-RoPE for llama_decode
mtmd_decode_use_non_causal(Pointer<mtmd_context> ctx, Pointer<mtmd_input_chunk> chunk) → bool: whether we need to set non-causal mask before llama_decode if chunk is nullptr, we assume the default case where chunk is an image chunk
mtmd_default_marker() → Pointer<Char>
mtmd_encode(Pointer<mtmd_context> ctx, Pointer<mtmd_image_tokens> image_tokens) → int: returns 0 on success TODO: deprecate
mtmd_encode_chunk(Pointer<mtmd_context> ctx, Pointer<mtmd_input_chunk> chunk) → int: returns 0 on success
mtmd_free(Pointer<mtmd_context> ctx) → void
mtmd_get_audio_sample_rate(Pointer<mtmd_context> ctx) → int: get audio sample rate in Hz, for example 16000 for Whisper return -1 if audio is not supported
mtmd_get_cap_from_file(Pointer<Char> mmproj_fname) → mtmd_caps
mtmd_get_output_embd(Pointer<mtmd_context> ctx) → Pointer<Float>: get output embeddings from the last encode pass the reading size (in bytes) is equal to: llama_model_n_embd_inp(model) * mtmd_input_chunk_get_n_tokens(chunk) * sizeof(float)
mtmd_helper_bitmap_init_from_buf(Pointer<mtmd_context> ctx, Pointer<UnsignedChar> buf, int len) → Pointer<mtmd_bitmap>: helper function to construct a mtmd_bitmap from a buffer containing a file supported formats: image: formats supported by stb_image: jpg, png, bmp, gif, etc. audio: formats supported by miniaudio: wav, mp3, flac note: audio files will be auto-detected based on magic bytes returns nullptr on failure this function is thread-safe
mtmd_helper_bitmap_init_from_file(Pointer<mtmd_context> ctx, Pointer<Char> fname) → Pointer<mtmd_bitmap>: helper function to construct a mtmd_bitmap from a file it calls mtmd_helper_bitmap_init_from_buf() internally returns nullptr on failure this function is thread-safe
mtmd_helper_decode_image_chunk(Pointer<mtmd_context> ctx, Pointer<llama_context> lctx, Pointer<mtmd_input_chunk> chunk, Pointer<Float> encoded_embd, int n_past, int seq_id, int n_batch, Pointer<Int32> new_n_past) → int: helper function to decode an image whose embeddings have already been calculated this helper will handle batching and pre/post decoding setup (for ex. gemma 3 requires non-causal attention) ret 0 on success, -1 on chunk not being a valid image chunk, 1 on decode failure
mtmd_helper_eval_chunk_single(Pointer<mtmd_context> ctx, Pointer<llama_context> lctx, Pointer<mtmd_input_chunk> chunk, int n_past, int seq_id, int n_batch, bool logits_last, Pointer<Int32> new_n_past) → int: works like mtmd_helper_eval_chunks(), but only for a single chunk this function is NOT thread-safe
mtmd_helper_eval_chunks(Pointer<mtmd_context> ctx, Pointer<llama_context> lctx, Pointer<mtmd_input_chunks> chunks, int n_past, int seq_id, int n_batch, bool logits_last, Pointer<Int32> new_n_past) → int: helper function that automatically:
mtmd_helper_get_n_pos(Pointer<mtmd_input_chunks> chunks) → int: helper to count the total position of tokens from a list of chunks, useful to keep track of n_past normally, n_pos is equal to n_tokens, but for M-RoPE it is different
mtmd_helper_get_n_tokens(Pointer<mtmd_input_chunks> chunks) → int: helper to count the total number of tokens from a list of chunks, useful to keep track of KV cache
mtmd_helper_image_get_decoder_pos(Pointer<mtmd_image_tokens> image, int pos_0, Pointer<mtmd_decoder_pos> out_pos) → void: helper to get the list of relative positions corresponding to the embedding tokens, to be used by M-RoPE out_pos must have length == mtmd_helper_get_n_tokens(image)
mtmd_helper_log_set(ggml_log_callback log_callback, Pointer<Void> user_data) → void: Set callback for all future logging events. If this is not called, or NULL is supplied, everything is output on stderr. Note: this also call mtmd_log_set() internally
mtmd_image_tokens_get_decoder_pos(Pointer<mtmd_image_tokens> image_tokens, int pos_0, int i) → mtmd_decoder_pos: get position for decoder attention, to be used by M-RoPE models i is the index of the embedding token, ranging from 0 to mtmd_image_tokens_get_n_tokens() - 1 pos_0 is the absolute position of the first token return relative position (for example, embedding 0 will have position (0, 0, 0); remember to adjust it to the current absolute position)
mtmd_image_tokens_get_id(Pointer<mtmd_image_tokens> image_tokens) → Pointer<Char>
mtmd_image_tokens_get_n_pos(Pointer<mtmd_image_tokens> image_tokens) → int: number of temporal positions (equals to max(t,h,w) for M-RoPE; equals to n_tokens otherwise)
mtmd_image_tokens_get_n_tokens(Pointer<mtmd_image_tokens> image_tokens) → int: mtmd_image_tokens
mtmd_image_tokens_get_nx(Pointer<mtmd_image_tokens> image_tokens) → int
mtmd_image_tokens_get_ny(Pointer<mtmd_image_tokens> image_tokens) → int
mtmd_init_from_file(Pointer<Char> mmproj_fname, Pointer<llama_model> text_model, mtmd_context_params ctx_params) → Pointer<mtmd_context>: initialize the mtmd context return nullptr on failure
mtmd_input_chunk_copy(Pointer<mtmd_input_chunk> chunk) → Pointer<mtmd_input_chunk>: in case you want to use custom logic to handle the chunk (i.e. KV cache management) you can move the chunk ownership to your own code by copying it remember to free the chunk when you are done with it
mtmd_input_chunk_free(Pointer<mtmd_input_chunk> chunk) → void
mtmd_input_chunk_get_id(Pointer<mtmd_input_chunk> chunk) → Pointer<Char>: returns nullptr for ID on text chunk
mtmd_input_chunk_get_n_pos(Pointer<mtmd_input_chunk> chunk) → int: number of temporal positions (equals to max(t,h,w) for M-RoPE; equals to n_tokens otherwise)
mtmd_input_chunk_get_n_tokens(Pointer<mtmd_input_chunk> chunk) → int
mtmd_input_chunk_get_tokens_image(Pointer<mtmd_input_chunk> chunk) → Pointer<mtmd_image_tokens>
mtmd_input_chunk_get_tokens_text(Pointer<mtmd_input_chunk> chunk, Pointer<Size> n_tokens_output) → Pointer<Int32>
mtmd_input_chunk_get_type(Pointer<mtmd_input_chunk> chunk) → mtmd_input_chunk_type: mtmd_input_chunk
mtmd_input_chunks_free(Pointer<mtmd_input_chunks> chunks) → void
mtmd_input_chunks_get(Pointer<mtmd_input_chunks> chunks, int idx) → Pointer<mtmd_input_chunk>
mtmd_input_chunks_init() → Pointer<mtmd_input_chunks>: mtmd_input_chunks
mtmd_input_chunks_size(Pointer<mtmd_input_chunks> chunks) → int
mtmd_log_set(ggml_log_callback log_callback, Pointer<Void> user_data) → void: Set callback for all future logging events. If this is not called, or NULL is supplied, everything is output on stderr.
mtmd_support_audio(Pointer<mtmd_context> ctx) → bool: whether the current model supports audio input
mtmd_support_vision(Pointer<mtmd_context> ctx) → bool: whether the current model supports vision input
mtmd_test_create_input_chunks() → Pointer<mtmd_input_chunks>: test function, to be used in test-mtmd-c-api.c
mtmd_tokenize(Pointer<mtmd_context> ctx, Pointer<mtmd_input_chunks> output, Pointer<mtmd_input_text> text, Pointer<Pointer<mtmd_bitmap>> bitmaps, int n_bitmaps) → int: tokenize an input text prompt and a list of bitmaps (images/audio) the prompt must have the input image marker (default: "<media>") in it the default marker is defined by mtmd_default_marker() the marker will be replaced with the image/audio chunk for example: "here is an image: <media>\ndescribe it in detail." this will gives 3 chunks:
noSuchMethod(Invocation invocation) → dynamic: Invoked when a nonexistent method or property is accessed.
inherited
toString() → String: A string representation of this object.
inherited

Operators

operator ==(Object other) → bool: The equality operator.
inherited

LlamaCppBindings class

Constructors

Properties

Methods

Operators

lib_llama_cpp_ffi library