diff --git a/CHANGELOG.md b/CHANGELOG.md index 9654d354e..69fc02b25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,97 @@ All notable changes to this project will be documented in this file. -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [0.3.44] Improved Windows DLL(OpenMP) Loading Reliability for GGML Backends + +- fix(ggml): preload bundled OpenMP runtime before loading ggml-base + - Preload the packaged `libomp140.x86_64.dll` on Windows before initializing + ggml-base to ensure CPU backend DLLs can resolve their OpenMP runtime + dependency. + - This only applies to Windows builds with llama-cpp-python >= 0.3.39 and + uses the bundled runtime from the package lib directory, avoiding the need + for users to configure system PATH or install additional OpenMP runtimes. + +- feat: Update llama.cpp to [ggml-org/llama.cpp/commit/846e991ec3c7ccec49112ff2c5b00b710e5f551d](https://github.com/ggml-org/llama.cpp/commit/846e991ec3c7ccec49112ff2c5b00b710e5f551d) + +## [0.3.43] Better llama.cpp ABI Compatibility, MTMD Performance and Extension API Support + +- patch(Gemma4ChatHandler): Synchronize huggingface gemma4 latest chat template + - fix: chat template — null handling, reasoning preservation, turn-tag balance, input validation + - https://huggingface.co/google/gemma-4-31B-it/commit/68abe48010cbe15293462fa11e901a60639a44e5 + +- feat(llama_ext): support optional llama-ext.h API bindings + - Add Python ctypes bindings for the experimental APIs exposed by + llama-ext.h, including NextN/MTP embeddings, and model metadata extraction. + - Extension symbols are loaded optionally to handle ABI changes, renamed + symbols, and builds that do not export experimental APIs without breaking + the main Python bindings. + +- feat(ctypes): handle missing optional symbols gracefully + - Allow ctypes bindings to mark symbols as optional through the `required` + flag. + - Missing symbols caused by ABI naming differences, API changes, or experimental + extensions will no longer break library loading. Optional APIs emit diagnostic + warnings and provide runtime unavailable stubs instead. + +- fix(ctypes): validate argument types before binding shared library functions + - Add explicit validation for ctypes function argument declarations before + assigning them to the loaded shared library function. + - This provides clearer error messages when invalid Python types are passed + to `argtypes`, instead of exposing the internal ctypes error about missing + `from_param()` methods. + +- fix(ctypes): validate argument types before binding shared library functions + - Add explicit validation for ctypes function argument declarations before + assigning them to the loaded shared library function. + - This provides clearer error messages when invalid Python types are passed + to `argtypes`, instead of exposing the internal ctypes error about missing + `from_param()` methods. + +- feat(ctypes): support ABI-compatible symbol aliases + - Allow ctypes_function_for_shared_library to accept either a single + symbol name or an ordered iterable of ABI-compatible aliases. + - Resolve aliases in order and bind the first exported symbol found while + preserving the selected symbol name for runtime diagnostics. Also improve + error reporting for empty alias lists and missing symbols. + +- refactor(mtmd): cache Generic MTMD chat template resolution for accelerate the processing speed of `__call__`. + - Refactor MTMD chat template handling to resolve and analyze the chat template only + once per handler instance instead of on every request. + - Add template initialization state, cache parsed media placeholder tags, and support + explicit chat template overrides through a dedicated field. Improve lifecycle cleanup + by resetting cached template state and MTMD resources during handler close. + - This keeps `GenericMTMDChatHandler` runtime processing focused on message rendering and media tokenization + while avoiding repeated chat template resolution overhead. + +- fix(mtmd): preserve subclass chat format during MTMD initialization + - Ensure MTMDChatHandler initialization remains compatible with specialized chat + handlers that define their own chat_format before calling super().__init__(). + - Initialize chat_format only when it is not already provided by the subclass, + then apply chat_format_override or fallback to the built-in MTMD template. + This prevents AttributeError during inherited handler initialization while + keeping template override behavior unchanged. + +- refactor(embedding): rename `llama_cpp` import alias to `llama_cpp_lib` + - Rename the `llama_cpp.llama_cpp` import alias to `llama_cpp_lib` to avoid potential namespace conflicts with the local `.llama_cpp` imports. Update all affected call sites in `llama_embedding.py`. + +- patch(Llama): Increase chunk preview limit to 128 in Llama.eval exception + - Raises the maximum tokens captured for the error message preview from 16 to 128, improving visibility into the offending chunk during fatal backend crashes. + +- ci(metal): get package version from importlib metadata + * Avoid importing llama_cpp when detecting the package version. + * This prevents initialization side effects and keeps CI version extraction reliable. + +- feat: Update llama.cpp to [ggml-org/llama.cpp/commit/86d86ed4396b4130922f7b9af26e3d9fc11a591b](https://github.com/ggml-org/llama.cpp/commit/86d86ed4396b4130922f7b9af26e3d9fc11a591b) + +- feat: Sync llama.cpp llama/mtmd/ggml API Binding 20260716 + +More information see: https://github.com/JamePeng/llama-cpp-python/compare/e522cecb93907c67ffe2e339b7009c93d3fb0f59...a64128351a1d04c6dd644e3908070f7ea2002f20 + ## [0.3.42] More Reliable Dynamic Backend Loading, Safer MTMD Processing, and Advanced Batch Support - fix(loader): improve Windows DLL search path handling and diagnostics diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index c6676c294..10e452d5f 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.42" +__version__ = "0.3.44" diff --git a/llama_cpp/_ctypes_extensions.py b/llama_cpp/_ctypes_extensions.py index 1b4c05ff2..a9a2c02e5 100644 --- a/llama_cpp/_ctypes_extensions.py +++ b/llama_cpp/_ctypes_extensions.py @@ -5,10 +5,12 @@ import ctypes import functools import pathlib +import importlib.metadata from ctypes.util import find_library from typing import ( Any, Callable, + Iterable, List, Union, Optional, @@ -18,6 +20,15 @@ ) from typing_extensions import TypeAlias +def _version_at_least(version: str) -> bool: + """Check whether installed llama-cpp-python version meets requirement.""" + try: + current = importlib.metadata.version("llama-cpp-python") + from packaging.version import Version + return Version(current) >= Version(version) + except Exception: + return False + def _format_library_dir_contents(base_paths: list[pathlib.Path]) -> str: """Format directory contents for diagnostics after library loading fails.""" sections = [] @@ -201,20 +212,100 @@ class CtypesRef(Generic[CtypesCData]): def ctypes_function_for_shared_library(lib: ctypes.CDLL): - """Decorator for defining ctypes functions with type hints""" + """Create a decorator used to bind typed Python declarations to C symbols. + + The returned decorator accepts either a single exported symbol name or an + iterable of ABI-compatible aliases. When aliases are provided, they are + checked in order and the first available symbol is selected. + """ def ctypes_function( - name: str, argtypes: List[Any], restype: Any, enabled: bool = True + name: Union[str, Iterable[str]], + argtypes: List[Any], + restype: Any, + enabled: bool = True, + required: bool = True, ): + """Bind a Python declaration to one of the requested C symbols. + + Args: + name: A symbol name or an ordered iterable of compatible aliases. + argtypes: The ctypes argument types assigned to the C function. + restype: The ctypes return type assigned to the C function. + enabled: Return the original Python declaration when disabled. + required: Raise if symbol is missing. If False, create a runtime unavailable stub. + + Raises: + ValueError: If no symbol names are provided. + AttributeError: If none of the requested symbols exist in the + shared library. + """ + symbol_names = (name,) if isinstance(name, str) else tuple(name) + + if not symbol_names: + raise ValueError("At least one shared library symbol name is required") + def decorator(f: F) -> F: - if enabled: - func = getattr(lib, name) + if not enabled: + return f + + for symbol_name in symbol_names: + try: + func = getattr(lib, symbol_name) + except AttributeError: + continue + # Validate ctypes argument declarations before assigning them. + # ctypes requires every argtype to provide from_param(). + for index, argtype in enumerate(argtypes): + if not hasattr(argtype, "from_param"): + raise TypeError( + "Invalid ctypes argument type:\n" + f" function: {f.__name__}\n" + f" symbol: {symbol_name}\n" + f" arg index: {index}\n" + f" arg type: {argtype!r}\n" + f" expected: a ctypes type with from_param()" + ) + func.argtypes = argtypes func.restype = restype - functools.wraps(f)(func) + functools.update_wrapper(func, f) + + # Preserve the actual exported symbol selected at runtime for + # diagnostics, especially when ABI aliases are being used. + func.__ctypes_symbol_name__ = symbol_name return func - else: - return f + + message = ( + "None of the shared library symbols were found: " + + ", ".join(symbol_names) + ) + + if required: + raise AttributeError(message) + + # Optional extension API. + # Keep import working when the symbol is unavailable. + print( + "[llama-cpp-python].ctypes_function: WARNING! optional API unavailable\n" + f" symbols: {', '.join(symbol_names)}\n" + f" library: {getattr(lib, '_name', '')}" + ) + + def unavailable(*args, **kwargs): + raise RuntimeError( + "This llama.cpp extension API is unavailable.\n" + f"Required symbol(s): {', '.join(symbol_names)}\n" + f"Library: {getattr(lib, '_name', '')}" + ) + + functools.update_wrapper(unavailable, f) + + # Mark unavailable extension API. + unavailable.__ctypes_symbol_name__ = None + unavailable.__ctypes_optional__ = True + + return unavailable return decorator diff --git a/llama_cpp/_ggml.py b/llama_cpp/_ggml.py index 0cb20d40e..9a7dac517 100644 --- a/llama_cpp/_ggml.py +++ b/llama_cpp/_ggml.py @@ -6,10 +6,9 @@ import enum import os import pathlib - from llama_cpp._ctypes_extensions import ( + _version_at_least, load_shared_library, - byref, ctypes_function_for_shared_library, ) @@ -21,12 +20,46 @@ TYPE_CHECKING, ) +def _preload_openmp_runtime(): + """Preload bundled OpenMP runtime before loading ggml-base. + + This is required on Windows when CPU backends depend on the packaged + OpenMP runtime DLL. + """ + + # Only Windows DLL loading requires this workaround. + if os.name != "nt": + return + + # Keep compatibility with older package versions. + if not _version_at_least("0.3.39"): + return + + libomp_path = (pathlib.Path(__file__).parent / "lib" / "libomp140.x86_64.dll") + + if not libomp_path.exists(): + print(f"[llama-cpp-python] WARNING: bundled OpenMP runtime not found: {libomp_path}") + return + + try: + ctypes.CDLL(str(libomp_path), winmode=ctypes.RTLD_GLOBAL) + print(f"[llama-cpp-python] loaded bundled OpenMP runtime: {libomp_path}") + except Exception as e: + print( + "[llama-cpp-python] WARNING: failed to load bundled OpenMP runtime:\n" + f" path: {libomp_path}\n" + f" error: {e}" + ) + libggml_base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) libggml_base_paths = [ libggml_base_path / "lib", libggml_base_path / "bin", ] +# Load bundled OpenMP runtime before ggml-base on Windows. +_preload_openmp_runtime() + libggml_base = load_shared_library("ggml-base", libggml_base_paths) ggml_base_function = ctypes_function_for_shared_library(libggml_base) @@ -122,6 +155,7 @@ class GGMLStatus(enum.IntEnum): # GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block) # GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) # GGML_TYPE_Q1_0 = 41, +# GGML_TYPE_Q2_0 = 42, # GGML_TYPE_COUNT = 42, # }; class GGMLType(enum.IntEnum): @@ -159,7 +193,8 @@ class GGMLType(enum.IntEnum): GGML_TYPE_MXFP4 = 39 GGML_TYPE_NVFP4 = 40 GGML_TYPE_Q1_0 = 41 - GGML_TYPE_COUNT = 42 + GGML_TYPE_Q2_0 = 42 + GGML_TYPE_COUNT = 43 # // precision @@ -201,6 +236,7 @@ class GGMLPrec(enum.IntEnum): # GGML_FTYPE_MOSTLY_MXFP4 = 25, // except 1d tensors # GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors # GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors +# GGML_FTYPE_MOSTLY_Q2_0 = 28, // except 1d tensors # }; class GGMLFType(enum.IntEnum): GGML_FTYPE_UNKNOWN = -1 @@ -230,6 +266,7 @@ class GGMLFType(enum.IntEnum): GGML_FTYPE_MOSTLY_MXFP4 = 25 GGML_FTYPE_MOSTLY_NVFP4 = 26 GGML_FTYPE_MOSTLY_Q1_0 = 27 + GGML_FTYPE_MOSTLY_Q2_0 = 28 # // available tensor operations: diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index a4086fd70..2f402cd74 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -10,6 +10,7 @@ ggml_backend_sched_eval_callback, ggml_log_callback, ggml_opt_get_optimizer_params, + ggml_cgraph ) from typing import ( @@ -364,6 +365,7 @@ class llama_token_type(enum.IntEnum): # LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors # LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors # LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors +# LLAMA_FTYPE_MOSTLY_Q2_0 = 41, // except 1d tensors # # LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file # }; @@ -372,6 +374,9 @@ class llama_ftype(enum.IntEnum): LLAMA_FTYPE_MOSTLY_F16 = 1 LLAMA_FTYPE_MOSTLY_Q4_0 = 2 LLAMA_FTYPE_MOSTLY_Q4_1 = 3 + # LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4 + # LLAMA_FTYPE_MOSTLY_Q4_2 = 5 + # LLAMA_FTYPE_MOSTLY_Q4_3 = 6 LLAMA_FTYPE_MOSTLY_Q8_0 = 7 LLAMA_FTYPE_MOSTLY_Q5_0 = 8 LLAMA_FTYPE_MOSTLY_Q5_1 = 9 @@ -406,6 +411,7 @@ class llama_ftype(enum.IntEnum): LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38 LLAMA_FTYPE_MOSTLY_NVFP4 = 39 LLAMA_FTYPE_MOSTLY_Q1_0 = 40 + LLAMA_FTYPE_MOSTLY_Q2_0 = 41 LLAMA_FTYPE_GUESSED = 1024 # // Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium" @@ -428,7 +434,7 @@ def llama_ftype_name( # LLAMA_ROPE_SCALING_TYPE_LINEAR = 1, # LLAMA_ROPE_SCALING_TYPE_YARN = 2, # LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3, -# LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_YARN, +# LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_LONGROPE, # }; class llama_rope_scaling_type(enum.IntEnum): LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1 @@ -436,7 +442,7 @@ class llama_rope_scaling_type(enum.IntEnum): LLAMA_ROPE_SCALING_TYPE_LINEAR = 1 LLAMA_ROPE_SCALING_TYPE_YARN = 2 LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3 - LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_YARN + LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_LONGROPE # enum llama_pooling_type { # LLAMA_POOLING_TYPE_UNSPECIFIED = -1, @@ -5064,3 +5070,295 @@ def llama_opt_epoch( callback_eval: ctypes.c_void_p, / ): ... + +############################## +# // llama.cpp/src/llama-ext.h +############################## + +# // this is a staging header for new llama.cpp API +# // breaking changes and C++ are allowed. everything here should be considered WIP +# // try as much as possible to not include this header in the rest of the codebase + +ctypes_function_llama_ext = ctypes_function_for_shared_library(_lib) + +# // Reserve a new compute graph. It is valid until the next call to llama_graph_reserve. +# LLAMA_API struct ggml_cgraph * llama_graph_reserve( +# struct llama_context * ctx, +# uint32_t n_tokens, +# uint32_t n_seqs, +# uint32_t n_outputs); +@ctypes_function_llama_ext( + [ + "llama_graph_reserve", + "?llama_graph_reserve@@YAPEAUggml_cgraph@@PEAUllama_context@@III@Z", + "__Z19llama_graph_reserveP13llama_contextjjj", + ], + [llama_context_p_ctypes, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32], + ctypes.POINTER(ggml_cgraph), + required=False, +) +def llama_graph_reserve( + ctx: llama_context_p, + n_tokens: ctypes.c_uint32, + n_seqs: ctypes.c_uint32, + n_outputs: ctypes.c_uint32, +) -> ctypes.POINTER(ggml_cgraph): # type: ignore + """ + Reserve a new compute graph. It is valid until the next call to llama_graph_reserve. + """ + ... + +# // Get the default ggml_type for a given ftype. +# LLAMA_API ggml_type llama_ftype_get_default_type(llama_ftype ftype); +@ctypes_function_llama_ext( + [ + "llama_ftype_get_default_type", + "?llama_ftype_get_default_type@@YA?AW4ggml_type@@W4llama_ftype@@@Z", + "__Z28llama_ftype_get_default_type11llama_ftype", + ], + [ctypes.c_int], + int, + required=False, +) +def llama_ftype_get_default_type( + ftype: llama_ftype +) -> int: + """ + Get the default ggml_type for a given ftype. + """ + ... + +# LLAMA_API int32_t llama_model_n_expert (const struct llama_model * model); +@ctypes_function_llama_ext( + [ + "llama_model_n_expert", + "?llama_model_n_expert@@YAHPEBUllama_model@@@Z", + "__Z20llama_model_n_expertPK11llama_model", + ], + [llama_model_p_ctypes], + ctypes.c_int32, + required=False, +) +def llama_model_n_expert( + model: llama_model_p +) -> ctypes.c_int32: + ... + +# LLAMA_API int32_t llama_model_n_devices(const struct llama_model * model); +@ctypes_function_llama_ext( + [ + "llama_model_n_devices", + "?llama_model_n_devices@@YAHPEBUllama_model@@@Z", + "__Z21llama_model_n_devicesPK11llama_model", + ], + [llama_model_p_ctypes], + ctypes.c_int32, + required=False, +) +def llama_model_n_devices( + model: llama_model_p +) -> ctypes.c_int32: + ... + +# LLAMA_API ggml_backend_dev_t llama_model_get_device(const struct llama_model * model, int i); +@ctypes_function_llama_ext( + [ + "llama_model_get_device", + "?llama_model_get_device@@YAPEAUggml_backend_device@@PEBUllama_model@@H@Z", + "__Z22llama_model_get_devicePK11llama_modeli", + ], + [llama_model_p_ctypes, ctypes.c_int], + ctypes.c_void_p, + required=False, +) +def llama_model_get_device( + model: llama_model_p, + i: int, +) -> ctypes.c_void_p: + ... + +# // Set whether the context outputs nextn embeddings or not +# // If masked == true, output the embeddings only for the tokens with batch.logits != 0 +# // If masked == false, output the embeddings for all tokens in the batch regardless of batch.logits +# LLAMA_API void llama_set_embeddings_nextn(struct llama_context * ctx, bool value, bool masked); +@ctypes_function_llama_ext( + [ + "llama_set_embeddings_nextn", + "?llama_set_embeddings_nextn@@YAXPEAUllama_context@@_N1@Z", + "__Z26llama_set_embeddings_nextnP13llama_contextbb", + ], + [llama_context_p_ctypes, ctypes.c_bool, ctypes.c_bool], + None, + required=False, +) +def llama_set_embeddings_nextn( + ctx: llama_context_p, + value: bool, + masked: bool, +): + """ + Set whether the context outputs nextn embeddings or not + If masked == true, output the embeddings only for the tokens with batch.logits != 0 + If masked == false, output the embeddings for all tokens in the batch regardless of batch.logits + """ + ... + +# // Select which appended NextN block the DECODER_MTP graph runs (offset past +# // the trunk: il = n_layer() + offset). Used by the speculative NextN driver to +# // chain multiple trained NextN heads. Default 0 (first head). +# LLAMA_API void llama_set_nextn_layer_offset(struct llama_context * ctx, int32_t offset); +@ctypes_function_llama_ext( + [ + "llama_set_nextn_layer_offset", + "?llama_set_nextn_layer_offset@@YAXPEAUllama_context@@H@Z", + "__Z28llama_set_nextn_layer_offsetP13llama_contexti", + ], + [llama_context_p_ctypes, ctypes.c_int32], + None, + required=False, +) +def llama_set_nextn_layer_offset( + ctx: llama_context_p, + offset: ctypes.c_int32, +): + """ + Select which appended NextN block the DECODER_MTP graph runs (offset past + the trunk: il = n_layer() + offset). Used by the speculative NextN driver to + chain multiple trained NextN heads. Default 0 (first head). + """ + ... + +# // mirrors: +# // LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); +# LLAMA_API float * llama_get_embeddings_nextn(struct llama_context * ctx); +@ctypes_function_llama_ext( + [ + "llama_get_embeddings_nextn", + "?llama_get_embeddings_nextn@@YAPEAMPEAUllama_context@@@Z", + "__Z26llama_get_embeddings_nextnP13llama_context", + ], + [llama_context_p_ctypes], + ctypes.POINTER(ctypes.c_float), + required=False, +) +def llama_get_embeddings_nextn( + ctx: llama_context_p, +) -> ctypes.POINTER(ctypes.c_float): # type: ignore + ... + +# // LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i); +# LLAMA_API float * llama_get_embeddings_nextn_ith(struct llama_context * ctx, int32_t i); +@ctypes_function_llama_ext( + [ + "llama_get_embeddings_nextn_ith", + "?llama_get_embeddings_nextn_ith@@YAPEAMPEAUllama_context@@H@Z", + "__Z30llama_get_embeddings_nextn_ithP13llama_contexti", + ], + [llama_context_p_ctypes, ctypes.c_int32], + ctypes.POINTER(ctypes.c_float), + required=False, +) +def llama_get_embeddings_nextn_ith( + ctx: llama_context_p, + i: ctypes.c_int32, +) -> ctypes.POINTER(ctypes.c_float): # type: ignore + ... + +# // Set whether the context outputs the input embeddings of a specific layer +# LLAMA_API void llama_set_embeddings_layer_inp(struct llama_context * ctx, uint32_t lid, bool value); +@ctypes_function_llama_ext( + [ + "llama_set_embeddings_layer_inp", + "?llama_set_embeddings_layer_inp@@YAXPEAUllama_context@@I_N@Z", + "__Z30llama_set_embeddings_layer_inpP13llama_contextjb", + ], + [llama_context_p_ctypes, ctypes.c_int32, ctypes.c_bool], + ctypes.POINTER(ctypes.c_float), + required=False, +) +def llama_set_embeddings_layer_inp( + ctx: llama_context_p, + lid: ctypes.c_int32, + value: bool, +) -> ctypes.POINTER(ctypes.c_float): # type: ignore + """ + Set whether the context outputs the input embeddings of a specific layer + """ + ... + +# // mirrors: +# // LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); +# LLAMA_API float * llama_get_embeddings_layer_inp(struct llama_context * ctx, uint32_t lid); +@ctypes_function_llama_ext( + [ + "llama_get_embeddings_layer_inp", + "?llama_get_embeddings_layer_inp@@YAPEAMPEAUllama_context@@I@Z", + "__Z30llama_get_embeddings_layer_inpP13llama_contextj", + ], + [llama_context_p_ctypes, ctypes.c_int32], + ctypes.POINTER(ctypes.c_float), + required=False, +) +def llama_get_embeddings_layer_inp( + ctx: llama_context_p, + lid: ctypes.c_int32, +) -> ctypes.POINTER(ctypes.c_float): # type: ignore + ... + +# LLAMA_API llama_context * llama_get_ctx_other(struct llama_context * ctx); +@ctypes_function_llama_ext( + [ + "llama_get_ctx_other", + "?llama_get_ctx_other@@YAPEAUllama_context@@PEAU1@@Z", + "__Z19llama_get_ctx_otherP13llama_context", + ], + [llama_context_p_ctypes], + llama_context_p_ctypes, + required=False, +) +def llama_get_ctx_other( + ctx: llama_context_p, +) -> llama_context_p: + ... + +# // model/context data extraction + +# // returns pointer to the target-model layer indices +# LLAMA_API const int32_t * llama_model_target_layer_ids (const struct llama_model * model); +@ctypes_function_llama_ext( + [ + "llama_model_target_layer_ids", + "?llama_model_target_layer_ids@@YAPEBHPEBUllama_model@@@Z", + "__Z28llama_model_target_layer_idsPK11llama_model", + ], + [llama_model_p_ctypes], + ctypes.POINTER(ctypes.c_int32), + required=False, +) +def llama_model_target_layer_ids( + model: llama_model_p +) -> ctypes.POINTER(ctypes.c_int32): # type: ignore + """ + returns pointer to the target-model layer indices + """ + ... + +# // returns the number of extracted layers from target model +# LLAMA_API uint32_t llama_model_target_layer_ids_n(const struct llama_model * model); +@ctypes_function_llama_ext( + [ + "llama_model_target_layer_ids_n", + "?llama_model_target_layer_ids_n@@YAIPEBUllama_model@@@Z", + "__Z30llama_model_target_layer_ids_nPK11llama_model" + ], + [llama_model_p_ctypes], + ctypes.POINTER(ctypes.c_uint32), + required=False, +) +def llama_model_target_layer_ids_n( + model: llama_model_p +) -> ctypes.POINTER(ctypes.c_uint32): # type: ignore + """ + returns the number of extracted layers from target model + """ + ... diff --git a/llama_cpp/llama_embedding.py b/llama_cpp/llama_embedding.py index 0c1df339c..d8f6e6bf2 100644 --- a/llama_cpp/llama_embedding.py +++ b/llama_cpp/llama_embedding.py @@ -1,6 +1,6 @@ import numpy as np from typing import Union, List, Optional, Dict, Any, Tuple -import llama_cpp.llama_cpp as llama_cpp +import llama_cpp.llama_cpp as llama_cpp_lib from .llama_types import Embedding from .llama import Llama # Pooling types from .llama_cpp @@ -141,7 +141,7 @@ def embed( # Determine the output dimension if is_rank: - out_dim = llama_cpp.llama_model_n_cls_out(self._model.model) + out_dim = llama_cpp_lib.llama_model_n_cls_out(self._model.model) else: out_dim = self.n_embd() @@ -166,9 +166,9 @@ def embed( # Reset Context and Batch if self.verbose: - llama_cpp.llama_perf_context_reset(ctx) + llama_cpp_lib.llama_perf_context_reset(ctx) self._batch.reset() - llama_cpp.llama_memory_clear(llama_cpp.llama_get_memory(ctx), True) + llama_cpp_lib.llama_memory_clear(llama_cpp_lib.llama_get_memory(ctx), True) # Initialize State Variables results: List[Any] = [] @@ -190,7 +190,7 @@ def _decode_batch(): doc_tokens_embd = [] for _ in range(seq_len): # Get the vector of the i-th token - ptr = llama_cpp.llama_get_embeddings_ith(ctx, curr_token_idx) + ptr = llama_cpp_lib.llama_get_embeddings_ith(ctx, curr_token_idx) if ptr is None: # Fallback: append zero vector or skip (here we zero-pad to keep shape) doc_tokens_embd.append([0.0] * out_dim) @@ -207,7 +207,7 @@ def _decode_batch(): else: for i in range(len(batch_seq_lens)): # Obtain the vector of the i-th sequence. - ptr = llama_cpp.llama_get_embeddings_seq(ctx, i) + ptr = llama_cpp_lib.llama_get_embeddings_seq(ctx, i) data = ptr[:out_dim] if not is_rank: @@ -219,7 +219,7 @@ def _decode_batch(): results.append(data) self._batch.reset() - llama_cpp.llama_memory_clear(llama_cpp.llama_get_memory(ctx), True) + llama_cpp_lib.llama_memory_clear(llama_cpp_lib.llama_get_memory(ctx), True) batch_seq_lens = [] # Main Streaming Loop @@ -272,7 +272,7 @@ def _decode_batch(): _decode_batch() if self.verbose: - llama_cpp.llama_perf_context_print(ctx) + llama_cpp_lib.llama_perf_context_print(ctx) final_result = results[0] if is_single else results diff --git a/llama_cpp/llama_multimodal.py b/llama_cpp/llama_multimodal.py index a8802baf6..cc159924f 100644 --- a/llama_cpp/llama_multimodal.py +++ b/llama_cpp/llama_multimodal.py @@ -155,19 +155,28 @@ def __init__( f"{self.log_prefix}(__init__): `extra_template_arguments` must be a dict." ) + # Preserve subclass attributes + if not hasattr(self, "chat_format"): + self.chat_format = None + + self.chat_format_override = chat_template_override self.extra_template_arguments: dict[str, Any] = dict(extra_template_arguments or {}) self.is_support_vision = False self.is_support_audio = False self.is_support_video = False + self.chat_template = None + self._chat_format_parser_tags = [] + self._template_initialized = False + # Pre-compile Jinja template - if (not hasattr(self, "chat_format") or self.chat_format is None) and chat_template_override is None: - self.chat_format = self.CHAT_FORMAT - elif chat_template_override is not None: - self.chat_format = chat_template_override + if self.chat_format is None: + if self.chat_format_override is not None: + self.chat_format = self.chat_format_override + else: + self.chat_format = self.CHAT_FORMAT - self._chat_format_parser_tags = [] self._change_chat_template(self.chat_format) self._exit_stack = ExitStack() @@ -250,11 +259,15 @@ def close(self) -> None: if getattr(self, "mtmd_ctx", None) is not None: try: self._mtmd_cpp.mtmd_free(self.mtmd_ctx) + self.mtmd_ctx = None except Exception: pass - self.mtmd_ctx = None - self.mctx_params = None - self.chat_template = None + self.mctx_params = None + self.chat_format = None + self.chat_template = None + self.chat_template_override = None + self._template_initialized = False + self._chat_format_parser_tags = [] if getattr(self, "_exit_stack", None) is not None and hasattr(self._exit_stack, "close"): self._exit_stack.close() @@ -727,7 +740,9 @@ def _mtmd_tokenize( ) input_text = self._mtmd_cpp.mtmd_input_text() - input_text.text = ctypes.c_char_p(text.encode("utf-8")) + encoded_text = text.encode("utf-8") + input_text.text = ctypes.c_char_p(encoded_text) + input_text.text_len = len(encoded_text) input_text.add_special = (llama.n_tokens == 0) input_text.parse_special = True @@ -1659,8 +1674,18 @@ def _resolve_chat_format(self, llama: llama_core.Llama) -> str: self.chat_format = chat_format return chat_format - def __call__(self, **kwargs): - llama = kwargs["llama"] + def _ensure_chat_template( + self, + llama: llama_core.Llama, + ) -> None: + """ + Resolve and analyze chat template once. + + Chat template metadata is static for a model instance, + so it should not be recomputed for every request. + """ + if self._template_initialized: + return self._resolve_chat_format(llama) @@ -1673,7 +1698,18 @@ def __call__(self, **kwargs): "a model that provides tokenizer.chat_template metadata." ) - self._chat_format_parser_tags = [tag for tag in self.KNOWN_MEDIA_TAGS if tag in self.chat_format] + self._chat_format_parser_tags = [ + tag + for tag in self.KNOWN_MEDIA_TAGS + if tag in self.chat_format + ] + + self._template_initialized = True + + def __call__(self, **kwargs): + llama = kwargs["llama"] + + self._ensure_chat_template(llama) if self.verbose: print(f"{self.log_prefix} - Start processing", file=sys.stderr) @@ -2546,7 +2582,9 @@ class Gemma4ChatHandler(MTMDChatHandler): " }\n" "{%- endmacro -%}\n" "{%- macro format_argument(argument, escape_keys=True) -%}\n" - " {%- if argument is string -%}\n" + " {%- if argument is none -%}\n" + " {{- 'null' -}}\n" + " {%- elif argument is string -%}\n" " {{- '<|\"|>' + argument + '<|\"|>' -}}\n" " {%- elif argument is boolean -%}\n" " {{- 'true' if argument else 'false' -}}\n" @@ -2602,18 +2640,21 @@ class Gemma4ChatHandler(MTMDChatHandler): " {{- '' -}}\n" "{%- endmacro -%}\n" "\n" + "{#- ===== SETUP ===== -#}" "{%- set ns = namespace(prev_message_type=None) -%}\n" "{%- set loop_messages = messages -%}\n" + "{%- set enable_thinking = enable_thinking | default(false) -%}\n" + "{%- set preserve_thinking = preserve_thinking | default(false) -%}\n" "{{- bos_token -}}\n" "{#- Handle System/Tool Definitions Block -#}\n" - "{%- if (enable_thinking is defined and enable_thinking) or tools or messages[0]['role'] in ['system', 'developer'] -%}\n" + "{%- if enable_thinking or tools or (messages and messages[0]['role'] in ['system', 'developer']) -%}\n" " {{- '<|turn>system\\n' -}}\n" " {#- Inject Thinking token at the very top of the FIRST system turn -#}\n" - " {%- if enable_thinking is defined and enable_thinking -%}\n" + " {%- if enable_thinking -%}\n" " {{- '<|think|>\\n' -}}\n" " {%- set ns.prev_message_type = 'think' -%}\n" " {%- endif -%}\n" - " {%- if messages[0]['role'] in ['system', 'developer'] -%}\n" + " {%- if messages and messages[0]['role'] in ['system', 'developer'] -%}\n" " {%- if messages[0]['content'] is string -%}\n" " {{- messages[0]['content'] | trim -}}\n" " {%- elif messages[0]['content'] is sequence -%}\n" @@ -2647,31 +2688,21 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- if message['role'] != 'tool' -%}\n" " {%- set ns.prev_message_type = None -%}\n" " {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}\n" - " {#- Detect continuation: suppress duplicate <|turn>model when previous non-tool message was also assistant -#}\n" - " {%- set prev_nt = namespace(role=None, found=false) -%}\n" - " {%- if loop.index0 > 0 -%}\n" - " {%- for j in range(loop.index0 - 1, -1, -1) -%}\n" - " {%- if not prev_nt.found -%}\n" - " {%- if loop_messages[j]['role'] != 'tool' -%}\n" - " {%- set prev_nt.role = loop_messages[j]['role'] -%}\n" - " {%- set prev_nt.found = true -%}\n" - " {%- endif -%}\n" - " {%- endif -%}\n" - " {%- endfor -%}\n" - " {%- endif -%}\n" - " {%- set continue_same_model_turn = (role == 'model' and prev_nt.role == 'assistant') -%}\n" + "{#- Detect continuation using tracked state — O(1) instead of O(n) backward scan -#}\n" + "{%- set continue_same_model_turn = (role == 'model' and ns.prev_non_tool_role == 'assistant') -%}\n" " {%- if not continue_same_model_turn -%}\n" " {{- '<|turn>' + role + '\\n' }}\n" " {%- endif -%}\n" "\n" " {#- Render reasoning/reasoning_content as thinking channel -#}\n" " {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%}\n" - " {%- if thinking_text and loop.index0 > ns_turn.last_user_idx and message.get('tool_calls') -%}\n" - " {{- '<|channel>thought\\n' + thinking_text + '\\n' -}}\n" + " {%- set thinking_gate = (loop.index0 > ns_turn.last_user_idx) or (preserve_thinking and message.get('tool_calls')) -%}\n" + " {%- if thinking_text and thinking_gate -%}\n" + " {{- '<|channel>thought\n' + thinking_text + '\n' -}}\n" " {%- endif -%}\n" "\n" " {%- if message.get('tool_calls') -%}\n" - " {%- for tool_call in message['tool_calls'] -%}\n" + " {%- for tool_call in message.get('tool_calls') -%}\n" " {%- set function = tool_call['function'] -%}\n" " {{- '<|tool_call>call:' + function['name'] + '{' -}}\n" " {%- if function['arguments'] is mapping -%}\n" @@ -2681,8 +2712,13 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- set ns_args.found_first = true -%}\n" " {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n" " {%- endfor -%}\n" - " {%- elif function['arguments'] is string -%}\n" - " {{- function['arguments'] -}}\n" + " {%- elif function['arguments'] is none -%}\n" + " {%- else -%}\n" + " {{- raise_exception(\n" + " \"chat_template: tool_calls[].function.arguments must be a \"\n" + " \"JSON object (mapping), not a string. Deserialize arguments \"\n" + " \"before passing to the template.\"\n" + " ) -}}\n" " {%- endif -%}\n" " {{- '}' -}}\n" " {%- endfor -%}\n" @@ -2692,8 +2728,8 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- set ns_tr_out = namespace(flag=false) -%}\n" " {%- if message.get('tool_responses') -%}\n" " {#- Legacy: tool_responses embedded on the assistant message (Google/Gemma native) -#}\n" - " {%- for tool_response in message['tool_responses'] -%}\n" - " {{- format_tool_response_block(tool_response['name'] | default('unknown'), tool_response['response']) -}}\n" + " {%- for tool_response in message.get('tool_responses') -%}\n" + " {{- format_tool_response_block(tool_response['name'] | default('unknown', true), tool_response['response']) -}}\n" " {%- set ns_tr_out.flag = true -%}\n" " {%- set ns.prev_message_type = 'tool_response' -%}\n" " {%- endfor -%}\n" @@ -2707,8 +2743,8 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- else -%}\n" " {%- set follow = loop_messages[k] -%}\n" " {#- Resolve tool_call_id to function name -#}\n" - " {%- set ns_tname = namespace(name=follow.get('name') | default('unknown')) -%}\n" - " {%- for tc in message['tool_calls'] -%}\n" + " {%- set ns_tname = namespace(name=follow.get('name') or 'unknown') -%}\n" + " {%- for tc in message.get('tool_calls') -%}\n" " {%- if tc.get('id') == follow.get('tool_call_id') -%}\n" " {%- set ns_tname.name = tc['function']['name'] -%}\n" " {%- endif -%}\n" @@ -2726,9 +2762,14 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- endfor -%}\n" " {{- format_tool_response_block(ns_tname.name, ns_txt.s) -}}\n" " {%- for part in tool_body -%}\n" - " {%- if part.get('type') == 'image_url' -%}\n" - " {%- set url_val = part['image_url'] if part['image_url'] is string else part['image_url']['url'] -%}\n" - " {{- '<|image|>' + url_val -}}\n" + " {%- if part.get('type') in ['image', 'image_url'] -%}\n" + " {%- if part.get('type') == 'image_url' -%}\n" + " {%- set url_val = part['image_url'] if part['image_url'] is string else part['image_url']['url'] -%}\n" + " {{- '<|image|>' + url_val -}}\n" + " {%- elif part.get('type') == 'image' -%}\n" + " {%- set url_val = part['image'] if part['image'] is string else part['image']['url'] -%}\n" + " {{- '<|image|>' + url_val -}}\n" + " {%- endif -%}\n" " {%- elif part.get('type') in ['audio_url', 'input_audio'] -%}\n" " {%- if part.get('type') == 'audio_url' -%}\n" " {%- set audio_val = part['audio_url'] if part['audio_url'] is string else part['audio_url']['url'] -%}\n" @@ -2737,9 +2778,14 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- set audio_val = part['input_audio'] if part['input_audio'] is string else ('data:audio/' + part['input_audio']['format'] + ';base64,' + part['input_audio']['data']) -%}\n" " {{- '<|audio|>' + audio_val -}}\n" " {%- endif -%}\n" - # " {%- elif part.get('type') == 'video_url' -%}\n" - # " {%- set video_val = part['video_url'] if part['video_url'] is string else part['video_url']['url'] -%}\n" - # " {{- '<|video|>' + video_val -}}\n" + " {%- elif part.get('type') in ['video', 'video_url'] -%}\n" + " {%- if part.get('type') == 'video_url' -%}\n" + " {%- set video_val = part['video_url'] if part['video_url'] is string else part['video_url']['url'] -%}\n" + " {{- '<|video|>' + video_val -}}\n" + " {%- elif part.get('type') == 'video' -%}\n" + " {%- set video_val = part['video'] if part['video'] is string else part['video']['url'] -%}\n" + " {{- '<|video|>' + video_val -}}\n" + " {%- endif -%}\n" " {%- endif -%}\n" " {%- endfor -%}\n" " {%- else -%}\n" @@ -2752,38 +2798,45 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- endif -%}\n" "\n" " {%- set captured_content -%}\n" - " {%- if message['content'] is string -%}\n" + " {%- if message.get('content') is string -%}\n" " {%- if role == 'model' -%}\n" " {{- strip_thinking(message['content']) -}}\n" " {%- else -%}\n" " {{- message['content'] | trim -}}\n" " {%- endif -%}\n" - " {%- elif message['content'] is sequence -%}\n" + " {%- elif message.get('content') is sequence -%}\n" " {%- for item in message['content'] -%}\n" - " {%- if item['type'] == 'text' -%}\n" + " {%- if item.get('type') == 'text' -%}\n" " {%- if role == 'model' -%}\n" " {{- strip_thinking(item['text']) -}}\n" " {%- else -%}\n" " {{- item['text'] | trim -}}\n" " {%- endif -%}\n" - " {%- elif item['type'] == 'image_url' -%}\n" - " {%- set url_val = item['image_url'] if item['image_url'] is string else item['image_url']['url'] -%}\n" - " {{- '<|image|>' + url_val -}}\n" - " {%- set ns.prev_message_type = 'image' -%}\n" - " {%- elif item['type'] in ['audio_url', 'input_audio'] -%}\n" - " {%- if item['type'] == 'audio_url' -%}\n" + " {%- elif item.get('type') in ['image', 'image_url'] -%}\n" + " {%- if item.get('type')== 'image_url' -%}\n" + " {%- set url_val = item['image_url'] if item['image_url'] is string else item['image_url']['url'] -%}\n" + " {{- '<|image|>' + url_val -}}\n" + " {%- elif item.get('type') == 'image' -%}\n" + " {%- set url_val = item['image'] if item['image'] is string else item['image']['url'] -%}\n" + " {{- '<|image|>' + url_val -}}\n" + " {%- endif -%}\n" + " {%- elif item.get('type') in ['audio_url', 'input_audio'] -%}\n" + " {%- if item.get('type') == 'audio_url' -%}\n" " {%- set audio_val = item['audio_url'] if item['audio_url'] is string else item['audio_url']['url'] -%}\n" " {{- '<|audio|>' + audio_val -}}\n" - " {%- elif item['type'] == 'input_audio' -%}\n" + " {%- elif item.get('type') == 'input_audio' -%}\n" " {%- set audio_val = item['input_audio'] if item['input_audio'] is string else ('data:audio/' + item['input_audio']['format'] + ';base64,' + item['input_audio']['data']) -%}\n" " {{- '<|audio|>' + audio_val -}}\n" " {%- endif -%}\n" - " {%- set ns.prev_message_type = 'audio' -%}\n" + " {%- elif item.get('type') in ['video', 'video_url'] -%}\n" + " {%- if item.get('type') == 'video_url' -%}\n" + " {%- set video_val = part['video_url'] if part['video_url'] is string else part['video_url']['url'] -%}\n" + " {{- '<|video|>' + video_val -}}\n" + " {%- elif item.get('type') == 'video' -%}\n" + " {%- set video_val = part['video'] if part['video'] is string else part['video']['url'] -%}\n" + " {{- '<|video|>' + video_val -}}\n" + " {%- endif -%}\n" " {%- endif -%}\n" - # " {%- elif item['type'] == 'video_url' -%}\n" - # " {%- set video_val = item['video_url'] if item['video_url'] is string else item['video_url']['url'] -%}\n" - # " {{- '<|video|>' + video_val -}}\n" - # " {%- set ns.prev_message_type = 'video' -%}\n" " {%- endfor -%}\n" " {%- endif -%}\n" " {%- endset -%}\n" @@ -2791,20 +2844,43 @@ class Gemma4ChatHandler(MTMDChatHandler): " {{- captured_content -}}\n" " {%- set has_content = captured_content | trim | length > 0 -%}\n" "\n" + " {#- Forward-scan: find next non-tool message role for continuation detection -#}\n" + " {%- set next_nt = namespace(role=None, found=false) -%}\n" + " {%- for j in range(loop.index0 + 1, loop_messages | length) -%}\n" + " {%- if not next_nt.found -%}\n" + " {%- if loop_messages[j]['role'] != 'tool' -%}\n" + " {%- set next_nt.role = loop_messages[j]['role'] -%}\n" + " {%- set next_nt.found = true -%}\n" + " {%- endif -%}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" + + " {%- set continues_into_next = (\n" + " role == 'model'\n" + " and next_nt.role == 'assistant'\n" + " and (not message.get('tool_calls') or ns_tr_out.flag)\n" + " ) -%}\n" + "\n" " {%- if ns.prev_message_type == 'tool_call' and not ns_tr_out.flag -%}\n" " {{- '<|tool_response>' -}}\n" - " {%- elif not (ns_tr_out.flag and not has_content) -%}\n" + " {%- elif continues_into_next -%}\n" + " {%- elif not (ns_tr_out.flag and not has_content and not next_nt.found) -%}\n" " {{- '\\n' -}}\n" " {%- endif -%}\n" + "\n" + " {#- Track previous non-tool role for next iteration (avoids O(n) backward scan) -#}\n" + " {%- set ns.prev_non_tool_role = message['role'] -%}\n" " {%- endif -%}\n" "{%- endfor -%}\n" "\n" "{%- if add_generation_prompt -%}\n" " {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%}\n" - " {{- '<|turn>model\\n' -}}\n" - " {%- if not enable_thinking | default(false) -%}\n" - " {{- '<|channel>thought\\n' -}}\n" + " {{- '<|turn>model\n' -}}\n" + " {%- if not enable_thinking -%}\n" + " {{- '<|channel>thought\n' -}}\n" " {%- endif -%}\n" + " {%- elif ns.prev_message_type == 'tool_response' and enable_thinking -%}\n" + " {{- '<|channel>thought\n' -}}\n" " {%- endif -%}\n" "{%- endif -%}\n" ) diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index 27a1a56d8..fcfaa86ee 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -178,12 +178,14 @@ class mtmd_pos_type(enum.IntEnum): # struct mtmd_input_text { # const char * text; +# size_t text_len; # bool add_special; # bool parse_special; # }; class mtmd_input_text(Structure): _fields_ = [ ("text", c_char_p), + ("text_len", c_size_t), ("add_special", c_bool), ("parse_special", c_bool), ] diff --git a/vendor/llama.cpp b/vendor/llama.cpp index e3546c794..4310aa4f8 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit e3546c7948e3af463d0b401e6421d5a4c2faf565 +Subproject commit 4310aa4f871c104698f6a6614a362bdec87c247a