From 8b7ca9b0a1c5b43e73f6a5d13c82c1cc6819068a Mon Sep 17 00:00:00 2001 From: Tai An Date: Fri, 4 Sep 2026 03:48:15 -0700 Subject: [PATCH] fix(llama): grow the detokenize buffer when a token piece exceeds 32 bytes LlamaModel.detokenize() hands llama_token_to_piece a fixed 32-byte buffer and never checks for the negative return that means "did not fit". llama.cpp returns -(required size) in that case (src/llama-vocab.cpp:3596-3597) without writing anything, so `n` goes negative, `assert n <= size` still passes, and `bytes(buffer[:n])` slices from the end of a 32-byte buffer - yielding b'' for any piece of 64 bytes or more. Tokens whose piece is longer than 32 bytes are therefore dropped silently: a run of 64 spaces tokenizes fine and detokenizes to nothing, so detokenize(tokenize(x)) != x. Long whitespace runs are the common case, but any oversized piece is affected, and detokenize() is on the generation path. tokenize() one method up already handles exactly this convention for llama_tokenize (n_tokens = abs(n_tokens), then retry with a bigger buffer); llama.cpp's own token_to_piece_for_cache does the same. Do it here too, keeping the grown buffer for the remaining tokens. Fixes #2362 Signed-off-by: Tai An --- llama_cpp/_internals.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index b45d34b2c..b0ae01163 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -196,6 +196,19 @@ def detokenize(self, tokens: List[int], special: bool = False) -> bytes: n = llama_cpp.llama_token_to_piece( self.vocab, llama_cpp.llama_token(token), buffer, size, 0, special ) + if n < 0: + # The piece did not fit; llama_token_to_piece wrote nothing and + # returned -(required size). Grow the buffer and ask again. The + # larger buffer is kept for the remaining tokens. + size = -n + buffer = (ctypes.c_char * size)() + n = llama_cpp.llama_token_to_piece( + self.vocab, llama_cpp.llama_token(token), buffer, size, 0, special + ) + if n < 0: + raise RuntimeError( + f"Failed to detokenize: token={token} n={n} size={size}" + ) assert n <= size output += bytes(buffer[:n]) # NOTE: Llama1 models automatically added a space at the start of the prompt