From 6d075bc62c1a9057fc76350d4cac0f042b41a179 Mon Sep 17 00:00:00 2001 From: mudler <2420543+mudler@users.noreply.github.com> Date: Sun, 16 Aug 2026 21:36:25 +0000 Subject: [PATCH 1/3] :arrow_up: Update ggml-org/llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- backend/cpp/llama-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile index 5d238969aff4..5c30a5c2410b 100644 --- a/backend/cpp/llama-cpp/Makefile +++ b/backend/cpp/llama-cpp/Makefile @@ -1,5 +1,5 @@ -LLAMA_VERSION?=84e908c625fb60992b4cdef8180fb12fa9b4c4bf +LLAMA_VERSION?=4df29be4f4c3673f428170fda944a5b19f743bb8 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp CMAKE_ARGS?= From f684a352fc4f004d3c14c147cd3b7e3f993c5821 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Sun, 16 Aug 2026 22:12:22 +0000 Subject: [PATCH 2/3] fix(llama-cpp): refresh server patch contexts The new llama.cpp pin changed the slot reset and prompt batch code. GNU patch accepted stale hunks with fuzz, which left the L4T build with invalid source. Refresh both server patches against the pinned source so each hunk applies at its intended location. Assisted-by: Codex:gpt-5 --- .../0001-add-server-task-type-score.patch | 25 ++++++++----------- .../0002-add-server-task-type-tts.patch | 6 ++++- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch b/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch index 905248ecbe60..f056d47cec81 100644 --- a/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch +++ b/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch @@ -96,7 +96,7 @@ index 3b5f6a1..d0e18e6 100644 bool has_next_token = true; bool has_new_line = false; bool truncated = false; -@@ -351,6 +378,10 @@ struct server_slot { +@@ -341,6 +368,10 @@ struct server_slot { } generated_tokens.clear(); generated_token_probs.clear(); @@ -106,7 +106,7 @@ index 3b5f6a1..d0e18e6 100644 + score_divergence = -1; json_schema = json(); - // clear speculative decoding stats + task_prev = std::move(task); @@ -2271,6 +2302,229 @@ private: queue_results.send(std::move(res)); } @@ -391,7 +391,7 @@ index 3b5f6a1..d0e18e6 100644 // make a checkpoint of the parts of the memory that cannot be rolled back. // checkpoints are created only if: -@@ -3523,10 +3799,17 @@ private: +@@ -3444,9 +3720,16 @@ private: // embedding requires all tokens in the batch to be output; // MTP also wants logits at every prompt position so the // streaming hook can mirror t_h_nextn into ctx_dft. @@ -404,16 +404,12 @@ index 3b5f6a1..d0e18e6 100644 + slot.prompt.n_tokens() + 1 < slot.task->n_tokens(); add_ok &= batch.add(slot.id, cur_tok, - slot.prompt.tokens.pos_next(), -- slot.need_embd()); -+ slot.need_embd() || need_score_logit); + /* pos = */ slot.prompt.tokens.pos_next(), +- /* output = */ slot.need_embd(), ++ /* output = */ slot.need_embd() || need_score_logit, + /* is_prompt = */ true); slot.prompt.tokens.push_back(cur_tok); - - slot.n_prompt_tokens_processed++; -@@ -3541,6 +3824,32 @@ private: - } - } - +@@ -3454,2 +3737,28 @@ private: + // score tasks: break at the shared-prompt boundary so the checkpoint + // below lands exactly there — the other candidates of the same + // scoring call re-process only their own tokens. Also break at the @@ -440,9 +436,8 @@ index 3b5f6a1..d0e18e6 100644 + } + } + - // process the last few tokens of the prompt separately in order to allow for a checkpoint to be created. - // create checkpoints that many tokens before the end of the prompt: - // - 4 + n_ubatch + // break at the last user message, or at user messages at least min step past the last checkpoint + if (do_checkpoint && spans.is_user_start(slot.prompt.n_tokens())) { @@ -3573,6 +3882,15 @@ private: const bool is_user_start = spans.is_user_start(n_tokens_start); const bool is_last_user_message = n_tokens_start == last_user_pos; diff --git a/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch b/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch index 566a62cfec5b..ee1e26673394 100644 --- a/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch +++ b/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch @@ -468,8 +468,12 @@ index 9069463fe..b7fa1e534 100644 { // special case: if input is provided via CLI, tokenize it first // otherwise, no need to tokenize as it's already done inside the HTTP thread -@@ -3092,6 +3174,14 @@ private: +@@ -3082,10 +3164,18 @@ private: + SRV_ERR("pre_decode() failed: %s\n", e.what()); abort_all_slots("pre_decode() failed: " + std::string(e.what())); + + // the batch is half-built and not rendered, skip now to avoid UB + return; } + // note: TTS slots bypass the shared batch entirely From 5c85bfd6d8719e083a1e86be984597c3d6b69de1 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Sun, 16 Aug 2026 15:05:00 +0000 Subject: [PATCH 3/3] fix(llama-cpp): adapt metrics result fields The updated llama.cpp groups cumulative counters under server_metrics. Probe the result layout so the shared adapter also compiles against older forks. Assisted-by: Codex:gpt-5 --- backend/cpp/llama-cpp/grpc-server.cpp | 6 ++++++ backend/cpp/llama-cpp/prepare.sh | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/backend/cpp/llama-cpp/grpc-server.cpp b/backend/cpp/llama-cpp/grpc-server.cpp index 1091a786d4ca..039976ca965e 100644 --- a/backend/cpp/llama-cpp/grpc-server.cpp +++ b/backend/cpp/llama-cpp/grpc-server.cpp @@ -3597,9 +3597,15 @@ class BackendServiceImpl final : public backend::Backend::Service { // Populate the response with metrics response->set_slot_id(0); response->set_prompt_json_for_slot(""); +#if LOCALAI_HAS_SERVER_METRICS + response->set_tokens_per_second(res_metrics->metrics.prompt_bucket.n_per_second()); + response->set_tokens_generated(res_metrics->metrics.predict.count); + response->set_prompt_tokens_processed(res_metrics->metrics.prompt.count); +#else response->set_tokens_per_second(res_metrics->n_prompt_tokens_processed ? 1.e3 / res_metrics->t_prompt_processing * res_metrics->n_prompt_tokens_processed : 0.); response->set_tokens_generated(res_metrics->n_tokens_predicted_total); response->set_prompt_tokens_processed(res_metrics->n_prompt_tokens_processed_total); +#endif return grpc::Status::OK; diff --git a/backend/cpp/llama-cpp/prepare.sh b/backend/cpp/llama-cpp/prepare.sh index 4df276fb82bc..82d1f9caeb4e 100644 --- a/backend/cpp/llama-cpp/prepare.sh +++ b/backend/cpp/llama-cpp/prepare.sh @@ -53,10 +53,16 @@ else echo "==> llama.cpp predates the load-mode enum, using the legacy mmap/mlock/direct-io booleans" LEGACY_LOAD_MODE=1 fi +if grep -q "server_metrics metrics;" llama.cpp/tools/server/server-task.h; then + HAS_SERVER_METRICS=1 +else + HAS_SERVER_METRICS=0 +fi cat > llama.cpp/tools/grpc-server/llama_compat.h <