diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile index 5d238969aff4..53d68a5a0ca7 100644 --- a/backend/cpp/llama-cpp/Makefile +++ b/backend/cpp/llama-cpp/Makefile @@ -1,5 +1,5 @@ -LLAMA_VERSION?=84e908c625fb60992b4cdef8180fb12fa9b4c4bf +LLAMA_VERSION?=0d9ceae1e38291035605613ab41a8f5e693d6fcd LLAMA_REPO?=https://github.com/ggerganov/llama.cpp CMAKE_ARGS?= diff --git a/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch b/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch index 905248ecbe60..962be69709c6 100644 --- a/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch +++ b/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch @@ -96,7 +96,7 @@ index 3b5f6a1..d0e18e6 100644 bool has_next_token = true; bool has_new_line = false; bool truncated = false; -@@ -351,6 +378,10 @@ struct server_slot { +@@ -351,4 +378,8 @@ struct server_slot { } generated_tokens.clear(); generated_token_probs.clear(); @@ -105,8 +105,6 @@ index 3b5f6a1..d0e18e6 100644 + score_suffix_pending = false; + score_divergence = -1; json_schema = json(); - - // clear speculative decoding stats @@ -2271,6 +2302,229 @@ private: queue_results.send(std::move(res)); } @@ -376,7 +374,7 @@ index 3b5f6a1..d0e18e6 100644 const auto n_cache_reuse = slot.task->params.n_cache_reuse; const bool can_cache_reuse = -@@ -3455,8 +3727,12 @@ private: +@@ -3366,8 +3638,12 @@ private: bool do_checkpoint = params_base.n_ctx_checkpoints > 0; @@ -391,58 +389,6 @@ index 3b5f6a1..d0e18e6 100644 // make a checkpoint of the parts of the memory that cannot be rolled back. // checkpoints are created only if: -@@ -3523,10 +3799,17 @@ private: - // embedding requires all tokens in the batch to be output; - // MTP also wants logits at every prompt position so the - // streaming hook can mirror t_h_nextn into ctx_dft. -+ // score tasks need outputs at the positions that predict -+ // each candidate token (the token at index i predicts the -+ // task token at index i+1). -+ const bool need_score_logit = -+ slot.task->type == SERVER_TASK_TYPE_SCORE && -+ slot.prompt.n_tokens() + 1 >= slot.task->n_score_prompt && -+ slot.prompt.n_tokens() + 1 < slot.task->n_tokens(); - add_ok &= batch.add(slot.id, - cur_tok, - slot.prompt.tokens.pos_next(), -- slot.need_embd()); -+ slot.need_embd() || need_score_logit); - slot.prompt.tokens.push_back(cur_tok); - - slot.n_prompt_tokens_processed++; -@@ -3541,6 +3824,32 @@ private: - } - } - -+ // score tasks: break at the shared-prompt boundary so the checkpoint -+ // below lands exactly there — the other candidates of the same -+ // scoring call re-process only their own tokens. Also break at the -+ // point where this task diverged from the previous cache: after a -+ // forced re-prefill a checkpoint there serves the next scoring call -+ // over the same stable prefix (e.g. a classifier's option list). -+ // The caller-declared stable-prefix boundary is the strongest of -+ // these: a checkpoint there is at or before every future task's -+ // divergence within the same option list, so it always survives -+ // and always restores. -+ if (do_checkpoint && slot.task->type == SERVER_TASK_TYPE_SCORE && -+ (slot.prompt.n_tokens() == slot.task->n_score_prompt - 1 || -+ (slot.task->n_stable_prompt > 0 && -+ slot.prompt.n_tokens() == slot.task->n_stable_prompt && -+ slot.prompt.n_tokens() < slot.task->n_score_prompt - 1) || -+ (slot.prompt.n_tokens() == slot.score_divergence && -+ slot.prompt.n_tokens() < slot.task->n_score_prompt - 1))) { -+ bool have_ckpt = false; -+ for (const auto & ckpt : slot.prompt.checkpoints) { -+ have_ckpt |= ckpt.n_tokens == slot.prompt.n_tokens(); -+ } -+ if (!have_ckpt) { -+ break; -+ } -+ } -+ - // process the last few tokens of the prompt separately in order to allow for a checkpoint to be created. - // create checkpoints that many tokens before the end of the prompt: - // - 4 + n_ubatch @@ -3573,6 +3882,15 @@ private: const bool is_user_start = spans.is_user_start(n_tokens_start); const bool is_last_user_message = n_tokens_start == last_user_pos; diff --git a/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch b/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch index 566a62cfec5b..d3eaef31ce0a 100644 --- a/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch +++ b/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch @@ -468,21 +468,6 @@ index 9069463fe..b7fa1e534 100644 { // special case: if input is provided via CLI, tokenize it first // otherwise, no need to tokenize as it's already done inside the HTTP thread -@@ -3092,6 +3174,14 @@ private: - abort_all_slots("pre_decode() failed: " + std::string(e.what())); - } - -+ // note: TTS slots bypass the shared batch entirely -+ try { -+ process_tts_slots(); -+ } catch (const std::exception & e) { -+ SRV_ERR("process_tts_slots() failed: %s\n", e.what()); -+ abort_all_slots("process_tts_slots() failed: " + std::string(e.what())); -+ } -+ - GGML_ASSERT(batch.slot_batched || batch.size() == 0); - - if (batch.slot_batched) { @@ -3162,10 +3252,77 @@ private: } } diff --git a/backend/cpp/llama-cpp/patches/0003-refresh-score-batch-context.patch b/backend/cpp/llama-cpp/patches/0003-refresh-score-batch-context.patch new file mode 100644 index 000000000000..a3dd87e95aeb --- /dev/null +++ b/backend/cpp/llama-cpp/patches/0003-refresh-score-batch-context.patch @@ -0,0 +1,18 @@ +diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp +--- a/tools/server/server-context.cpp ++++ b/tools/server/server-context.cpp +@@ -3086,6 +3086,14 @@ private: + return; + } + ++ // note: TTS slots bypass the shared batch entirely ++ try { ++ process_tts_slots(); ++ } catch (const std::exception & e) { ++ SRV_ERR("process_tts_slots() failed: %s\n", e.what()); ++ abort_all_slots("process_tts_slots() failed: " + std::string(e.what())); ++ } ++ + GGML_ASSERT(batch.slot_batched || batch.size() == 0); + + if (batch.slot_batched) { diff --git a/backend/cpp/llama-cpp/patches/0004-refresh-score-logit-context.patch b/backend/cpp/llama-cpp/patches/0004-refresh-score-logit-context.patch new file mode 100644 index 000000000000..507d334c528d --- /dev/null +++ b/backend/cpp/llama-cpp/patches/0004-refresh-score-logit-context.patch @@ -0,0 +1,52 @@ +diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp +--- a/tools/server/server-context.cpp ++++ b/tools/server/server-context.cpp +@@ -3877,13 +3877,46 @@ private: + // embedding requires all tokens in the batch to be output; + // MTP also wants logits at every prompt position so the + // streaming hook can mirror t_h_nextn into ctx_dft. ++ // score tasks need outputs at the positions that predict ++ // each candidate token (the token at index i predicts the ++ // task token at index i+1). ++ const bool need_score_logit = ++ slot.task->type == SERVER_TASK_TYPE_SCORE && ++ slot.prompt.n_tokens() + 1 >= slot.task->n_score_prompt && ++ slot.prompt.n_tokens() + 1 < slot.task->n_tokens(); + add_ok &= batch.add(slot.id, + cur_tok, + /* pos = */ slot.prompt.tokens.pos_next(), +- /* output = */ slot.need_embd(), ++ /* output = */ slot.need_embd() || need_score_logit, + /* is_prompt = */ true); + slot.prompt.tokens.push_back(cur_tok); ++ ++ // score tasks: break at the shared-prompt boundary so the checkpoint ++ // below lands exactly there — the other candidates of the same ++ // scoring call re-process only their own tokens. Also break at the ++ // point where this task diverged from the previous cache: after a ++ // forced re-prefill a checkpoint there serves the next scoring call ++ // over the same stable prefix (e.g. a classifier's option list). ++ // The caller-declared stable-prefix boundary is the strongest of ++ // these: a checkpoint there is at or before every future task's ++ // divergence within the same option list, so it always survives ++ // and always restores. ++ if (do_checkpoint && slot.task->type == SERVER_TASK_TYPE_SCORE && ++ (slot.prompt.n_tokens() == slot.task->n_score_prompt - 1 || ++ (slot.task->n_stable_prompt > 0 && ++ slot.prompt.n_tokens() == slot.task->n_stable_prompt && ++ slot.prompt.n_tokens() < slot.task->n_score_prompt - 1) || ++ (slot.prompt.n_tokens() == slot.score_divergence && ++ slot.prompt.n_tokens() < slot.task->n_score_prompt - 1))) { ++ bool have_ckpt = false; ++ for (const auto & ckpt : slot.prompt.checkpoints) { ++ have_ckpt |= ckpt.n_tokens == slot.prompt.n_tokens(); ++ } ++ if (!have_ckpt) { ++ break; ++ } ++ } +- ++ + // break at the last user message, or at user messages at least min step past the last checkpoint + if (do_checkpoint && spans.is_user_start(slot.prompt.n_tokens())) { + const auto pos = slot.prompt.n_tokens();