diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6cfa60ef8..dfdc0cbea 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -47,6 +47,36 @@ env: # directory for licensing). No download step is needed; CI just points # mvn test at the committed path. VISION_IMAGE_PATH: "llama/src/test/resources/images/test-image.jpg" +# Supersede an in-flight run when a PR branch is pushed again. +# +# Without this every push starts a full parallel pipeline and the older ones keep +# draining -- four were live at once during one session, which makes "what is CI +# saying right now" genuinely ambiguous and wastes a lot of runner time on results +# nobody will read. +# +# cancel-in-progress is deliberately scoped to pull_request ONLY. A push to main or +# to a v* tag is a release path: cancelling one midway could leave a partially +# published set of artifacts. +# +# cancel-in-progress: false is NOT sufficient on its own to protect a release run. +# GitHub cancels a *pending* run whenever a newer run joins the same group behind an +# in-progress one -- that rule is independent of cancel-in-progress. So with a plain +# `workflow-ref` group, a queued `publish_to_central` dispatch on main could be +# silently dropped by a later push to main, both sharing `Publish-refs/heads/main`. +# Giving every non-PR run its own group (via the unique run_id) means such a run is +# never queued behind a sibling and therefore can never be cancelled, while PR runs +# still share a group per ref and supersede each other as intended. +# +# One-time effect when this expression changes: GitHub reads `concurrency` from the +# workflow file at each run's own ref, so a run started before the change sits in the +# old group and a run started after it sits in the new one. They are different groups, +# so the new push does NOT supersede the in-flight old run -- exactly once, on the +# commit that lands this. It self-heals from the next push on. Expect the same overlap +# when porting this to a sibling repo; it is not a sign the expression is wrong. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == 'pull_request' && 'pr' || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + permissions: contents: read jobs: @@ -442,6 +472,52 @@ jobs: -Dnet.ladenthin.llama.langchain4j.rerank.model=models/${RERANKING_MODEL_NAME} -Dnet.ladenthin.llama.langchain4j.tool.model=models/${TOOL_MODEL_NAME} + # This job is model-backed and crosses JNI, so a forked test JVM here can abort exactly the + # way the six test-java-* jobs can -- but it had neither of their diagnostics. Same step and + # same path set as those, scoped to this module. See + # ../workspace/policies/ci-test-diagnostics.md section 3.1. + - name: Print crash logs (on failure) + if: failure() + shell: bash + run: | + shopt -s nullglob + found=0 + for f in llama-langchain4j/hs_err_pid*.log; do + found=1 + echo "===== $f (first 200 lines; full file in the uploaded artifact) =====" + sed -n '1,200p' "$f" + done + for f in llama-langchain4j/target/surefire-reports/*.dumpstream llama-langchain4j/target/surefire-reports/*.dump; do + found=1 + echo "===== $f =====" + cat "$f" + done + if [ "$found" = 0 ]; then + echo "No hs_err_pid*.log and no surefire dump/dumpstream was written." + echo + echo "For an ordinary test failure that is EXPECTED, not a finding: this step runs on" + echo "any job failure, and an assertion failure, a timeout or a compile error writes no" + echo "crash log. Read the surefire output above for the real cause." + echo + echo "It points at a JVM-level abort only if the log ALSO shows a fork ending abnormally" + echo "-- 'The forked VM terminated without properly saying goodbye', or an exit with no" + echo "test results. In that case the abort bypassed the JVM error handler (a native" + echo "exit()/terminate() rather than a raised signal), which is why no file was written." + fi + - if: failure() + uses: actions/upload-artifact@v7 + with: + name: error-log-langchain4j-integration + path: | + ${{ github.workspace }}/llama-langchain4j/hs_err_pid*.log + ${{ github.workspace }}/core.* + ${{ github.workspace }}/llama-langchain4j/*.hprof + ${{ github.workspace }}/llama-langchain4j/target/surefire-reports/*.dump + ${{ github.workspace }}/llama-langchain4j/target/surefire-reports/*.dumpstream + ${{ github.workspace }}/llama-langchain4j/target/surefire-reports/*.txt + ${{ github.workspace }}/llama-langchain4j/target/surefire-reports/TEST-*.xml + if-no-files-found: warn + # --------------------------------------------------------------------------- # Build the llama.cpp WebUI ONCE, from the same pinned tag CMakeLists.txt fetches, # and share it to every native build as the generated, platform-independent @@ -2271,6 +2347,40 @@ jobs: - name: Memory after tests if: always() run: free -h + # A forked test JVM that aborts leaves an hs_err_pid log and a surefire + # dumpstream -- both otherwise ONLY inside the artifact uploaded below, + # which is unreachable from anywhere that cannot fetch from Azure Blob + # (a phone, a restricted network, an agent sandbox). Echo them here so the + # aborting frame is readable from the run page itself. See + # ../workspace/policies/ci-test-diagnostics.md section 3.1. + - name: Print crash logs (on failure) + if: failure() + shell: bash + run: | + shopt -s nullglob + found=0 + for f in llama/hs_err_pid*.log; do + found=1 + echo "===== $f (first 200 lines; full file in the uploaded artifact) =====" + sed -n '1,200p' "$f" + done + for f in llama/target/surefire-reports/*.dumpstream llama/target/surefire-reports/*.dump; do + found=1 + echo "===== $f =====" + cat "$f" + done + if [ "$found" = 0 ]; then + echo "No hs_err_pid*.log and no surefire dump/dumpstream was written." + echo + echo "For an ordinary test failure that is EXPECTED, not a finding: this step runs on" + echo "any job failure, and an assertion failure, a timeout or a compile error writes no" + echo "crash log. Read the surefire output above for the real cause." + echo + echo "It points at a JVM-level abort only if the log ALSO shows a fork ending abnormally" + echo "-- 'The forked VM terminated without properly saying goodbye', or an exit with no" + echo "test results. In that case the abort bypassed the JVM error handler (a native" + echo "exit()/terminate() rather than a raised signal), which is why no file was written." + fi - if: failure() uses: actions/upload-artifact@v7 with: @@ -2370,6 +2480,40 @@ jobs: - name: Memory after tests if: always() run: vm_stat && sysctl hw.memsize hw.physmem + # A forked test JVM that aborts leaves an hs_err_pid log and a surefire + # dumpstream -- both otherwise ONLY inside the artifact uploaded below, + # which is unreachable from anywhere that cannot fetch from Azure Blob + # (a phone, a restricted network, an agent sandbox). Echo them here so the + # aborting frame is readable from the run page itself. See + # ../workspace/policies/ci-test-diagnostics.md section 3.1. + - name: Print crash logs (on failure) + if: failure() + shell: bash + run: | + shopt -s nullglob + found=0 + for f in llama/hs_err_pid*.log; do + found=1 + echo "===== $f (first 200 lines; full file in the uploaded artifact) =====" + sed -n '1,200p' "$f" + done + for f in llama/target/surefire-reports/*.dumpstream llama/target/surefire-reports/*.dump; do + found=1 + echo "===== $f =====" + cat "$f" + done + if [ "$found" = 0 ]; then + echo "No hs_err_pid*.log and no surefire dump/dumpstream was written." + echo + echo "For an ordinary test failure that is EXPECTED, not a finding: this step runs on" + echo "any job failure, and an assertion failure, a timeout or a compile error writes no" + echo "crash log. Read the surefire output above for the real cause." + echo + echo "It points at a JVM-level abort only if the log ALSO shows a fork ending abnormally" + echo "-- 'The forked VM terminated without properly saying goodbye', or an exit with no" + echo "test results. In that case the abort bypassed the JVM error handler (a native" + echo "exit()/terminate() rather than a raised signal), which is why no file was written." + fi - if: failure() uses: actions/upload-artifact@v7 with: @@ -2439,6 +2583,40 @@ jobs: - name: Memory after tests if: always() run: vm_stat && sysctl hw.memsize hw.physmem + # A forked test JVM that aborts leaves an hs_err_pid log and a surefire + # dumpstream -- both otherwise ONLY inside the artifact uploaded below, + # which is unreachable from anywhere that cannot fetch from Azure Blob + # (a phone, a restricted network, an agent sandbox). Echo them here so the + # aborting frame is readable from the run page itself. See + # ../workspace/policies/ci-test-diagnostics.md section 3.1. + - name: Print crash logs (on failure) + if: failure() + shell: bash + run: | + shopt -s nullglob + found=0 + for f in llama/hs_err_pid*.log; do + found=1 + echo "===== $f (first 200 lines; full file in the uploaded artifact) =====" + sed -n '1,200p' "$f" + done + for f in llama/target/surefire-reports/*.dumpstream llama/target/surefire-reports/*.dump; do + found=1 + echo "===== $f =====" + cat "$f" + done + if [ "$found" = 0 ]; then + echo "No hs_err_pid*.log and no surefire dump/dumpstream was written." + echo + echo "For an ordinary test failure that is EXPECTED, not a finding: this step runs on" + echo "any job failure, and an assertion failure, a timeout or a compile error writes no" + echo "crash log. Read the surefire output above for the real cause." + echo + echo "It points at a JVM-level abort only if the log ALSO shows a fork ending abnormally" + echo "-- 'The forked VM terminated without properly saying goodbye', or an exit with no" + echo "test results. In that case the abort bypassed the JVM error handler (a native" + echo "exit()/terminate() rather than a raised signal), which is why no file was written." + fi - if: failure() uses: actions/upload-artifact@v7 with: @@ -2508,6 +2686,40 @@ jobs: - name: Memory after tests if: always() run: vm_stat && sysctl hw.memsize hw.physmem + # A forked test JVM that aborts leaves an hs_err_pid log and a surefire + # dumpstream -- both otherwise ONLY inside the artifact uploaded below, + # which is unreachable from anywhere that cannot fetch from Azure Blob + # (a phone, a restricted network, an agent sandbox). Echo them here so the + # aborting frame is readable from the run page itself. See + # ../workspace/policies/ci-test-diagnostics.md section 3.1. + - name: Print crash logs (on failure) + if: failure() + shell: bash + run: | + shopt -s nullglob + found=0 + for f in llama/hs_err_pid*.log; do + found=1 + echo "===== $f (first 200 lines; full file in the uploaded artifact) =====" + sed -n '1,200p' "$f" + done + for f in llama/target/surefire-reports/*.dumpstream llama/target/surefire-reports/*.dump; do + found=1 + echo "===== $f =====" + cat "$f" + done + if [ "$found" = 0 ]; then + echo "No hs_err_pid*.log and no surefire dump/dumpstream was written." + echo + echo "For an ordinary test failure that is EXPECTED, not a finding: this step runs on" + echo "any job failure, and an assertion failure, a timeout or a compile error writes no" + echo "crash log. Read the surefire output above for the real cause." + echo + echo "It points at a JVM-level abort only if the log ALSO shows a fork ending abnormally" + echo "-- 'The forked VM terminated without properly saying goodbye', or an exit with no" + echo "test results. In that case the abort bypassed the JVM error handler (a native" + echo "exit()/terminate() rather than a raised signal), which is why no file was written." + fi - if: failure() uses: actions/upload-artifact@v7 with: @@ -2596,6 +2808,40 @@ jobs: if: always() run: Get-CimInstance Win32_OperatingSystem | Select-Object FreePhysicalMemory,TotalVisibleMemorySize | Format-List shell: pwsh + # A forked test JVM that aborts leaves an hs_err_pid log and a surefire + # dumpstream -- both otherwise ONLY inside the artifact uploaded below, + # which is unreachable from anywhere that cannot fetch from Azure Blob + # (a phone, a restricted network, an agent sandbox). Echo them here so the + # aborting frame is readable from the run page itself. See + # ../workspace/policies/ci-test-diagnostics.md section 3.1. + - name: Print crash logs (on failure) + if: failure() + shell: bash + run: | + shopt -s nullglob + found=0 + for f in llama/hs_err_pid*.log; do + found=1 + echo "===== $f (first 200 lines; full file in the uploaded artifact) =====" + sed -n '1,200p' "$f" + done + for f in llama/target/surefire-reports/*.dumpstream llama/target/surefire-reports/*.dump; do + found=1 + echo "===== $f =====" + cat "$f" + done + if [ "$found" = 0 ]; then + echo "No hs_err_pid*.log and no surefire dump/dumpstream was written." + echo + echo "For an ordinary test failure that is EXPECTED, not a finding: this step runs on" + echo "any job failure, and an assertion failure, a timeout or a compile error writes no" + echo "crash log. Read the surefire output above for the real cause." + echo + echo "It points at a JVM-level abort only if the log ALSO shows a fork ending abnormally" + echo "-- 'The forked VM terminated without properly saying goodbye', or an exit with no" + echo "test results. In that case the abort bypassed the JVM error handler (a native" + echo "exit()/terminate() rather than a raised signal), which is why no file was written." + fi - if: failure() uses: actions/upload-artifact@v7 with: @@ -2690,6 +2936,40 @@ jobs: if: always() run: Get-CimInstance Win32_OperatingSystem | Select-Object FreePhysicalMemory,TotalVisibleMemorySize | Format-List shell: pwsh + # A forked test JVM that aborts leaves an hs_err_pid log and a surefire + # dumpstream -- both otherwise ONLY inside the artifact uploaded below, + # which is unreachable from anywhere that cannot fetch from Azure Blob + # (a phone, a restricted network, an agent sandbox). Echo them here so the + # aborting frame is readable from the run page itself. See + # ../workspace/policies/ci-test-diagnostics.md section 3.1. + - name: Print crash logs (on failure) + if: failure() + shell: bash + run: | + shopt -s nullglob + found=0 + for f in llama/hs_err_pid*.log; do + found=1 + echo "===== $f (first 200 lines; full file in the uploaded artifact) =====" + sed -n '1,200p' "$f" + done + for f in llama/target/surefire-reports/*.dumpstream llama/target/surefire-reports/*.dump; do + found=1 + echo "===== $f =====" + cat "$f" + done + if [ "$found" = 0 ]; then + echo "No hs_err_pid*.log and no surefire dump/dumpstream was written." + echo + echo "For an ordinary test failure that is EXPECTED, not a finding: this step runs on" + echo "any job failure, and an assertion failure, a timeout or a compile error writes no" + echo "crash log. Read the surefire output above for the real cause." + echo + echo "It points at a JVM-level abort only if the log ALSO shows a fork ending abnormally" + echo "-- 'The forked VM terminated without properly saying goodbye', or an exit with no" + echo "test results. In that case the abort bypassed the JVM error handler (a native" + echo "exit()/terminate() rather than a raised signal), which is why no file was written." + fi - if: failure() uses: actions/upload-artifact@v7 with: diff --git a/.gitignore b/.gitignore index c2a7c61a1..7003d4ae2 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,11 @@ target build build-android +# Per-bump verification trees. The llama.cpp upgrade runbook requires a FRESH build +# directory for every version bump -- the patch applier pins its stamp to the checked-out +# llama.cpp commit and deliberately aborts rather than guess when an existing dir's stamp +# names a different one -- and the convention names them build-b. +build-b* cmake-build-* .DS_Store .directory diff --git a/CHANGELOG.md b/CHANGELOG.md index b16593565..46d7024cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,15 +9,310 @@ from version 5.0.0 onward. Pre-fork releases (`1.x`–`4.2.0`) were authored by ## [Unreleased] +> The entries below also cover the **b9917 → b10456** window (PRs #341–#394), which went unrecorded +> here while it happened; they were reconstructed from the git history and from +> [`docs/history/llama-cpp-breaking-changes.md`](docs/history/llama-cpp-breaking-changes.md), which +> has a row per upgrade range and stays authoritative for the per-range detail. + +### Fixed +- **JVM crash (SIGSEGV) on the first request after an idle-sleep window.** With + `--sleep-idle-seconds` set, upstream's `handle_sleeping_state(true)` calls `destroy()`, which frees + the model and context and nulls `ctx_tgt`/`model_tgt`. Two things in the JNI layer assumed they + outlived that: `server_context::get_meta()`, read on every request before the task is posted, and + the `jctx->vocab` pointer captured once after the initial load — dangling after the reload replaces + the model. The first was a null dereference that aborted the JVM at `llama_context::get_model()`; + the second a use-after-free on every tokenize/detokenize/rerank path. Both now go through a single + `wake_server()` choke point that waits out the sleep and re-reads the vocab, called from every entry + point that touches the model. The earlier `wake_and_post()` fix was necessary but not sufficient: + it woke at *post* time, and these reads happen before the post. + Fixing that exposed a third, latent defect in our own `patches/0002`: it guarded upstream's + progress-callback install on `== nullptr`, but `load_progress_text` is a **local** of + `load_model()` whose address upstream re-assigns on every call. On resume the guard saw our own + callback from the first load, skipped the re-assignment, and left `user_data` pointing into a dead + stack frame — a second SIGSEGV, this time inside `load_progress_callback()`. The guard now also + accepts its own callback, so our `user_data` is refreshed on every load while a caller-supplied + callback still survives. +- **`ModelParameters.setSleepIdleSeconds`** now rejects `0` and values below `-1`, which upstream's + own handler throws on. Emitting them aborted the whole argv parse and surfaced only as + `"Failed to parse model parameters"`, naming neither the flag nor the reason. Its Javadoc also said + the server "shuts down" after the idle window; it does not — it releases the model and reloads it on + the next request. + ### Added +- **`ModelParameters.setCpuMoeLayers(int)` / `setCpuFfnLayers(int)`** — keep the first N layers' + Mixture-of-Experts weights, or dense FFN weights, on the CPU (upstream `--n-cpu-moe` / `-ncmoe` and + `--n-cpu-ffn` / `-ncffn`). The companions to `setGpuLayers`: where that moves whole layers, these move + only the weight class that dominates a model's size, usually fitting a much larger model into the same + VRAM at a smaller speed cost. Only `--n-cpu-ffn` is new (llama.cpp b10645); `--n-cpu-moe` has existed + upstream since b6089 but had never been exposed here. +- **`ModelParameters.setVideoFps(float)` / `setVideoTimestampInterval(long)` / `setVideoFfmpegDir(String)`** + — the video-decoding knobs upstream added in llama.cpp b10647 (`--video-fps`, + `--video-timestamp-interval`, `--video-ffmpeg-dir`). They apply to any media attached to a request + once a projector is loaded: `server_context` copies them into the `mtmd_helper_init_opt` it passes + to `process_mtmd_prompt`, and video decoding is compiled into the shipped library (`MTMD_VIDEO` + defaults on). `setVideoFfmpegDir` is the significant one — upstream otherwise looks `ffmpeg` and + `ffprobe` up on `PATH`, which a JVM process often does not have them on. +- **`ModelParameters.setKvUnifiedPerSlot(int)`** — caps the context each parallel slot may use + (upstream `--kv-unified-per-slot`, new in llama.cpp b10662). The cap reaches this binding through + `server_context_meta::slot_n_ctx`: it becomes every `slot.n_ctx` and is the context budget passed to + `format_prompt_infill`. Upstream's second effect — sizing the + shared KV pool to `n_parallel * N` when no context size is given — lives in `llama_server()` and + therefore applies to `NativeServer` only, not to a model loaded from `ModelParameters`; the + Javadoc says so. +- **`ModelParameters.setTensorReadLazy(TensorReadLazyMode)`** and the new + **`net.ladenthin.llama.args.TensorReadLazyMode`** enum (`OFF` / `AUTO` / `ON`) — on-demand reading + of tensors the model architecture marks as lazy-loadable, such as per-layer embeddings (upstream + `--tensor-read-lazy`, new in llama.cpp b10653, mapping to `llama_lazy_mode`). Trades resident + memory for disk reads and requires mmap. It reaches the plain `LlamaModel` load path too, because + `common_model_params_to_llama` copies `lazy_mode` into `llama_model_params`. +- **`ServerMetrics.getWindowPromptProcessingMillis()` / `getWindowTokenGenerationMillis()` / + `getWindowTimings()`** — typed access to the current-window timing keys `t_prompt_processing` and + `t_tokens_generation`. Both were always emitted; only the cumulative `_total` variants had accessors. +- **`ModelMeta.supportsVideo()`**, and `getModelMeta()` now emits `modalities.video`. Upstream has + tracked `has_inp_video` on `server_context_meta` for releases and emits all three modalities from its + own `/props`; this binding emitted only vision and audio, so feature detection concluded no model + ever accepts video. - `QuantizationType.Q2_0` — maps the new upstream `LLAMA_FTYPE_MOSTLY_Q2_0` (llama.cpp b9916) for `LlamaQuantizer`. +- **Voice cloning and language selection for `TextToSpeech`**: `synthesize(String text, String speakerReferenceAudioPath, String language, int maxFrames, int topK, int seed)` — a speaker-reference clip makes the model imitate that voice. Part of the Qwen3-TTS rework (see Changed). +- **`ModelParameters.setMmprojDevice(String)`** — places the multimodal projector on a device of its own + (llama.cpp `--mmproj-device`, added upstream in b10541), independently of `setDevices(...)`. Exactly one + device may be named; the literal `"none"` keeps the projector on the CPU. `OpenAiCompatServer`'s CLI + accepts the same flag as `-mmdev`/`--mmproj-device`; `NativeServer` already forwarded it verbatim. +- **`RouterClient` API-key constructors** (`RouterClient(int, String)`, `RouterClient(String, int, String)`) — + send `Authorization: Bearer `, which a router started with `--api-key` requires for *every* call: + `/models/load` and `/models/unload` were always gated, and since b10519 (upstream #26347) the listing + endpoints are too. An empty key behaves like none, and `toString()` never prints it. +- **`ServerMetrics` cache and speculative-decoding counters** — `getCumulativeCachedPromptTokens()`, + `getDraftTokensTotal()`, `getDraftAcceptedTotal()`, `getDraftVerifyStepsTotal()`, + `getDraftAcceptedPerPosition()` and the derived `getDraftAcceptanceRate()`. Upstream exposes these only + as Prometheus text; they now arrive in the JSON payload. ### Changed +- **Deprecated `InferenceParameters.withTfsZ`, `withPenalizeNl` and both `withPenaltyPrompt` overloads.** + `tfs_z`, `penalize_nl` and `penalty_prompt` appear nowhere in upstream `common/` or `tools/server/` + at the pinned build, and the request schema discards unknown fields rather than rejecting them — so + these have been silently doing nothing. Kept compiling for now; they will be removed. +- `ModelParameters.setMmprojDevice` and `setMmprojOffload` now clear each other. Both write upstream's + single `mmproj_use_gpu` field, and the rendered argv comes out of a `HashMap`, so leaving both present + left the winner to hash order. Clearing in only one direction still lost the race whenever + `setMmprojOffload` was called second; the contract is now simply "the last of the two calls wins". +- **Deprecated `InferenceParameters.withUseChatTemplate` and `withChatTemplate`.** Both are load-time + settings upstream, not per-request ones: `common_params::use_jinja` is set only by `--jinja` / + `--no-jinja`, and the only `"chat_template"` string in upstream `common/` or `tools/server/` is the one + the server *emits* from `/props`. Neither key is ever read from a request body, so both calls were + silently doing nothing — including at three call sites in this library that used + `withUseChatTemplate(true)` to "enable jinja for tools", which those calls could not do. Use + `ModelParameters.enableJinja()` / `setChatTemplate(String)` instead. Tool calling was unaffected in + practice only because upstream defaults `use_jinja` to true. - `ch.qos.logback:logback-classic` bumped 1.6.2 → 1.6.3 (test/runtime binding only). - CI actions bumped to latest: `actions/setup-java` v5 → v6. - Upgraded llama.cpp from **b9894 to b9917** (all eight local patches re-verified across the range). +- **BREAKING — `TextToSpeech` was reworked onto Qwen3-TTS** (llama.cpp **b10270**, upstream #26254, which + upstream itself labels a breaking change). llama.cpp deleted the OuteTTS pipeline outright: + `tools/tts/tts.cpp` shrank from ~1450 to 205 lines and `mtmd_gen_audio_type` has only + `NONE`/`QWEN3TTS`, so there is no OuteTTS code path left anywhere upstream and no compatibility shim + was possible. The two-argument constructor keeps its **signature** but changes **meaning**: + `(ttcModelPath, vocoderModelPath)` → `(modelPath, mmprojPath)`, i.e. a Qwen3-TTS backbone plus the + mmproj that bundles speaker encoder, code predictor and code2wav decoder — an OuteTTS + WavTokenizer + pair no longer works and fails at load, not at compile time. `synthesize`'s `maxCodeTokens` parameter + became `maxFrames`, and the single-argument overload's default dropped 4096 → 512. +- **BREAKING — `-1` is no longer accepted for the repetition-penalty windows** (llama.cpp **b10273**). + `repeat_last_n` and `dry_penalty_last_n` used to take `-1` for "the whole context"; upstream removed + the sentinel, moving the request schema's hard limits to `[0, INT32_MAX]` and making + `common_params_parse` throw on a negative value. `ModelParameters.setRepeatLastN` / + `setDryPenaltyLastN` and `InferenceParameters.withRepeatLastN` / `withDryPenaltyLastN` had kept + advertising and accepting `-1`, so the value reached llama.cpp and failed there — at model load for + the launch flags, as a rejected request for the per-request withers. All four now reject a negative + value with a message naming the change; pass the context size explicitly for the old behaviour. + (Verified exhaustively: these are the **only** two request-field limits that moved in the whole + b9994 → b10618 range.) +- Upgraded llama.cpp from **b9917 to b10456** across PRs #341–#394. Local patches `0005` (b9981) and + `0004` (b9982) were dropped after upstream merged equivalent — and broader — fixes, and `0009` + (`subprocess.h` old-glibc build break) was dropped at b10280 once upstream vendored the same fix. +- `server-mcp.cpp` is compiled into `libjllama` (llama.cpp **b10154** added upstream MCP-server + support; `server.cpp` and `server-tools.cpp` reference `server_mcp`, so omitting it is latent on + Linux but a hard link error on macOS/ld64 and Windows/MSVC). The `subprocess.h` `addchdir_np` use is + guarded for old glibc in the same change. +- Android/Gradle toolchain: Gradle pins moved 8.14.3 → 9.6.1 and the dockcross cross-compile images + were bumped, alongside the AGP/Compose pin updates the Android builds needed. +- **Post-upgrade audit of the whole b10456→b10644 range** — three independent sweeps over the + upstream diff (completeness, adaptation correctness, test integrity) against the files the binding + actually consumes. No missed adaptation was found: the request-field set, their bounds and the + emitted response keys are identical at both ends of the range, and `libjllama` links with zero + undefined upstream symbols. The audit did surface documentation and coverage gaps, fixed here: + - The `-1` context-size sentinel was dropped upstream at **b10273** (#26524), not b10275 — corrected + in 4 Javadoc blocks, 4 exception messages and every doc that cited it. The `server-schema.h` + signature break is **the same** upstream commit, not an unrelated one: #26524 at b10273 dropped + `eval_llama_cmpl_schema`'s `n_ctx_slot` parameter in the same change that removed the sentinel + (`git diff b10273 b10275 -- tools/server/server-schema.h` is empty). + - `LlamaModel.saveSlot`/`restoreSlot` now document that the on-disk format is version-locked to the + linked llama.cpp build, and that a mismatch surfaces as upstream's misleading + `"No available space in KV cache or invalid slot save file"`. + - `getMetrics()` documents that the merged payload is not an atomic snapshot and that it defers + idle-sleep, which upstream's own `/metrics` stopped doing at b10519 (#27376, which introduced + `task_resets_idle_timer`). It cannot have been b10644: `git diff --name-only b10639 b10644 -- + tools/server/` is empty. + - **`--tools get_datetime` no longer starts.** Upstream deleted that built-in tool in this range and + an unknown name is fatal (`server_tools::setup` throws), so a `NativeServer` command line carrying + it now fails at startup. Same block: `server_tool::type()` reports `"server"` instead of + `"builtin"`, changing the `/tools` payload in full `NativeServer` mode. + - The four `t_*` keys in `getMetrics()` are now fractional rather than whole milliseconds, because + the merge divides upstream's microseconds. `ServerMetrics` reads them as doubles; a consumer + parsing the raw JSON with an integer parser sees a type change. + +- Upgraded llama.cpp from **b10649 to b10679**. No project-source change: all twelve + `tools/server/*.h` headers, `server-schema.cpp`, `server-task.cpp`, `server-common.cpp`, + `common/chat.h` and `tools/mtmd/mtmd-helper.h` are byte-identical across the range (compared by blob + SHA), so the request-field set, its bounds and the emitted response keys cannot have moved and the + three mechanical contract checks are moot. The whole in-scope delta is 8 files, 172 insertions and + 15 deletions — the rest of the 159-file range is `tools/ui` (rebuilt from `GIT_TAG` by CI), the ggml + backends, and `conversion/`, `gguf-py/`, `tests/`, `.github/`, `docs/`, `scripts/` and the standalone + `tools/` binaries, none of which this project compiles. + Two additive upstream features are new and both are now exposed (see Added): + `--kv-unified-per-slot` and `--tensor-read-lazy` / `llama_lazy_mode`. Three patch-target files were + touched (`common/arg.cpp`, `tools/server/server-context.cpp`, `tools/server/server.cpp`) and all + eight patches still apply with zero fuzz; patch `0007`'s invariant holds because the new + KV-pool-sizing block in `llama_server()` sits before the extracted route table, not inside it. + `llama_model_quantize_params` gained `max_buf_size`, which needs no adaptation because + `LlamaQuantizer` builds its params from `llama_model_quantize_default_params()`. Upstream's private + `get_slot_n_ctx()` → `n_ctx_slot()` rename is invisible here — the project reads the value through + the unchanged `server_context_meta::slot_n_ctx`. + Patch `0001` shrank from 37 to 36 files: upstream rewrote `tests/test-save-load-state.cpp`'s + `main()` to build its own filtered argv, so by the patch's own rule that call site now wants + `common_params_parse` and no longer the `_main()` flip. The patch itself is still required — + `common_params_parse` at b10679 still carries the count-guarded `GetCommandLineW` override and + `common_params_parse_main` does not exist upstream. +- Upgraded llama.cpp from **b10644 to b10649**. The first range in this bump to break the project's own + compile: upstream threaded a new `mtmd_helper_init_opt` (video-decode settings) through every helper + that can ingest media, changing the signature of `mtmd_helper_bitmap_init_from_file`, + `tokenize_input_prompts` and `format_prompt_rerank`. Four call sites were adapted — all of them pass + `mctx = nullptr` or handle audio, so each now passes upstream's own `mtmd_helper_init_opt_default()`. + The wire contract is unchanged: 68 request fields and 23 bounds identical across the range, and + the emitted response-key set identical for every server TU the project compiles (the exact key count + depends on which TUs are swept — the load-bearing half is that it does not move). Zero CLI flags were + removed or renamed, and all eight local patches apply unchanged even though six patch-target files + were touched. + Of the 6 new upstream flags, four are now exposed (see Added): `--n-cpu-ffn` and the three + `--video-*` knobs. `--n-cpu-moe` is exposed alongside them but is not new — it has existed upstream + since b6089 and had simply never been surfaced here. The two `--spec-synth-*` flags stay unexposed: + upstream marks them "benchmarking only" — they synthesise fake acceptance probabilities to measure + llama.cpp's own speculative harness. The `--video-*` trio was initially refused as inert without a + `ContentPart` video factory; a follow-up audit showed that was wrong on both counts (they reach the + task path this binding drives, and `MTMD_VIDEO` is compiled into the shipped library), so they are + exposed. The content part itself — upstream's `input_video`, which takes raw base64 rather than a + `data:` URI — remains in `TODO.md`. +- Upgraded llama.cpp from **b10639 to b10644**. No project-source change, and the only file on the + priority API-review list that the range touches is `include/llama.h`, whose entire diff is two + constants: `LLAMA_SESSION_VERSION` 9 → 10 and `LLAMA_STATE_SEQ_VERSION` 2 → 3. They follow from a new + `tok` field on `llama_kv_cell_ext` (n-gram input embeddings) that has to survive a state save/restore. + Everything else is the Snapdragon/Hexagon backend rework, a one-line fix in the nanbeige model graph, + and the WebUI. Nothing under `common/`, `tools/server/` or `tools/mtmd/` changed, so no request field, + no bound and no response key can have moved, and all eight local patches apply unchanged. + **One consumer-visible consequence:** the version bumps are a *state-file format* break. A slot state + saved by an earlier build — via the public `LlamaModel.saveSlot(int, String)`, or the server's + `/slots/{id}?action=save` — is rejected by `LlamaModel.restoreSlot` after this upgrade and has to be + regenerated. No Java or native signature changed. The rejection is graceful but its message is + upstream's misleading `"No available space in KV cache or invalid slot save file"`, which does not + name the version mismatch; `saveSlot`'s Javadoc now spells this out. Slot state files are a cache to + regenerate on upgrade, not durable storage. The in-memory `Session` snapshot/fork feature is + unaffected — it never writes a file. +- Upgraded llama.cpp from **b10631 to b10639**, in two reviewed steps. Neither range changes any + project source. b10631→b10636 is ggml-cuda quantised-matmul configs for Pascal, ggml-metal + SSM/Mamba kernels, an upstream `LLAMA_BUILD_UI` default flip that is inert here (this project + compiles its own `webui-generated/ui.cpp`), and the WebUI. b10636→b10639 is the RPC backend's + event/async APIs (#18626, protocol 5.1 → 6.0 — `GGML_RPC` is never enabled in this project, so + `ggml-rpc.cpp` is not compiled) plus Vulkan `cross_entropy_loss` kernels (#27216) and a warptile + clamp for warp sizes > 64 (#27726). Neither range touches `common/`, `include/llama.h`, + `tools/server/` or `tools/mtmd/`, so no request field, no bound and no response key can have + moved. All seven local patches apply unchanged. +- Upgraded llama.cpp from **b10618 to b10631**. No project-source change. The only **project-relevant** edits in the + range are a narrowing input validation in `oaicompat_chat_params_parse` (continuing a final + assistant message that carries `tool_calls` now throws), a Qwen3-Coder-only grammar refinement + in `common_chat_params_init_qwen3_coder`, a cosmetic `LLAMA_VERSION_MINOR` bump, and the WebUI. + `server-schema.cpp`, `server-task.cpp`, `server-context.cpp`, the `tools/server/*.h` headers, + `common/common.h`, `include/llama.h` and `mtmd-helper.h` are byte-identical across the range, so + neither the request-field set and its bounds nor the emitted response keys can have moved. All + seven local patches re-verified against a clean b10631 checkout; C++ suite 499/499. +- Upgraded llama.cpp from **b10456 to b10618**, in 25 reviewed steps. Patch `0007` refreshed (upstream + #26347 deleted comments inside its removal block, breaking `git apply` at every tag from b10519 on) and + a new patch `0010` carries a one-line upstream fix: `GET /models` emitted `vocab_type` as a JSON boolean + after the `common_json` switch (#27511), because an unscoped enum binds to the `bool` constructor. + The project's own C++ moved to `common_json` in the same range. +- **`apply-llama-patches.cmake` is now genuinely idempotent**, via a stamp file (llama.cpp commit plus each + patch's SHA-256) gated on git's clean/dirty state. Reconfiguring an existing build directory is a no-op + instead of aborting with a misleading "does not apply cleanly"; a real mismatch fails with an accurate + message. A source tree supplied via `-DFETCHCONTENT_SOURCE_DIR_LLAMA.CPP` that is not a git work tree + keeps the previous per-patch behaviour. +- `ServerMetrics.getStartTimestamp()` is documented correctly: `t_start` is a monotonic-clock **microsecond** + reading (`ggml_time_us()`), not milliseconds since the epoch. The value is unchanged. ### Fixed +- **With `setSleepIdleSeconds(> 0)`, the model became permanently unusable after the first idle + period.** Once llama.cpp's task queue enters its sleeping state, posting a task does not leave it: + `server_queue::post()` only notifies the condition variable, whose sleeping predicate tests + `req_stop_sleeping`, so the loop woke, re-tested, and went straight back to sleep with the task + still queued. Upstream performs the wake on the caller's behalf in `server_res_generator`'s + constructor (`wait_until_no_sleep()`), but only for readers built through `create_response()`; + this binding builds its readers with the CLI-facing `get_response_reader()`, which does not, and + nothing in the JNI layer called `wait_until_no_sleep()` at all. Every subsequent call then either + blocked until `close()` (completions, embeddings, rerank, infill) or threw `"No result"` + (`getMetrics`, LoRA and slot operations), for the lifetime of the process. All six post sites now + wake the queue first. Idle-sleep is off by default (`-1`), so a default configuration was never + affected. +- **A single malformed UTF-8 byte in a model's output turned a finished generation into an HTTP 500.** + The server parses *every* completion through `common_chat_parse()`; with no chat parser configured + (plain `/completion`) that is llama.cpp's content-only fallback, whose scan tolerates an incomplete + trailing UTF-8 sequence in lenient mode — which is the only mode the chat parser ever uses — but + rejected an *invalid* byte outright. The request then failed with `"The model produced output that + does not match the expected Content-only format"` even though generation had completed normally. + Carried as local patch `0011`, which makes the invalid-byte branch respect leniency the same way + (keeping the text up to the bad byte); strict-mode parsing is unchanged. Upstream-submittable. +- **`TextToSpeech` crashed the JVM on every platform when loading a model.** A hand-built + `common_params` never passes through `common_params_parse`, and `common/arg.cpp` is upstream's + only caller of `postprocess_cpu_params` — `common_init_from_params` does not call it. So + `cpuparams_batch.n_threads` kept its `-1` default, `common_threadpools::init` created a second + threadpool with -1 threads, and `ggml_threadpool_new` sized its worker array as + `sizeof(ggml_compute_state) * -1` — a huge `size_t`, so the allocation returned `NULL` and the + unchecked `memset` that follows it faulted at address 0. `tts_engine.cpp` and `train_engine.cpp` now mirror `arg.cpp`'s two + calls; the `LlamaModel` paths were never affected because their params are parsed. Guarded by + five model-free C++ tests over the extracted `build_tts_params`. +- **`LlamaQuantizer` never worked in any published jar — every call threw `UnsatisfiedLinkError`.** + The `extern "C"` declarations that give the JNI entry points C linkage come from the + javac-generated `jllama.h`, which covers **only** `LlamaModel`; a JNI function for any other class + has to declare its own (as `train_engine.cpp` and `native_server.cpp` do). + `Java_net_ladenthin_llama_LlamaQuantizer_quantizeNative` did not, so it was exported under its + C++-mangled name and the JVM could never resolve it — on every platform, not just the two Windows + jobs that reported it. The only coverage was `QuantizerIntegrationTest`, which gates on a GGUF and + so skipped in CI for as long as the model paths resolved to the wrong directory. Fixed, and guarded + model-free by `NativeLibraryLoadSmokeTest.quantizerNativeEntryPointResolves` so a future entry point + that forgets `extern "C"` fails a test that runs wherever the library exists. +- **The macOS arm64 native library shipped corrupt in 5.0.6 and in several 5.0.7 snapshots.** All three + macOS arm64 build jobs uploaded their dylib under a `*-libraries` artifact name, and the packaging + job collects those with one globbed download — so three builds landed on the same + `Mac/aarch64/libjllama.dylib` and the survivor could be a byte-level hybrid of two of them rather + than either input. Its ad-hoc signature then no longer matched its own `__TEXT` pages (66/4078 and + 1141/4097 code pages failed their stored hashes) and macOS **SIGKILLed every process that loaded + it**. Fixed by naming the test-only variants outside the glob and selecting the shipped variant by + an explicit download step (thanks to **@linking12**, #388), plus two guards so it cannot recur: + `merge-native-artifacts.sh` fails the build when any relative path is claimed by more than one + artifact — checked *before* the merge, since a collision leaves exactly one file behind and is + invisible afterwards — and the new `smoke-fatjar-macos` job runs `codesign --verify --strict` and a + real JVM load of the dylib extracted from the **packaged** fat jar (#390). +- **`LlamaModel.getMetrics()` returned the wrong shape.** Upstream reduced the payload to a bare slot array + at b10408 (#26920) and split the task in two at b10519 (#27376), so the counter getters on + `value.ServerMetrics`, `LlamaModelTest#testGetMetrics` and `OpenAiCompatServer`'s metrics routes had all + been reading keys that no longer existed. The JNI layer now posts both tasks and merges them, restoring the + documented object rather than following upstream's transport split. +- **`GET /slots` answered HTTP 200 with a zero-length body** whenever the metrics payload carried no `slots` + key (`MissingNode.toString()` is `""`). It now always answers with a JSON array. +- **Model-gated Java tests silently self-skipped in CI.** Surefire's working directory is the module basedir + while the shared GGUF cache is restored to the reactor root, so every `models/…` path resolved to nothing, + every such class aborted in its `@BeforeAll`, and the job still reported success — which is why the stale + `getMetrics()` assertions above never failed. Test paths now resolve against either layout. + `llama-langchain4j` had the identical defect. +- **`RouterClient.awaitModelLoaded` misdiagnosed hidden router models.** A cache model deduplicated by a + preset with `dedup-cache-models` (b10505, #27346) is omitted from `GET /models` although it still loads and + serves by name; the error now names that cause instead of sending callers to re-check `--models-dir`. - **CVE-2026-49844** (GHSA-qv9r-c865-cp47, moderate): `org.apache.logging.log4j:log4j-api` 2.25.3 arrives as a **test-scope** transitive of `io.github.hakky54:logcaptor` 2.12.6, and Dependabot could not update it on its own. Pinned `log4j-api` **and** `log4j-to-slf4j` to diff --git a/CLAUDE.md b/CLAUDE.md index 3abaabe1d..e20539f39 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,7 +6,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co Java bindings for [llama.cpp](https://github.com/ggerganov/llama.cpp) via JNI, providing a high-level API for LLM inference in Java. The Java layer communicates with a native C++ library through JNI. -Current llama.cpp pinned version: **b10456** +Current llama.cpp pinned version: **b10679** ## Upgrading CUDA Version @@ -490,7 +490,7 @@ needs no extra step here, `build-webui` re-reads the tag and rebuilds the matchi ships no UI): ```bash # needs node/npm + network; embed.cpp is plain C++17 (no npm) -git clone --depth 1 --branch b10456 https://github.com/ggml-org/llama.cpp /tmp/lc +git clone --depth 1 --branch b10679 https://github.com/ggml-org/llama.cpp /tmp/lc ( cd /tmp/lc/tools/ui && npm ci && npm run build \ && ( cd dist && find . -type f -not -path './_gzip/*' \ | while read -r f; do mkdir -p "_gzip/$(dirname "$f")"; gzip -9 -c "$f" > "_gzip/$f"; done ) \ @@ -530,7 +530,7 @@ cache lives in **Depot Cache** over sccache's **WebDAV** backend: - `SCCACHE_WEBDAV_TOKEN: ${{ secrets.DEPOT_TOKEN }}` — a Depot **organization** token, stored as the repo secret **`DEPOT_TOKEN`**. -Because `sccache` is **content-addressed** and llama.cpp is pinned (`GIT_TAG b10456`), the +Because `sccache` is **content-addressed** and llama.cpp is pinned (`GIT_TAG b10679`), the ~280 upstream object files are byte-identical every run, so a warm cache recompiles only the *changed* files. Depot's cache is **shared across all branches** (unlike GitHub's per-branch `actions/cache`), so every branch builds incrementally; a `b` version bump @@ -641,9 +641,20 @@ The fetched llama.cpp source is patched before it compiles, via a generic mechan ordered. Each must be a `git apply`-compatible unified diff with paths relative to the llama.cpp source root (`a/common/arg.cpp` / `b/common/arg.cpp`, i.e. `-p1`). - **`llama/cmake/apply-llama-patches.cmake`** — the applier. Cross-platform (`cmake -P`, so identical on - Linux/macOS/Windows), **idempotent** (`git apply --reverse --check` skips already-applied patches - so a reconfigure never double-applies) and **fail-loud** (a patch that no longer applies aborts - the configure — a stale patch can't be silently dropped from a release build). + Linux/macOS/Windows), **idempotent** and **fail-loud** (a patch that no longer applies aborts the + configure — a stale patch can't be silently dropped from a release build). Idempotency comes from + a **stamp file** (`/.jllama-patches-applied`, recording the checked-out llama.cpp + commit plus each patch's SHA-256) combined with git's clean/dirty state, not from per-patch + probing: a **clean** source tree means nothing is applied yet (fresh fetch, or a re-checkout after + a version bump) so everything is applied forward and the stamp written; a **dirty** tree is + already patched, and the reconfigure is a no-op when the stamp matches this exact commit + patch + set, or aborts with a "configure into a fresh build directory" message when it does not. + A per-patch `git apply --reverse --check` cannot do this — `--check` never mutates the tree, so an + earlier patch whose region a later one rewrote (`0001` vs `0006`/`0007` in + `tools/server/server.cpp`) always reverse-checks as "not applied", and the forward re-apply then + aborted every reconfigure of an existing build dir with a misleading "does not apply cleanly". + A source tree supplied via `-DFETCHCONTENT_SOURCE_DIR_LLAMA.CPP=` that is not a git work + tree has neither oracle and falls back to the old per-patch path (same caveat as before). - **`llama/CMakeLists.txt`** — wired as the llama.cpp `FetchContent_Declare(... PATCH_COMMAND ...)`, so it runs for **every** C++ build (all CI jobs *and* local `cmake -B build`) from one place — no per-build-step plumbing. @@ -656,11 +667,13 @@ Current patches: | Patch | Fixes | |-------|-------| -| `0001-win32-arg-parse-embed-guard.patch` | Windows JNI regression from llama.cpp **#24779** (introduced b9739): on Windows `common_params_parse` re-derived argv from the **process** command line (`GetCommandLineW`) and adopted it, so an embedded/JNI caller (`java.exe`) lost its `--model …` args → "Failed to parse model parameters". b9789 narrowed the unconditional override to a **count-guard** (`if (static_cast(utf8.buf.size()) == argc) { argv = utf8.ptrs.data(); }`), but that is exactly the variant the project already found breaks its Windows server-integration tests (when the embedded argv length coincides with `java.exe`'s). The patch carries the **complete upstream change** (so it can be submitted to llama.cpp verbatim and then dropped here): **(1)** `common_params_parse` parses **exactly the argv it is given** (no `GetCommandLineW` magic) and a new `common_params_parse_main()` wrapper holds the UTF-8 recovery for the standalone tools' `main()` (`common/arg.{cpp,h}`); **(2)** the **~34 standalone `main()` call sites** (every `common_params_parse(argc, argv, …)` across `tools/*`, `examples/*` and the `tests/*` programs) flip to `common_params_parse_main()`; **(3)** a `tests/test-arg-parser.cpp` regression case pins that `common_params_parse` honors a caller-supplied argv. The embedded caller (`jllama.cpp`) keeps calling `common_params_parse` and is never overridden. **Our subproject build compiles only the `arg.{cpp,h}` core** — `LLAMA_BUILD_TOOLS`/`LLAMA_BUILD_TESTS` are OFF for a FetchContent subproject — so the flips + test are applied-but-not-compiled here; they were validated via a one-off `-DLLAMA_BUILD_TOOLS=ON -DLLAMA_BUILD_TESTS=ON` build (the new test compiles and its asserts pass; `test-arg-parser`'s only red there is the live `ggml.ai` download check, which is sandbox-network, not the patch). Because it spans **37 files** it must be refreshed on every llama.cpp bump (the applier fails loud). The upstream-facing write-up, including a standalone reproducer that makes llama.cpp's own `test-arg-parser` fail on unmodified `master`, lives in [docs/upstream-investigation-win32-argv-substitution.md](docs/upstream-investigation-win32-argv-substitution.md). **Reported upstream as [ggml-org/llama.cpp#26416](https://github.com/ggml-org/llama.cpp/issues/26416)** (2026-08-01, label `bug-unconfirmed`, first bad commit `508a475`); the issue asks which of the two directions the maintainers prefer before a PR is opened, so this patch stays downstream until they answer. | -| `0002-server-preserve-caller-load-progress-callback.patch` | Load-progress-callback regression introduced in llama.cpp **b9789**: `server_context::load_model` (`tools/server/server-context.cpp`) now **unconditionally** installs the server's own load-progress reporter on `params_base.load_progress_callback` immediately before `common_init_from_params`, clobbering any callback the embedding caller already set. libjllama's `LoadProgressCallback` feature wires `common_params.load_progress_callback` to a JNI trampoline *before* calling `load_model`, so the bump silently killed it — `LoadProgressCallbackTest` saw zero progress updates and the abort-on-`false` path never threw. The patch guards the assignment with `if (params_base.load_progress_callback == nullptr)`, so the server installs its own reporter **only when the caller hasn't** — a caller-supplied callback survives and fires during load. Standalone `llama-server` (no caller callback, so the field is null) is unaffected. Same JNI-vs-standalone divergence class as `0001`. | +| `0001-win32-arg-parse-embed-guard.patch` | Windows JNI regression from llama.cpp **#24779** (introduced b9739): on Windows `common_params_parse` re-derived argv from the **process** command line (`GetCommandLineW`) and adopted it, so an embedded/JNI caller (`java.exe`) lost its `--model …` args → "Failed to parse model parameters". b9789 narrowed the unconditional override to a **count-guard** (`if (static_cast(utf8.buf.size()) == argc) { argv = utf8.ptrs.data(); }`), but that is exactly the variant the project already found breaks its Windows server-integration tests (when the embedded argv length coincides with `java.exe`'s). The patch carries the **complete upstream change** (so it can be submitted to llama.cpp verbatim and then dropped here): **(1)** `common_params_parse` parses **exactly the argv it is given** (no `GetCommandLineW` magic) and a new `common_params_parse_main()` wrapper holds the UTF-8 recovery for the standalone tools' `main()` (`common/arg.{cpp,h}`); **(2)** the **~34 standalone `main()` call sites** (every `common_params_parse(argc, argv, …)` across `tools/*`, `examples/*` and the `tests/*` programs) flip to `common_params_parse_main()`; **(3)** a `tests/test-arg-parser.cpp` regression case pins that `common_params_parse` honors a caller-supplied argv. The embedded caller (`jllama.cpp`) keeps calling `common_params_parse` and is never overridden. **Our subproject build compiles only the `arg.{cpp,h}` core** — `LLAMA_BUILD_TOOLS`/`LLAMA_BUILD_TESTS` are OFF for a FetchContent subproject — so the flips + test are applied-but-not-compiled here; they were validated via a one-off `-DLLAMA_BUILD_TOOLS=ON -DLLAMA_BUILD_TESTS=ON` build (the new test compiles and its asserts pass; `test-arg-parser`'s only red there is the live `ggml.ai` download check, which is sandbox-network, not the patch). Because it spans **36 files** it must be refreshed on every llama.cpp bump (the applier fails loud). **Refreshed at the b10679 bump:** upstream rewrote `tests/test-save-load-state.cpp`'s `main()` to take a `--models DIR` option, which it strips itself into a `filtered_argv` before calling `common_params_parse(fargc, filtered_argv.data(), …)`. That call site therefore stopped qualifying for the `_main()` flip — by this patch's own rule a caller that builds its own argv must use `common_params_parse` directly, so its argv is kept — and the hunk was **dropped** rather than refreshed (37 → 36 files). Caveat for whoever submits this upstream: that `main()` now filters a possibly-mojibake Windows argv *before* any UTF-8 recovery, so the fully correct upstream form there is recover-then-filter, not a one-line flip. It is out of scope for the downstream carry because `LLAMA_BUILD_TESTS` is OFF here, so the file is never compiled. **Still required at b10679, verified rather than assumed:** `common_params_parse` in pristine `b10679:common/arg.cpp` still carries the `#ifdef _WIN32` count-guarded `argv = utf8.ptrs.data()` override, and `common_params_parse_main` appears nowhere in `b10679:common/arg.h` — upstream has not adopted the fix. The upstream-facing write-up, including a standalone reproducer that makes llama.cpp's own `test-arg-parser` fail on unmodified `master`, lives in [docs/upstream-investigation-win32-argv-substitution.md](docs/upstream-investigation-win32-argv-substitution.md). **Reported upstream as [ggml-org/llama.cpp#26416](https://github.com/ggml-org/llama.cpp/issues/26416)** (2026-08-01, label `bug-unconfirmed`, first bad commit `508a475`); the issue asks which of the two directions the maintainers prefer before a PR is opened, so this patch stays downstream until they answer. | +| `0002-server-preserve-caller-load-progress-callback.patch` | Load-progress-callback regression introduced in llama.cpp **b9789**: `server_context::load_model` (`tools/server/server-context.cpp`) now **unconditionally** installs the server's own load-progress reporter on `params_base.load_progress_callback` immediately before `common_init_from_params`, clobbering any callback the embedding caller already set. libjllama's `LoadProgressCallback` feature wires `common_params.load_progress_callback` to a JNI trampoline *before* calling `load_model`, so the bump silently killed it — `LoadProgressCallbackTest` saw zero progress updates and the abort-on-`false` path never threw. The patch guards the assignment with `if (params_base.load_progress_callback == nullptr)`, so the server installs its own reporter **only when the caller hasn't** — a caller-supplied callback survives and fires during load. Standalone `llama-server` (no caller callback, so the field is null) is unaffected. Same JNI-vs-standalone divergence class as `0001`. **The guard is `== nullptr || == load_progress_callback`, and the second disjunct must never be dropped:** `load_progress_text` is a **local** of `load_model()`, and upstream re-assigns both fields on every call so the `user_data` always points at the current frame. `load_model()` runs a **second** time when resuming from the sleeping state (`--sleep-idle-seconds`), and by then `params_base` holds *our own* callback from the first load — a bare nullptr check skips the re-assignment and leaves `user_data` pointing into a **dead stack frame**, which segfaults inside `load_progress_callback()` on the first request after an idle window. That was a latent defect in this patch from the day it was written; only a second `load_model()` can reach it, and nothing exercised sleep until `IdleSleepWakeIntegrationTest` was added. | | `0003-pr22393-server-add-slot-prompt-similarity-getter-setter.patch` | **Upstream-PR carry** of [ggml-org/llama.cpp#22393](https://github.com/ggml-org/llama.cpp/pull/22393) ("server : add slot_prompt_similarity getter/setter"). Purely additive: adds `server_context::get_slot_prompt_similarity()` / `set_slot_prompt_similarity(float)` (`tools/server/server-context.{cpp,h}`) so an embedding/JNI caller can query and tune the slot-selection threshold at runtime without reloading the model. Verbatim copy of the PR, which **upstream closed without merging** (rejected as exposing unsafe internal state — see the patch header). Carried permanently; it will not be droppable via a version bump. | -| `0007-server-attach-http-frontend.patch` | **Adds `llama_server_attach(argc, argv, server_context&)`** so the `NativeServer` *attach mode* can serve an **already-loaded `LlamaModel`** over the upstream HTTP frontend — no second model load, no `start_loop()`; the LlamaModel's worker keeps driving the shared `server_context` and the HTTP routes post tasks to its queue (the queue is the synchronization point). Mechanically: (1) extracts the **pure core route table** (`health` … `slots`) out of `llama_server()` into `static void llama_server_register_common_routes(ctx_http, routes)` (shared, so the two entry points cannot drift on the core endpoint set). **Scope note (narrowed at the b10154 bump):** the helper deliberately carries **only** the stable, state-independent route table — **not** the resumable-streaming routes (their handlers differ between router / non-router), the GCP-compat shim, or the experimental **CORS-proxy / MCP-server / built-in-tools** wiring. b10154 (upstream MCP-server support) moved the streaming routes into the middle of that block and coupled tools/CORS to a per-call `server_mcp mcp_mgr` lifecycle, so the earlier contiguous "route-table + CORS-proxy + tools" extraction is no longer possible; `llama_server()` keeps all of that inline, **byte-identical to upstream b10154** (only the route-table block is factored out). (2) adds `llama_server_attach`, which parses only the HTTP-side argv via `common_params_parse`, starts the stream-session GC + `server_http_context`, registers the common route table, the **non-router** resumable-streaming handlers (upstream b10154 paths `/v1/stream` GET/DEL + `/v1/streams/lookup` POST), the GCP-compat shim, and **403 "disabled" stubs for `/cors-proxy` + `/tools`** (attach mode does not wire the experimental CORS-proxy / MCP / built-in-tools host — those belong to a full `llama-server`, not an embedded model), marks ready immediately (model already loaded), and blocks on the HTTP thread until `llama_server_request_shutdown()` — never calling `common_init()`, backend init, `ctx_server.terminate()` or `llama_backend_free()` (the embedding caller owns those). Applies after `0001`+`0006` (same file); closes the "NativeServer — reuse an already-loaded LlamaModel" TODO. Upstream-submittable ("server: let embedding callers attach the HTTP frontend to an existing server_context"). | +| `0007-server-attach-http-frontend.patch` | **Adds `llama_server_attach(argc, argv, server_context&)`** so the `NativeServer` *attach mode* can serve an **already-loaded `LlamaModel`** over the upstream HTTP frontend — no second model load, no `start_loop()`; the LlamaModel's worker keeps driving the shared `server_context` and the HTTP routes post tasks to its queue (the queue is the synchronization point). Mechanically: (1) extracts the **pure core route table** (`health` … `slots`) out of `llama_server()` into `static void llama_server_register_common_routes(ctx_http, routes)` (shared, so the two entry points cannot drift on the core endpoint set). **Scope note (narrowed at the b10154 bump):** the helper deliberately carries **only** the stable, state-independent route table — **not** the resumable-streaming routes (their handlers differ between router / non-router), the GCP-compat shim, or the experimental **CORS-proxy / MCP-server / built-in-tools** wiring. b10154 (upstream MCP-server support) moved the streaming routes into the middle of that block and coupled tools/CORS to a per-call `server_mcp mcp_mgr` lifecycle, so the earlier contiguous "route-table + CORS-proxy + tools" extraction is no longer possible; `llama_server()` keeps all of that inline, **byte-identical to upstream b10154** (only the route-table block is factored out). (2) adds `llama_server_attach`, which parses only the HTTP-side argv via `common_params_parse`, starts the stream-session GC + `server_http_context`, registers the common route table, the **non-router** resumable-streaming handlers (upstream b10154 paths `/v1/stream` GET/DEL + `/v1/streams/lookup` POST), the GCP-compat shim, and **403 "disabled" stubs for `/cors-proxy` + `/tools`** (attach mode does not wire the experimental CORS-proxy / MCP / built-in-tools host — those belong to a full `llama-server`, not an embedded model), marks ready immediately (model already loaded), and blocks on the HTTP thread until `llama_server_request_shutdown()` — never calling `common_init()`, backend init, `ctx_server.terminate()` or `llama_backend_free()` (the embedding caller owns those). Applies after `0001`+`0006` (same file); closes the "NativeServer — reuse an already-loaded LlamaModel" TODO. Upstream-submittable ("server: let embedding callers attach the HTTP frontend to an existing server_context"). **Refreshed at the b10519 bump:** upstream #26347 dropped the API key from the `/models` + `/v1/models` public-endpoint set and deleted the two trailing `// public endpoint (no API key check)` comments on those route registrations. Those two lines sit inside this patch's route-table removal block, so `git apply` failed ("patch does not apply", `server.cpp:258`) at **every** tag from b10519 on; the fix was to drop the now-wrong comment from all four affected lines (2 on the `-` side, 2 in the extracted helper on the `+` side), keeping the helper byte-identical to the block it replaces. **This is the invariant to re-check on every bump:** the `+` side of `llama_server_register_common_routes()` must stay a verbatim copy of the route table it factors out of `llama_server()`. | | `0008-server-models-worker-cmd-override.patch` | **Makes router mode usable in-JVM.** The router (`server-models.cpp`) spawns each model worker by re-executing its own binary (`get_server_exec_path()` = `/proc/self/exe` & friends) — inside a JVM that binary is `java`, not a llama-server, so embedded router workers could never start. The patch adds env `LLAMA_SERVER_WORKER_CMD` (whitespace-split; read in `server_model_meta::update_args`) which replaces only the leading binary-path token of the rendered worker args, letting an embedding host relaunch workers through its own bootstrap — e.g. `java -cp app.jar net.ladenthin.llama.server.NativeServer` (each worker is then a fresh JVM running the classic single-model `NativeServer`). Exposed in Java as `NativeServer.setWorkerCommand(String...)` (JNI `setenv`); exercised by `RouterModeIntegrationTest` (Linux CI). Upstream-submittable (also useful for containerized/wrapped deployments). | +| `0010-server-cast-vocab-type-for-common-json.patch` | **Upstream regression from the b10585 `common_json` switch (#27511), one line.** `get_res_model_info()` (`tools/server/server-context.cpp`) builds the `GET /models` + `GET /v1/models` payload and emits `{"vocab_type", meta.model_vocab_type}` — an **unscoped enum**. `common_json_value`'s integral constructor template is `std::is_integral`-gated, which *excludes* enums, so the value binds to `common_json_value(bool)` and serialises as `true`/`false` instead of the numeric vocab type. It was correct while the alias was `nlohmann::ordered_json` (nlohmann serialises an enum as an integer), so upstream regressed it silently when they flipped the alias. The project ships this: `server-context.cpp` is compiled into `libjllama` and both routes are served by `NativeServer` — the default fat-jar `Main-Class` — in full **and** attach mode (`patches/0007`'s common route table registers them). The patch casts the value to `int` at the emit site, mirroring what `jllama.cpp` does for its own two `"vocab_type"` sites. Upstream-submittable; **not yet filed upstream**. Applies after `0002`/`0003` (same file) — numbered `0010` because `0009` is burned: it names the subprocess.h patch dropped at the b10280 bump (see the note below this table), and reusing the number would make that note read as if it were about this patch. **On every bump, check whether upstream cast the value themselves; if they did, DROP this patch rather than refreshing it** — the fail-loud applier only detects "does not apply", never "upstream already fixed this", and no test can catch a redundant carry here because `get_res_model_info` is `static` inside `server-context.cpp` and unreachable from `jllama_test`. See the `CommonJsonEnumTrap` tests in `test_json_helpers.cpp` for the mechanism the cast defends against. | +| `0011-peg-parser-lenient-invalid-utf8.patch` | **A model that emits one malformed UTF-8 byte turns a finished generation into an HTTP 500.** The server parses *every* completion through `common_chat_parse()`; with no chat parser configured (plain `/completion`) that is the content-only fallback `content(rest()) + end()`, whose scan is `common_peg_until_parser` (`common/peg-parser.cpp`). `common_chat_peg_parse()` always parses in **lenient** mode, and that scan tolerates an `INCOMPLETE` trailing UTF-8 sequence by keeping the text before it — but the `INVALID` branch right below it returns `FAIL` unconditionally, ignoring leniency. One stray byte anywhere in the generated text therefore throws `"The model produced output that does not match the expected Content-only format"` and the request 500s even though generation completed normally (`stop processing: n_tokens = 4, truncated = 0`). The patch makes the `INVALID` branch respect `ctx.is_lenient()` exactly like the `INCOMPLETE` branch — keep the text up to the malformed byte — and adds an upstream `tests/peg-parser/test-unicode.cpp` case pinning both the lenient and the still-failing strict behavior. **Strict mode is unchanged**, which is what keeps upstream's own tests green: `tests/peg-parser/test-unicode.cpp` *does* assert `FAIL` on invalid UTF-8 through the *until* parser (a `malformed UTF-8` block with three `p.until("")` cases), but each builds a bare `common_peg_parse_context` with no `COMMON_PEG_PARSE_FLAG_LENIENT`, so the lenient-only change cannot reach them. This patch adds its case inside that same block. Found by `NativeServerAttachIntegrationTest.completion_overHttp_served`, which 500s on all six Java CI platforms. Upstream-submittable; **not yet filed upstream**. Touches only `common/peg-parser.cpp` + that test, which no other patch touches, so it is independent of `0001`/`0006`/`0007`. Runnable guard: the `ContentOnlyParseUtf8` tests in `src/test/cpp/test_utils.cpp` — unlike the upstream test they are compiled and run in CI on every platform, so a bump that drops this patch reds `C++ Tests` instead of one Java job. | | `0006-server-embed-native-server-jni.patch` | **Makes `server.cpp`'s `llama_server` embeddable in the JVM** so the `NativeServer` JNI bridge can run the full upstream HTTP server (WebUI included) inside `libjllama` — see "Two server modes" below. b9870 already exposes `int llama_server(int, char**)` (non-static; no `main` in the file), so the patch only adds embedded-mode support: (1) a `g_llama_server_embedded` flag + `llama_server_set_embedded()` / `llama_server_request_shutdown()` (declared in the committed `src/main/cpp/native_server_bridge.h`); (2) skips installing the process-wide SIGINT/SIGTERM handlers when embedded (they would hijack the JVM's); (3) in embedded mode parses the **forwarded** argv via `common_params_parse` instead of `common_params_parse_main` (whose `GetCommandLineW` recovery would pick up `java.exe`'s command line — the same Windows class of bug `0001` fixes). `llama_server_request_shutdown()` mirrors the SIGTERM path (invokes the installed `shutdown_handler` → `ctx_server.terminate()` unblocks `start_loop()`), giving JNI an out-of-band stop since `ctx_server` is loop-local. Applies **after `0001`** (which flips this call site to `common_params_parse_main`), so its context is the post-`0001` tree; regenerate against `0001`+source on a bump. Only touches `tools/server/server.cpp`. | **`0009` was dropped at the b10280 bump.** Upstream merged @@ -741,8 +754,9 @@ edit/verify/commit loop below. Use it for any non-trivial bump; the steps here a To change the llama.cpp version, update the following **four** files (and re-verify `patches/`): -1. **llama/CMakeLists.txt** — the `GIT_TAG` line for llama.cpp: `GIT_TAG b8831` (and the - cosmetic `-DLLAMA_TAG=b8831` a few lines below, passed to the TTS generator — keep them equal) +1. **llama/CMakeLists.txt** — the `GIT_TAG` line for llama.cpp: `GIT_TAG b8831`. (There is + no second tag to keep in sync any more: the cosmetic `-DLLAMA_TAG=` that fed the old build-time + TTS extraction went away with the Qwen3-TTS rework — see "Qwen3-TTS via `mtmd_helper::gen_audio`".) 2. **README.md** — the badge and link line with the version number 3. **CLAUDE.md** — the "Current llama.cpp pinned version" line 4. **llama/src/main/java/net/ladenthin/llama/value/LlamaCppVersion.java** — the @@ -754,7 +768,7 @@ To change the llama.cpp version, update the following **four** files (and re-ver Example: To upgrade from b8808 to b8831: ```bash -# Edit llama/CMakeLists.txt: change GIT_TAG b8808 to b8831 (and the -DLLAMA_TAG line) +# Edit llama/CMakeLists.txt: change GIT_TAG b8808 to b8831 # Edit README.md: change b8808 to b8831 (in both badge and link) # Edit CLAUDE.md: change b8808 to b8831 # Edit LlamaCppVersion.java: change LLAMA_CPP_VERSION "b8808" to "b8831" @@ -815,9 +829,33 @@ jllama.cpp / server.hpp / utils.hpp **Priority-ordered review list for upgrade diffs** (highest break risk first) -The top 8 rows cover all known API-level breaking changes from b5022 → b8831. -For future upgrades, provide diffs for at least these 8 files rather than the full patch. -Also review the project `CMakeLists.txt` for build-system-level breaks (e.g. renamed link targets, new required headers) — those are not visible in header file diffs alone. +The rows below cover the known **compile/link-level** breaks from b5022 to the current pin; start any +upgrade review with them rather than the full patch. Also review the project `CMakeLists.txt` for +build-system-level breaks (e.g. renamed link targets, new required headers) — those are not visible in +header file diffs alone. + +**Two failure classes this list does NOT catch, both of which have bitten the project:** + +1. **A same-repo header the project includes directly but that is not reachable through the + dependency graph above.** `tools/server/server-schema.h` broke a full build at b10273 while sitting + outside this table; it is in it now, and the `tools/server/*.h` rule in its row generalises that. +2. **A silent *contract* change behind an unchanged signature.** b10408 reduced + `server_task_result_metrics::to_json()` to a bare slot array and b10519 split the task; no + signature moved, every chunk compiled and linked clean, and `LlamaModel.getMetrics()` quietly + returned the wrong shape for hundreds of builds. The same class hit `repeat_last_n` / + `dry_penalty_last_n` at b10273, where only a *value range* moved. Two cheap mechanical checks catch + these where a header diff cannot — run them on any bump that touches `tools/server/`: + +```bash +# request-field set + their bounds, old tag vs new +git show :tools/server/server-schema.cpp | grep -oE 'field_[a-z_]+\("[a-z_0-9]+"' | sort -u +git show :tools/server/server-schema.cpp | tr '\n' ' ' \ + | grep -oE 'field_[a-z]+[^(]*\("[a-z_0-9]+"[^;]*?set_(hard_)?limits\([^)]*\)' +# response keys emitted by the result types +# response keys -- both emit forms; a single-form grep misses res["k"] = ... entirely +git show :tools/server/server-task.cpp | { grep -oE '\{ *"[A-Za-z_0-9.]+" *,'; \ + git show :tools/server/server-task.cpp | grep -oE '\[ *"[A-Za-z_0-9.]+" *\] *='; } | sort -u +``` | File | What to watch for | |------|-------------------| @@ -1120,30 +1158,57 @@ If the local check passes (`BUILD SUCCESS`), the `mvn package` job in - **`server` package — OpenAI-compatible HTTP endpoint (a single implementation).** - `server.OpenAiCompatServer` — built only on the JDK's `com.sun.net.httpserver` (no new dependency), embeddable and runnable via `java -cp net.ladenthin.llama.server.OpenAiCompatServer …` (the fat-jar default `Main-Class` is now `NativeServer` — see "Two server modes"). Serves `POST /v1/chat/completions` (streaming via SSE + non-streaming), `POST /v1/completions`, `POST /v1/embeddings`, `POST /v1/rerank`, `POST /infill`, `GET /v1/models` and `GET /health` (every route is also reachable without the `/v1` prefix), so editors that speak the OpenAI protocol (e.g. VS Code Copilot "Custom Endpoint", Cline, Roo Code, Continue) can drive a local model. Streaming chat uses the native OAI chunk path (`LlamaModel.streamChatCompletion` → `requestChatCompletionStream` / `receiveChatCompletionChunk` + the C++ `wrap_stream_chunk` helper), preserving `delta.tool_calls`; completions/embeddings/infill forward verbatim to the matching `LlamaModel.handle*`; rerank reshapes `handleRerank` into the OAI `results`/`data` shape. The chat mapper forwards `stream_options` and `response_format` and defaults `cache_prompt=true`; a CORS `Filter` answers `OPTIONS` preflights; `OpenAiSseFormatter.ensureUsageCachedTokens` guarantees `usage.prompt_tokens_details.cached_tokens` on the streamed usage chunk (Copilot crash fix, microsoft/vscode #273482). **Agentic tool-calling is the primary target**; a C++ guard (`test_server.cpp`) pins `tool_calls.function.arguments` as a JSON string (llama.cpp #20198). - **Alternative protocol surfaces** (pure translation over the OpenAI chat core — no second inference path; each reconstructs streamed tool calls via `ToolCallDeltaAccumulator`): **Ollama-native** (`GET /api/version`, `/api/tags`, `POST /api/show`, `/api/chat` with NDJSON streaming, `/api/generate` prompt-completion/FIM — `OllamaApiSupport`; `/api/show` advertises tools/insert/vision capabilities + context length for Copilot's Ollama provider), **Anthropic Messages** (`POST /v1/messages`, SSE event stream — `AnthropicApiSupport` + `AnthropicStreamTranslator`), and **OpenAI Responses** (`POST /v1/responses`, SSE event stream — `ResponsesApiSupport` + `ResponsesStreamTranslator`). The llama.cpp-native `GET /props` (context length + `modalities`) is served via `OpenAiSseFormatter.propsJson` for autocomplete clients that size their context from it. - - Supporting classes: `OpenAiServerConfig` (builder; optional bearer auth; binds `127.0.0.1`; `corsAllowOrigin`; `supportsVision`), `OpenAiServerCli` (testable CLI arg parser → `ModelParameters` + `OpenAiServerConfig`; flags incl. `--mmproj`/`--embedding`/`--reranking`), `OpenAiRequestMapper` (OAI chat request → `InferenceParameters`), `OpenAiSseFormatter` (SSE/models/error JSON + usage normalization), `OaiRerankSupport` (pure rerank request/response shaping), and the model-free test seam `OpenAiBackend`/`ChunkSink` + `LlamaModelBackend`. The streaming envelope is parsed by `json.ChatStreamChunkParser`. + - Supporting classes: `OpenAiServerConfig` (builder; optional bearer auth; binds `127.0.0.1`; `corsAllowOrigin`; `supportsVision`), `OpenAiServerCli` (testable CLI arg parser → `ModelParameters` + `OpenAiServerConfig`; flags incl. `--mmproj`/`-mmdev,--mmproj-device`/`--embedding`/`--reranking`), `OpenAiRequestMapper` (OAI chat request → `InferenceParameters`), `OpenAiSseFormatter` (SSE/models/error JSON + usage normalization), `OaiRerankSupport` (pure rerank request/response shaping), and the model-free test seam `OpenAiBackend`/`ChunkSink` + `LlamaModelBackend`. The streaming envelope is parsed by `json.ChatStreamChunkParser`. - The `server` package is a dedicated top layer in the ArchUnit `layeredArchitecture` rule (the only layer allowed to access the root `Api`); `noInternalJdkImports` carries an explicit exception for the supported `com.sun.net.httpserver` (the exported `jdk.httpserver` module, which `module-info.java` `requires`). See README "OpenAI-compatible HTTP server". **Native layer** (`src/main/cpp/`): -- `jllama.cpp` — JNI implementation bridging Java calls to llama.cpp. ~1,650 lines; 34 native methods (30 `LlamaModel` + 3 `TextToSpeech` + 1 `LlamaQuantizer`). +- `jllama.cpp` — JNI implementation bridging Java calls to llama.cpp. ~1,900 lines; 34 native methods (30 `LlamaModel` + 3 `TextToSpeech` + 1 `LlamaQuantizer`) plus `JNI_OnLoad`/`JNI_OnUnload`. - `utils.hpp` — Helper utilities (format helpers, argv stripping, token-piece serialisation). - `json_helpers.hpp` — Pure JSON transformation helpers (no JNI, no llama state). Independently unit-testable. - `jni_helpers.hpp` — JNI bridge helpers (handle management + server orchestration). Includes `json_helpers.hpp`. -- Uses `nlohmann/json` for JSON deserialization of parameters. -- The upstream server library (`server-context.cpp`, `server-queue.cpp`, `server-task.cpp`, `server-schema.cpp`, `server-models.cpp`, and — since b9829 — `server-stream.cpp`) is compiled directly into `jllama` via CMake — there is no hand-ported `server.hpp` fork. **`server-stream.cpp` is mandatory, not optional:** it defines the resumable-streaming SSE replay buffer (`g_stream_sessions`, `stream_session_attach_pipe`, `stream_aware_should_stop`, `stream_conv_id_from_headers`, the `stream_pipe_*` types) that `server-context.cpp` / `server-http.cpp` / `server-models.cpp` now `#include "server-stream.h"` and call, so omitting it fails the link with undefined references. It is platform-neutral (threads + std mutex/condvar, no `subprocess.h`/`posix_spawn_*`), so it builds on Android too and sits outside the `server-models.cpp` Android guard. `jllama` wires its own JNI routes and never calls `g_stream_sessions.start_gc()` (only the excluded standalone `server.cpp` `main()` does), so its GC thread stays dormant. **Phase 2:** the upstream HTTP transport (`tools/server/server-http.cpp`) and its `cpp-httplib` backend (`vendor/cpp-httplib/httplib.cpp`) are now compiled into `jllama` too, so the OpenAI-compatible server can be driven natively from JNI *inside* `libjllama` — no separate `llama-server` executable (a JNI shared library loads anywhere a JVM runs, which a standalone binary does not). `server-http.cpp` does `#include "ui.h"` (the WebUI asset table that `tools/ui`/`llama-ui` normally generates); since the Svelte WebUI is not shipped, `src/main/cpp/webui_stub/ui.h` supplies the upstream **empty-asset** interface and leaves `LLAMA_UI_HAS_ASSETS` undefined (all static-asset-serving blocks compile out). `` already resolves via `llama-common`'s `vendor/` include dir (same nlohmann/json 3.12.0 as the FetchContent copy). No SSL: `CPPHTTPLIB_OPENSSL_SUPPORT` is left undefined (plain-HTTP; bind localhost / front with a TLS proxy). **`server.cpp`, `server-tools.cpp` and `server-mcp.cpp` are now compiled in too** (on non-Android — they pull in `subprocess.h`/`posix_spawn_*`, so they share `server-models.cpp`'s Android guard): b9870 exposes `server.cpp`'s entry as `int llama_server(int, char**)` (no `main` in the file), and `patches/0006` makes it embeddable (no process signal handlers, forwarded-argv parse, out-of-band shutdown). **`server-mcp.cpp` is new in b10154** (upstream MCP-server support): both `server.cpp` (`llama_server`'s `mcp_mgr` lifecycle) and `server-tools.cpp` (`tools.setup(..., mcp_mgr)` / `server_mcp::call_tool`) reference `server_mcp`, so it **must** be in the `target_sources` list or the link fails with undefined `server_mcp::{start,shutdown,call_tool,list_tools,~server_mcp}` — **latent on Linux** (a shared object tolerates undefined symbols) but a **hard link error on macOS/ld64 and Windows/MSVC**. It is compiled only into `jllama`, not `jllama_test` (which links neither `server.cpp` nor `server-tools.cpp`). The `NativeServer` JNI bridge (`src/main/cpp/native_server.cpp`) calls `llama_server` on a worker thread, so the **full** upstream server — WebUI and all — runs inside `libjllama`. See "Two server modes" below. +- **The `json` alias is upstream's `common_json`, not `nlohmann::ordered_json` (since llama.cpp b10585, upstream #27511).** `tools/server/server-common.h` now says `using json = common_json;` — a deliberately small pimpl wrapper (`common/json.{h,cpp}`, compiled into `llama-common`) around the vendored nlohmann copy. Two traps this cost the project once, both of which **compile silently**: + 1. **An unscoped enum becomes a JSON boolean.** `common_json_value`'s integral constructor template is `std::is_integral`-gated, which excludes enums, so an enum binds to `common_json_value(bool)`. Always `static_cast(...)` an enum before putting it in JSON — `jllama.cpp`'s two `"vocab_type"` sites do, and `patches/0010` does the same for upstream's own `/models` handler. Guards: `test_json_helpers.cpp`'s `CommonJsonEnumTrap` pair pins the mechanism and **does** run in CI; `LlamaModelTest`'s `isIntegralNumber()` assertion pins the real wire value and now runs in CI too (the model-gated suite no longer self-skips — see "CI model policy" below). + 2. **`common_json` converts to `std::string` implicitly**, so it binds happily to a `const nlohmann::json &` parameter (via nlohmann's string-constructible converting constructor) and then throws `json::type_error 302` at runtime. Never declare a project helper as taking `nlohmann::json` when callers pass the `json` alias — `require_json_field_impl` is a template for exactly this reason. + Other differences to know: no `get_ref`/`array_t`/`type_name()`; a braced list in *value* position does not build an array (write `json::array({...})`); `at(key)` needs an explicit `.get()`; errors are `common_json_error`; and `get()` is limited to the types explicitly specialised in `common/json.cpp`. `log_helpers.hpp` and `train_engine.cpp` keep their own `nlohmann::json` alias — they never touch the server's `json`. +- Uses `nlohmann/json` for JSON deserialization of parameters in the two files named above; everything on the server path uses `common_json`. +- The upstream server library (`server-context.cpp`, `server-queue.cpp`, `server-task.cpp`, `server-schema.cpp`, `server-models.cpp`, and — since b9829 — `server-stream.cpp`) is compiled directly into `jllama` via CMake — there is no hand-ported `server.hpp` fork. **`server-stream.cpp` is mandatory, not optional:** it defines the resumable-streaming SSE replay buffer (`g_stream_sessions`, `stream_session_attach_pipe`, `stream_aware_should_stop`, `stream_conv_id_from_headers`, the `stream_pipe_*` types) that `server-context.cpp` / `server-http.cpp` / `server-models.cpp` now `#include "server-stream.h"` and call, so omitting it fails the link with undefined references. It is platform-neutral (threads + std mutex/condvar, no `subprocess.h`/`posix_spawn_*`), so it builds on Android too and sits outside the `server-models.cpp` Android guard. `jllama` wires its own JNI routes and never calls `g_stream_sessions.start_gc()` (only the excluded standalone `server.cpp` `main()` does), so its GC thread stays dormant. **Phase 2:** the upstream HTTP transport (`tools/server/server-http.cpp`) and its `cpp-httplib` backend (`vendor/cpp-httplib/httplib.cpp`) are now compiled into `jllama` too, so the OpenAI-compatible server can be driven natively from JNI *inside* `libjllama` — no separate `llama-server` executable (a JNI shared library loads anywhere a JVM runs, which a standalone binary does not). `server-http.cpp` does `#include "ui.h"` (the WebUI asset table that `tools/ui`/`llama-ui` normally generates); since the Svelte WebUI is not shipped, `src/main/cpp/webui_stub/ui.h` supplies the upstream **empty-asset** interface and leaves `LLAMA_UI_HAS_ASSETS` undefined (all static-asset-serving blocks compile out). `` already resolves through `llama-common` — since upstream #27304 (b10488) not from a `PUBLIC ../vendor` include dir of its own but transitively, via the `vendor::nlohmann` / `vendor::sheredom` INTERFACE targets it links PUBLIC, each of which exports the `vendor/` root (same nlohmann/json 3.12.0 as the FetchContent copy). No SSL: `CPPHTTPLIB_OPENSSL_SUPPORT` is left undefined (plain-HTTP; bind localhost / front with a TLS proxy). **`server.cpp`, `server-tools.cpp` and `server-mcp.cpp` are now compiled in too** (on non-Android — they pull in `subprocess.h`/`posix_spawn_*`, so they share `server-models.cpp`'s Android guard): b9870 exposes `server.cpp`'s entry as `int llama_server(int, char**)` (no `main` in the file), and `patches/0006` makes it embeddable (no process signal handlers, forwarded-argv parse, out-of-band shutdown). **`server-mcp.cpp` is new in b10154** (upstream MCP-server support): both `server.cpp` (`llama_server`'s `mcp_mgr` lifecycle) and `server-tools.cpp` (`tools.setup(..., mcp_mgr)` / `server_mcp::call_tool`) reference `server_mcp`, so it **must** be in the `target_sources` list or the link fails with undefined `server_mcp::{start,shutdown,call_tool,list_tools,~server_mcp}` — **latent on Linux** (a shared object tolerates undefined symbols) but a **hard link error on macOS/ld64 and Windows/MSVC**. It is compiled only into `jllama`, not `jllama_test` (which links neither `server.cpp` nor `server-tools.cpp`). The `NativeServer` JNI bridge (`src/main/cpp/native_server.cpp`) calls `llama_server` on a worker thread, so the **full** upstream server — WebUI and all — runs inside `libjllama`. See "Two server modes" below. ### Two server modes (`OpenAiCompatServer` vs `NativeServer`) The library exposes **two** ways to serve a model over HTTP, on two different transports. The fat jar's `Main-Class` is `server.ServerLauncher`, a tiny dispatcher: it runs `OpenAiCompatServer` when `--jllama-openai-compat` is present (that marker is stripped, the rest forwarded) and the default `NativeServer` otherwise. Both mains are also runnable directly by class name via `java -cp`. The two modes: 1. **`server.OpenAiCompatServer` (Java transport).** OpenAI/Ollama/Anthropic-compatible JSON API on the JDK's `com.sun.net.httpserver`, driving the compiled server *core* over JNI. Embeddable, no extra dependency, and it can share/reuse a `LlamaModel`. It serves **no** static assets — its `/` route is a 404, so **no WebUI**. It has its own `main` (run via `java -cp net.ladenthin.llama.server.OpenAiCompatServer …`); its CLI (`OpenAiServerCli`) maps a curated flag subset (`-m/-c/-b/-ub/-ngl/-t/-tb/-ctk/-ctv/--jinja/--chat-template-kwargs/--host/--port/--parallel/--mmproj/--api-key/--embedding/--reranking`). -2. **`server.NativeServer` (native transport) — the default fat-jar server (when `--jllama-openai-compat` is absent).** Runs the **full upstream `llama_server`** (via `patches/0006` + `native_server.cpp`) inside `libjllama`, forwarding the raw llama-server argv verbatim — so **every** llama-server flag works and the **embedded WebUI is served** (when the assets are compiled in; CI's released jars have them, local `cmake` builds use the empty-asset stub). With the classic constructor it is an **independent lifecycle** (loads its own model from the argv, like `llama-server.exe`; owns the process's llama backend + stderr logging while running); the **attach constructor** (`NativeServer(LlamaModel, String...)`, via `patches/0007`'s `llama_server_attach`) instead serves an **already-loaded `LlamaModel`** — one copy of the weights, the model's worker keeps driving inference, the HTTP routes post to its queue; caller closes the server before the model. **Router mode** (start without a model argument: `--models-dir`, `GET/POST /models`, per-request model selection) works in-JVM after `NativeServer.setWorkerCommand(...)` redirects the worker spawn to a fresh JVM (`patches/0008` — upstream re-execs its own binary, which in a JVM is `java`); the typed `server.RouterClient` (+ `value.RouterModel`, `json.RouterModelsResponseParser`) wraps the model-management endpoints (list/load/unload/await-loaded with fail-fast on failed workers) so callers don't hand-roll HTTP+JSON. Either way it is **single-instance per process** (upstream keeps shutdown state in file-scope globals) and **not available on Android** (the `subprocess.h` guard). `libjllama` loading anywhere a JVM runs is what makes this "no separate `llama-server.exe`" possible. +2. **`server.NativeServer` (native transport) — the default fat-jar server (when `--jllama-openai-compat` is absent).** Runs the **full upstream `llama_server`** (via `patches/0006` + `native_server.cpp`) inside `libjllama`, forwarding the raw llama-server argv verbatim — so **every** llama-server flag works and the **embedded WebUI is served** (when the assets are compiled in; CI's released jars have them, local `cmake` builds use the empty-asset stub). With the classic constructor it is an **independent lifecycle** (loads its own model from the argv, like `llama-server.exe`; owns the process's llama backend + stderr logging while running); the **attach constructor** (`NativeServer(LlamaModel, String...)`, via `patches/0007`'s `llama_server_attach`) instead serves an **already-loaded `LlamaModel`** — one copy of the weights, the model's worker keeps driving inference, the HTTP routes post to its queue; caller closes the server before the model. **Router mode** (start without a model argument: `--models-dir`, `GET/POST /models`, per-request model selection) works in-JVM after `NativeServer.setWorkerCommand(...)` redirects the worker spawn to a fresh JVM (`patches/0008` — upstream re-execs its own binary, which in a JVM is `java`); the typed `server.RouterClient` (+ `value.RouterModel`, `json.RouterModelsResponseParser`) wraps the model-management endpoints (list/load/unload/await-loaded with fail-fast on failed workers) so callers don't hand-roll HTTP+JSON, and its `apiKey` constructors send `Authorization: Bearer ` — required for **every** one of those calls against a router started with `--api-key` since b10519 (#26347 dropped `/models` + `/v1/models` from the public-endpoint set; `/models/load` and `/models/unload` were always gated). `awaitModelLoaded` cannot observe a model hidden by a preset with `dedup-cache-models` (b10505/#27346 omits it from `GET /models` although it still loads and serves by name), so its "not listed" message names that cause explicitly; such a model is reached by issuing the request directly instead. Either way it is **single-instance per process** (upstream keeps shutdown state in file-scope globals) and **not available on Android** (the `subprocess.h` guard). `libjllama` loading anywhere a JVM runs is what makes this "no separate `llama-server.exe`" possible. + +### `getMetrics()` — one object rebuilt from two upstream tasks + +`LlamaModel.getMetrics()` / `getMetricsTyped()` return the single server-introspection object the +Java side has always documented: `idle` / `processing` / `deferred` / `t_start`, the cumulative and +current-window `n_*` / `t_*` counter pairs, and a `slots` array. Upstream stopped emitting that in +one piece — **b10408** (#26920) reduced `server_task_result_metrics::to_json()` to the slot array, +and **b10519** (#27376) split the task in two: `SERVER_TASK_TYPE_METRICS` keeps only the counters +(its `to_json()` is unused and returns JSON null; `to_metrics()` renders them as Prometheus text) +while `SERVER_TASK_TYPE_SLOT_GET` carries the slot array plus the idle-slot count. + +`handleSlotAction(0, …)` therefore posts **both** tasks and merges the results through the pure +helper `server_metrics_to_json` (`json_helpers.hpp`, unit-tested in `test_json_helpers.cpp`), rather +than letting the Java contract follow upstream's transport split. Durations are converted from +upstream microseconds to the milliseconds the payload has always used. The merge also surfaces the +counters upstream added since — `n_prompt_tokens_cached_total` and the speculative-decoding tallies +(`n_draft_tokens_total`, `n_draft_accepted_total`, `n_draft_verif_steps_total`, +`n_accepted_per_pos_total` — upstream's own spellings, kept verbatim) — which upstream emits only +as Prometheus counters from `to_metrics()`, with no JSON representation at all; `value.ServerMetrics` exposes them with typed getters (plus a derived +`getDraftAcceptanceRate()`). No second JNI entry point and no Prometheus-text parser were needed. + +The metrics task is posted with `server_task::metrics_reset_bucket` left at its default `false`, so +`getMetrics()` never resets the current-measurement window; only an HTTP `/metrics` scrape does. ### Native Helper Architecture The project C++ helpers follow a strict semantic split: **`json_helpers.hpp`** — Pure data transforms. -- Input: `nlohmann::json`, `server_task_result_ptr`, plain C++ types. +- Input: the `json` alias (upstream `common_json` since b10585), `server_task_result_ptr`, plain C++ types. - Output: `json`, `std::vector`, `std::optional`, plain C++ types. - Zero JNI calls (`JNIEnv*` never appears). - Zero llama state (`llama_context*`, `llama_vocab*`, `server_context*` never appear). @@ -1153,7 +1218,8 @@ The project C++ helpers follow a strict semantic split: Functions: `get_result_error_message`, `results_to_json`, `rerank_results_to_json`, `parse_encoding_format`, `extract_embedding_prompt`, `is_infill_request`, -`parse_slot_prompt_similarity`, `parse_positive_int_config`, `wrap_stream_chunk`. +`parse_slot_prompt_similarity`, `parse_positive_int_config`, `wrap_stream_chunk`, +`server_metrics_to_json`. **`log_helpers.hpp`** — Pure log-formatting transforms. - Input: `ggml_log_level`, message text (`const char*`), an explicit `std::time_t` timestamp. @@ -1173,7 +1239,7 @@ Functions: `log_level_name`, `format_log_as_json`. worker thread, cached `vocab`, saved `params`, and a `readers` map for streaming tasks. - `get_jllama_context_impl` — reads Java `ctx` handle, returns the `jllama_context*` wrapper. Does NOT throw on zero handle (valid no-op for destructor-style calls). -- `require_json_field_impl` — throws `" is required"` if key is absent. +- `require_json_field_impl` — throws `" is required"` if key is absent. **Templated on the JSON type on purpose**: a plain `const nlohmann::json &` parameter still accepts a `common_json` (through its `operator std::string()`) and turns the presence check into a runtime `type_error 302`. - `jint_array_to_tokens_impl` — reads a Java `int[]` into `std::vector`. *Layer B* (requires upstream server headers in the TU before `jni_helpers.hpp`): orchestration. @@ -1285,11 +1351,38 @@ model + mmproj, and the Qwen3-TTS backbone + mmproj (`ggml-org/Qwen3-TTS-12Hz-1. smallest available quants: `Qwen3-TTS-12Hz-1.7B-Base-Q4_K_M.gguf` backbone + `mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf` mmproj — no smaller mmproj quant is published), with their `-Dnet.ladenthin.llama.*` properties set, so `LlamaEmbeddingsTest`, `MultimodalIntegrationTest`, -and `TtsIntegrationTest` **run on every platform** rather than self-skipping. `validate-models.{sh,bat}` +and `TtsIntegrationTest` are **intended** to run on every platform rather than self-skipping. + +**How the paths resolve (this was silently broken until it was fixed after the b10618 bump).** +Surefire's working directory defaults to the **module** basedir (`/llama`), while the +shared GGUF cache is restored to `/models/` and every model path — the `TestConstants` +constants and the `-Dnet.ladenthin.llama.*` properties alike — is stated relative as `models/…`. +Those therefore resolved to `/llama/models/…`, which does not exist: **every** +model-gated class aborted in its `@BeforeAll` `Assumptions.assumeTrue(file.exists())` and reported +as *nothing at all* while the job still went green, on every `test-java-*` job. Note the precise +shape, because it defeats the obvious guard: a class-level `@BeforeAll` assumption makes Surefire +record `tests="0" errors="0" skipped="0"` — the class contributes **no** test entries, so a check of +the form "did this run skip anything?" is blind to it. The only thing that catches it directly is a +floor on the number of tests actually executed (see `TODO.md`). It is why several stale +assertions (e.g. `LlamaModelTest#testGetMetrics` against a payload shape upstream had dropped at +b10408) never failed in CI. The fix is **`TestConstants.resolveModelPath` / +`resolveModelProperty`**, which accept either layout — module-relative first, then the reactor root +— so a developer with models under `llama/models/` and CI with them at the workspace root both +work, with no workflow change. Every `TestConstants` path constant is routed through it, as is +every `-Dnet.ladenthin.llama.*` fixture property; `TestConstantsTest` pins both the resolver and +the wiring (a future edit that drops the wrapper from a constant fails that test rather than +silently re-muting the suite). `llama-langchain4j` had the identical defect and carries the same +resolver as `TestModelPaths` (test classes are not shared between modules). + +`validate-models.{sh,bat}` treats all of these as **required** (a missing model hard-fails the job before tests run, so a -download regression can never silently downgrade to a skip). Only the audio-input model -(`AudioInputIntegrationTest`) still self-skips — the prompt clip is committed -(`src/test/resources/audios/sample.wav`) but the audio model + mmproj have no CI download. +download regression can never silently downgrade to a skip). **Two** classes still self-skip on +every platform, both because their model is outside the manifest: `AudioInputIntegrationTest` — the +prompt clip is committed (`src/test/resources/audios/sample.wav`) but the audio model + mmproj have +no CI download — and `LlamaTrainerIntegrationTest`, whose `net.ladenthin.llama.train.model` property +is set by no job and whose model is in no `models.csv` row. The trainer one matters more than it +looks: `train_engine.cpp` carries the same `postprocess_cpu_params` pair as `tts_params.hpp`, so the +JVM-abort class of bug documented under "Qwen3-TTS" can regress there with no runnable guard. The model set has a **single source of truth: `.github/models.csv`** (one `filename,url` row per model; `#` comments). Everything derives from it: the **`download-models`** job (ubuntu, `needs: startgate`) is the only place models are fetched from HuggingFace (one manifest-driven @@ -1346,18 +1439,19 @@ ctest --test-dir build --output-on-failure -R "ResultsToJson" | File | Tests | Scope | |------|-------|-------| -| `src/test/cpp/test_utils.cpp` | 162 | Upstream helpers: `server_tokens`, `server_grammar_trigger`, `gen_tool_call_id`, `json_value`, `json_get_nested_values`, UTF-8 helpers, `format_response_rerank`, `format_embeddings_response_oaicompat`, `oaicompat_completion_params_parse`, `oaicompat_chat_params_parse`, `are_lora_equal`, `strip_flag_from_argv`, `token_piece_value`, `json_is_array_and_contains_numbers`, `format_oai_sse`, `format_oai_resp_sse`, `format_anthropic_sse`, `parse_lora_request` | -| `src/test/cpp/test_server.cpp` | 204 | Upstream result types: `server_slot_stats` (the `timings` JSON payload; replaced `result_timings` in b10408), `task_params::to_json()` (incl. `dry_sequence_breakers`, `preserved_tokens`, `timings_per_token`), `completion_token_output`, `server_task_result_cmpl_partial` (non-oaicompat + `to_json_oaicompat` + logprobs + `to_json_oaicompat_chat` + `to_json_anthropic` + dispatcher), `server_task_result_cmpl_final` (non-oaicompat + `to_json_oaicompat` + `to_json_oaicompat_chat` + `to_json_oaicompat_chat_stream` + `to_json_anthropic` + `to_json_anthropic_stream` + tool_calls + dispatcher), `server_task_result_embd`, `server_task_result_rerank`, `server_task_result_metrics` (`to_json()` = the `/slots` array + `to_metrics()` = the `/metrics` Prometheus text), `server_task_result_slot_save_load`, `server_task_result_slot_erase`, `server_task_result_apply_lora`, `server_task_result_get_lora`, `server_task_result_error`, `format_error_response`, `server_task::need_sampling()`, `server_task::n_tokens()`, `server_schema::eval_llama_cmpl_schema()` (parsing pipeline + grammar routing + error paths + per-request `dry_*` and `sse_ping_interval` field round-trips incl. hard-limit + server-default inheritance), `response_fields` projection | -| `src/test/cpp/test_json_helpers.cpp` | 50 | All functions in `json_helpers.hpp`: `get_result_error_message`, `results_to_json`, `rerank_results_to_json` (incl. missing/out-of-range `index` rejection), `parse_encoding_format`, `extract_embedding_prompt`, `is_infill_request`, `parse_slot_prompt_similarity`, `parse_positive_int_config`, `wrap_stream_chunk` | +| `src/test/cpp/test_utils.cpp` | 167 | Upstream helpers: `server_tokens`, `server_grammar_trigger`, `gen_tool_call_id`, `json_value`, `json_get_nested_values`, UTF-8 helpers, `format_response_rerank`, `format_embeddings_response_oaicompat`, `oaicompat_completion_params_parse`, `oaicompat_chat_params_parse`, `are_lora_equal`, `strip_flag_from_argv`, `token_piece_value`, `json_is_array_and_contains_numbers`, `format_oai_sse`, `format_oai_resp_sse`, `format_anthropic_sse`, `parse_lora_request`, `common_chat_parse` over malformed UTF-8 (the `ContentOnlyParseUtf8` guard for `patches/0011`) | +| `src/test/cpp/test_server.cpp` | 206 | Upstream result types: `server_slot_stats` (the `timings` JSON payload; replaced `result_timings` in b10408), `task_params::to_json()` (incl. `dry_sequence_breakers`, `preserved_tokens`, `timings_per_token`), `completion_token_output`, `server_task_result_cmpl_partial` (non-oaicompat + `to_json_oaicompat` + logprobs + `to_json_oaicompat_chat` + `to_json_anthropic` + dispatcher), `server_task_result_cmpl_final` (non-oaicompat + `to_json_oaicompat` + `to_json_oaicompat_chat` + `to_json_oaicompat_chat_stream` + `to_json_anthropic` + `to_json_anthropic_stream` + tool_calls + dispatcher), `server_task_result_embd`, `server_task_result_rerank`, `server_task_result_metrics` (`to_metrics()` = the `/metrics` Prometheus exposition text; its `to_json()` has been unused since b10519 and returns `json{}` = JSON null), `server_task_result_slots` (`to_json()` = the `/slots` array, fed by the b10519 `SERVER_TASK_TYPE_SLOT_GET` task), `server_task_result_slot_save_load`, `server_task_result_slot_erase`, `server_task_result_apply_lora`, `server_task_result_get_lora`, `server_task_result_error`, `format_error_response`, `server_task::need_sampling()`, `server_task::n_tokens()`, `server_schema::eval_llama_cmpl_schema()` (parsing pipeline + grammar routing + error paths + per-request `dry_*` and `sse_ping_interval` field round-trips incl. hard-limit + server-default inheritance), `response_fields` projection | +| `src/test/cpp/test_json_helpers.cpp` | 63 | All functions in `json_helpers.hpp`: `get_result_error_message`, `results_to_json`, `rerank_results_to_json` (incl. missing/out-of-range `index` rejection), `parse_encoding_format`, `extract_embedding_prompt`, `is_infill_request`, `parse_slot_prompt_similarity`, `parse_positive_int_config`, `wrap_stream_chunk`, `server_metrics_to_json` | | `src/test/cpp/test_log_helpers.cpp` | 13 | All functions in `log_helpers.hpp`: `log_level_name`, `format_log_as_json` | -| `src/test/cpp/test_jni_helpers.cpp` | 54 | All functions in `jni_helpers.hpp` using a zero-filled `JNINativeInterface_` mock (incl. the `utf8_to_jstring_impl` byte-array string path: emoji byte-preservation, truncated-UTF-8 replace-not-throw) | -| `src/test/cpp/test_tts_wav.cpp` | 2 | The in-memory WAV writer `pcm_to_wav16_bytes` in `tts_wav.hpp` (WAV header/payload + little-endian clamping) — our own code, not upstream. The Qwen3-TTS pipeline it pairs with (`mtmd_helper::gen_audio`) is entirely upstream-owned (no project-side DSP to unit-test here) and covered end-to-end by the Java `TtsIntegrationTest`. | +| `src/test/cpp/test_jni_helpers.cpp` | 56 | All functions in `jni_helpers.hpp` using a zero-filled `JNINativeInterface_` mock (incl. the `utf8_to_jstring_impl` byte-array string path: emoji byte-preservation, truncated-UTF-8 replace-not-throw) | +| `src/test/cpp/test_tts_wav.cpp` | 2 | The in-memory WAV writer `pcm_to_wav16_bytes` in `tts_wav.hpp` (WAV header/payload + little-endian clamping) — our own code, not upstream. The Qwen3-TTS pipeline it pairs with (`mtmd_helper::gen_audio`) is entirely upstream-owned (no project-side DSP to unit-test here). The load path is additionally covered by `test_tts_params.cpp` (3 tests over `tts_params.hpp`'s `build_tts_params`, plus 2 pinning the upstream `-1` default it depends on), which pins the CPU-thread resolution whose absence used to crash the JVM on every platform — see the `TODO.md` entry for the mechanism. End-to-end coverage is `TtsIntegrationTest`, which is model-gated. | +| `src/test/cpp/test_tts_params.cpp` | 13 | The **three** builders every hand-assembled `common_params` goes through: `build_tts_params` (`tts_params.hpp`), `build_train_params` (`train_params.hpp`) and the shared `jllama::resolve_cpu_params` (`cpu_params.hpp`). Each builder is guarded separately on purpose — testing the resolver alone does **not** cover its call sites, because `train_engine.cpp` is compiled into `jllama` only, never into `jllama_test`, and `LlamaTrainerIntegrationTest` is gated on `net.ladenthin.llama.train.model`, which no CI job sets. Without these the JVM-abort bug could regress in the trainer on every platform, unseen. | -**Current total: 485 tests (all passing).** +**Current total: 520 tests (all passing).** #### Upstream source location (in CMake build tree) -llama.cpp is fetched via CMake FetchContent, pinned to `GIT_TAG b10456`. +llama.cpp is fetched via CMake FetchContent, pinned to `GIT_TAG b10679`. **GoogleTest** is a separate `BUILD_TESTING`-only FetchContent (`GIT_TAG v1.17.0`), used solely by the `jllama_test` C++ unit-test binary — not by the shipped library, and not coupled to the @@ -1462,27 +1556,34 @@ EXPECT_TRUE(j.contains("timings")); **3. Parameter parsing (`eval_llama_cmpl_schema`) without a model** -`server_schema::eval_llama_cmpl_schema(vocab, params_base, n_ctx_slot, logit_bias_eog, data)` +`server_schema::eval_llama_cmpl_schema(vocab, params_base, logit_bias_eog, data)` can be called with `nullptr` vocab **if the JSON does not trigger grammar/preserved_tokens tokenisation** (those are the only vocab-dependent paths). This lets us test the full -parsing pipeline including error throws: +parsing pipeline including error throws. **It takes four arguments** — the `n_ctx_slot` +parameter was dropped at b10275; a five-argument call has not compiled since. `test_server.cpp` +wraps it in a `parse_params` helper, which is the form to copy: ```cpp -common_params params_base; -std::vector no_bias; -const int n_ctx = 512; - -// test: repeat_last_n=-1 is expanded to n_ctx_slot -json data = {{"repeat_last_n", -1}}; -auto p = server_schema::eval_llama_cmpl_schema(nullptr, params_base, n_ctx, no_bias, data); -EXPECT_EQ(p.sampling.penalty_last_n, n_ctx); - -// test: invalid value throws std::runtime_error -json bad = {{"dry_sequence_breakers", json::array()}}; // empty → error -EXPECT_THROW(server_schema::eval_llama_cmpl_schema(nullptr, params_base, n_ctx, no_bias, bad), - std::runtime_error); +namespace { +task_params parse_params(const json &data) { + common_params params_base; + std::vector no_bias; + return server_schema::eval_llama_cmpl_schema(nullptr, params_base, no_bias, data); +} +} // namespace + +// test: a value inside the hard limits round-trips +EXPECT_EQ(parse_params({{"sse_ping_interval", 5}}).sse_ping_interval, 5); + +// test: out-of-range and malformed values throw std::invalid_argument +EXPECT_THROW(parse_params({{"repeat_last_n", -1}}), std::invalid_argument); +EXPECT_THROW(parse_params({{"dry_sequence_breakers", json::array()}}), std::invalid_argument); ``` +Note what the second line pins: `repeat_last_n` and `dry_penalty_last_n` carry +`set_hard_limits(0, INT32_MAX)` since **b10273**, so `-1` is simply out of range. It does +**not** expand to the slot context size any more — an older version of this section said it did. + **4. Array-returning formatters** Some methods (e.g. `to_json_oaicompat_chat_stream()`) return a JSON array of event objects, @@ -1546,8 +1647,11 @@ See [`../workspace/policies/pit-mutation-testing.md`](../workspace/policies/pit- Run PIT with the lifecycle prefix — `mvn test-compile org.pitest:pitest-maven:mutationCoverage` (from the repo root add `-f llama/pom.xml`). The gate is **hermetic** — no model or audio fixture needed: `ContentPartTest`'s `@TempDir` tests cover `value.ContentPart.audioFile(Path)` (verified -295/295, 0 NO_COVERAGE in a fixture-less sandbox; the former audio-fixture gotcha is resolved, -see `TODO.md`). +318/318 killed, 0 NO_COVERAGE, test strength 100% in a fixture-less sandbox; the former +audio-fixture gotcha is resolved). +**`net.ladenthin.llama.value.*` is a target at `mutationThreshold` 100**, so a new getter on a +`value` type needs its own test or the gate reds — the `ServerMetrics` counters added for the +`getMetrics()` merge are covered by `ServerMetricsTest`. ## JPMS Module Descriptor diff --git a/README.md b/README.md index 6b7fffb76..539aea7dc 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ **Build:** ![Java 8+](https://img.shields.io/badge/Java-8%2B-informational) ![Platform](https://img.shields.io/badge/Platform-Linux%20%7C%20macOS%20%7C%20Windows%20%7C%20Android-lightgrey) -[![llama.cpp b10456](https://img.shields.io/badge/llama.cpp-%23b10456-informational)](https://github.com/ggml-org/llama.cpp/releases/tag/b10456) +[![llama.cpp b10679](https://img.shields.io/badge/llama.cpp-%23b10679-informational)](https://github.com/ggml-org/llama.cpp/releases/tag/b10679) [![JPMS](https://img.shields.io/badge/JPMS-modular%20JAR-25A162)](https://openjdk.org/projects/jigsaw/) ![JUnit](https://img.shields.io/badge/tested%20with-JUnit6-25A162) [![JSpecify](https://img.shields.io/badge/JSpecify-1.0.0%20%40NullMarked-25A162)](https://jspecify.dev) @@ -489,6 +489,45 @@ OpenAI-compatible `/v1/chat/completions` server. For a strictly CPU-only run, us `setDevices("none").setMmprojOffload(false)` in addition to `setGpuLayers(0)`; projector offload has its own upstream default. +On a multi-GPU host the projector can be placed independently of the weights with +`setMmprojDevice("CUDA1")` (llama.cpp `--mmproj-device`, added upstream in b10541). Exactly one +device may be named; the literal `"none"` keeps the projector on the CPU. `OpenAiCompatServer`'s CLI +accepts the same flag as `-mmdev`/`--mmproj-device`, and `NativeServer` forwards it verbatim like +every other llama-server flag. + +`setMmprojDevice(...)` and `setMmprojOffload(...)` write the **same** upstream field +(`common_params::mmproj_use_gpu`), so where they disagree the outcome would depend on argv order — +and the rendered argv comes from a `HashMap`, whose order is unspecified. The builder therefore +resolves the two genuinely ambiguous combinations by dropping the earlier call, and leaves the rest +alone: + +| Combination | Resolves to | Builder behaviour | +|---|---|---| +| named device + `setMmprojOffload(true)` | `(use_gpu=true, device)` in either order | both kept — no clash | +| named device + `setMmprojOffload(false)` | order-dependent | **last call wins** | +| `"none"` + `setMmprojOffload(true)` | order-dependent | **last call wins** | +| `"none"` + `setMmprojOffload(false)` | `(use_gpu=false)` in either order | both kept — no clash | + +So a multi-GPU projector pin survives an explicit `setMmprojOffload(true)`; only a call that would +actually contradict the other is dropped. If you need a device after disabling offload, call +`setMmprojDevice` last. + +**Video input — decode settings only, so far.** `mtmd` has carried a video path since llama.cpp +**b9562** (#24269); **b10647** (#24318) added the `--video-*` CLI flags and the +`mtmd_helper_init_opt` plumbing that surfaces them. It is compiled into the shipped desktop library +(`MTMD_VIDEO` is on by default, gated on `LLAMA_SUBPROCESS`, which upstream force-disables on +Android and iOS). Its decode settings are exposed +as `setVideoFps(float)`, `setVideoTimestampInterval(long)` and `setVideoFfmpegDir(String)`. The last +one matters most in a JVM: upstream shells out to `ffmpeg`/`ffprobe` and resolves them from `PATH`, +which an application server, an Android app or a JAR-only container frequently does not have them on — +naming the directory is then the only way for video to work at all. + +What is **not** here yet is the content part: upstream's wire type for a video is +`{"type":"input_video","input_video":{"data":""}}` (raw base64, not a `data:` URI, unlike +`image_url`), gated server-side on `mtmd_helper_support_video`. `ContentPart` has no `videoFile(...)` +factory emitting that shape, so these knobs currently configure a path this API cannot yet feed +directly. Tracked in `TODO.md`. + **Audio input** works identically — load an audio-capable model (Ultravox, Qwen2.5-Omni, …) with its audio `--mmproj` and add a `ContentPart.audioFile(...)` (or `inputAudio(bytes, "wav"|"mp3")`) part. It serializes to the OpenAI `input_audio` content part and routes through the same `mtmd` pipeline: @@ -694,7 +733,11 @@ request, so generation and `save`/`restore` operate on the same KV state. Typed results expose logical prompt, generated, cached prompt, and evaluated prompt counts through `Usage`. Per-request timing also remains available through `Timings.getCacheN()`. `LlamaModel.getMetricsTyped().getSlotMetrics()` reports each slot's logical, processed, cached, -decoded, and remaining token counts. +decoded, and remaining token counts, and the same `ServerMetrics` view carries the server-wide +lifetime counters — including cached prompt tokens (`getCumulativeCachedPromptTokens()`) and the +speculative-decoding tallies (`getDraftTokensTotal()`, `getDraftAcceptedTotal()`, +`getDraftVerifyStepsTotal()`, `getDraftAcceptedPerPosition()`, plus the derived +`getDraftAcceptanceRate()`), which upstream otherwise exposes only as Prometheus text. The embedded HTTP server exposes the same native JSON at authenticated `GET /metrics`, with the slot array alone at `GET /slots`. OpenAI responses preserve @@ -779,7 +822,7 @@ java -cp target/llama-.jar net.ladenthin.llama.server.OpenAiCompatServe Run with `--help` for the full option list (`-m/--model`, `--host`, `-p/--port`, `-c/--ctx-size`, `-b/--batch-size`, `-ub/--ubatch-size`, `-ngl/--n-gpu-layers`, `-t/--threads`, `-tb/--threads-batch`, `-ctk/--cache-type-k`, `-ctv/--cache-type-v`, `--jinja`, `--chat-template-kwargs`, `--parallel`, -`--model-id`, `--api-key`, `--mmproj`, `--embedding`, `--reranking`). The tuning flags mirror +`--model-id`, `--api-key`, `--mmproj`, `-mmdev/--mmproj-device`, `--embedding`, `--reranking`). The tuning flags mirror llama.cpp's server, so an invocation like `--jinja --chat-template-kwargs '{"reasoning_effort":"low"}' -ctk q8_0 -ctv q8_0 -b 4096 -ub 2048` works directly. @@ -923,6 +966,21 @@ client.unloadModel("Qwen3-0.6B-Q4_K_M"); // POST /models/unload failed-worker marker. Chat requests then select a model per request via the standard `"model"` field on `POST /v1/chat/completions`. +Against a router started with `--api-key`, pass the key — it is sent as +`Authorization: Bearer ` on every call. All of them need it: `/models/load` and +`/models/unload` were always gated, and since llama.cpp b10519 the listing endpoints are too. + +```java +RouterClient client = new RouterClient(8080, System.getenv("LLAMA_API_KEY")); +// or, for a remote router: new RouterClient("router.internal", 8080, key) +``` + +> [!NOTE] +> `awaitModelLoaded` waits by polling `GET /models`, so it cannot observe a model the router +> deliberately hides from that listing — a cache model deduplicated by a preset with +> `dedup-cache-models` still loads and still serves by name, but never appears. For those, skip the +> await and issue the request directly; with autoload the router waits for the worker itself. + ### LangChain4j integration A separate artifact, **`net.ladenthin:llama-langchain4j`**, adapts a `LlamaModel` to diff --git a/TODO.md b/TODO.md index 22893e42f..f97ac1488 100644 --- a/TODO.md +++ b/TODO.md @@ -13,6 +13,186 @@ cross-cutting initiative. ## Open — jllama-specific +### Model-backed tests that the CI-skip fix newly exposed (b10618 PR #403) + +Making the model-gated suite actually run in CI (`TestConstants.resolveModelPath`, see the Done +section) turned a green-but-silent pipeline into a red-and-honest one. Everything below was already +broken before this PR; none of it was visible while every model-gated class self-skipped. Two items +were fixed in this PR, four are open. + +**How to read the CI evidence.** Surefire runs classes in filesystem order, which differs per OS, and +`TtsIntegrationTest` kills the fork. On Linux/macOS it lands early (Ubuntu got through only **85** +tests), so those jobs report *nothing but* the crash. The two Windows jobs happen to run it last and +therefore reach **1461** tests — they are the only jobs whose failure list is complete. Do not read a +short Linux failure list as "Linux is healthier". + +**Confirmed platform-independent (macOS 14, head `7be24a6`).** That job reached **517** tests before +the crash and reproduced `SessionForkRewindIntegrationTest` (both cases) and +`NativeServerAttachIntegrationTest` **identically** — same assertions, same messages. So these are +not a Windows quirk. Its exit code was **141 (SIGPIPE)** rather than Ubuntu's 134 (SIGABRT), on the +same crashed test. + +- **[FIXED — confirmed in CI] `TtsIntegrationTest` aborted the JVM natively on all 6 test + platforms.** **Root cause: our own hand-built `common_params`.** + `common_cpu_params::n_threads` defaults to **-1**, and `postprocess_cpu_params` — the function + that resolves it — is called **only** from `common/arg.cpp`, i.e. only for params that came + through `common_params_parse`. `common_init_from_params` does not call it. `tts_engine.cpp` + assembled `common_params` by hand and set only `cpuparams.n_threads`, so `cpuparams_batch` + stayed at -1; `common_threadpools::init` then saw a mismatch between the two and built a + *second* threadpool with **-1 threads**. `ggml_threadpool_new` sizes its worker array as + `sizeof(struct ggml_compute_state) * tpp->n_threads`, which for -1 wraps to a huge `size_t`; + `ggml_aligned_malloc` returns `NULL` (after logging *"insufficient memory"*) and the next line + is an unchecked `memset(workers, 0, workers_size)` — the `bzero`-at-address-0 seen on every + platform. + Fixed by mirroring `arg.cpp`'s two calls, including the `role_model` argument that makes the + batch pool inherit rather than resolve independently. `train_engine.cpp` had the same latent + defect (it set *neither* count, so both stayed -1 — they matched, so it built one pool with + -1 threads instead of two) and got the same fix. `jllama.cpp` / `jni_helpers.hpp` are safe: + their params come from `common_params_parse`, so `arg.cpp` resolves them. + Guarded by 5 new model-free C++ tests (`test_tts_params.cpp`, suite 499 → **504**), verified + by negative control: removing the two calls turns `TtsParams.ResolvesBothCpuThreadCounts` red + with `actual: -1 vs 0`. The builder was extracted to `tts_params.hpp` so the test exercises + the *real* production path rather than a copy that could drift. + + **CI confirmation** (Ubuntu, run 32950691947 on `999034b`): **1689 tests ran** where the fork + previously died at 85, and the crash-print step found **no `hs_err_pid*.log` at all**. The only + thing it printed was the router worker's stderr (see the router entry below). + + **How it was localised** (kept because the method generalises, not because the bug is still + open). The crash log became readable in the job log itself with `cfda4a9` (the section-3.1 print + step), and that is what produced everything below. The abort is a **null-pointer dereference + while zeroing a buffer during the TTS model load**, identically on two OS families: + + | | Linux x86-64 (run 32941522341) | macOS 15 arm64 (same run) | + |---|---|---| + | signal | `SIGSEGV`, `si_code 1 (SEGV_MAPERR)`, `si_addr 0x0` | `SIGSEGV` | + | frame | `C [libc.so.6+0x1896ca]` | `C [libsystem_platform.dylib+0x2fb0] __bzero+0x20` | + | registers | `RDI=0`, `RSI=0` | — | + + On x86-64 SysV `memset(void *s, int c, size_t n)` passes `s` in RDI and `c` in RSI, so + `RDI=0, RSI=0` is `memset(NULL, 0, n)` — the same call macOS names outright as `bzero`. The Java + frames are `TextToSpeech.loadNative` -> `TextToSpeech.` -> + `TtsIntegrationTest.synthesizesWellFormedWav`, on the `main` thread in `_thread_in_native`, + after ~258 s (Linux) / ~288-300 s (macOS) of **total JVM elapsed time** -- that is time since the + fork started and ran the rest of the suite, NOT time spent inside the load, so it says nothing + about how far the load got. + + Ruled out already, do not re-investigate: (1) a JNI signature mismatch of the kind that broke + `LlamaQuantizer` — `loadNative` is `(String, String, int, int) -> long` on both the Java and the + C++ side, verified; (2) `parse_jstring` — it guards a null `jstring`, a pending exception and a + null byte array, returning an empty string in each case; (3) the obvious null checks in + `tts_engine.cpp`'s load, which do test `model` / `ctx` / `mctx` and return `nullptr` with a + message. The faulting allocation is therefore *inside* the load, before those checks are reached. + + **The macOS 15 Metal job resolved the native stack, and it is exactly two frames:** + `__bzero+0x20` called from `libjllama.dylib` `Java_..._TextToSpeech_loadNative+0x60`. Do NOT read + that as "the bug is in `loadNative`". Disassembling the shipped Linux library shows `loadNative` + contains **no** `memset`/`bzero` at all — its only calls are `parse_jstring`, `engine_init` (via + the PLT, so not inlined), `operator delete` and `__stack_chk_fail`. The JVM's frame-pointer + walker dropped the intermediate frames, leaving only the outermost and innermost. The fault is + therefore under `engine_init` — in `common_init_from_params` or the mtmd/mmproj init — not in the + JNI wrapper. (That much held up: it is in `common_init_from_params`, via + `common_threadpools::init`.) (Symbol attribution itself *is* trustworthy here: the library exports 12 529 symbols, + the whole llama/common layer included, so a PC inside `common_init_from_params` would have been + named as such. What is unreliable is the *depth* of the walk, not the naming.) + + The shape of the guess at that point — *an allocation that returns null and is then zeroed + unchecked* — was right; the attribution was not. It was blamed on host memory pressure (the + macOS runner has **7 GB RAM / 3 cores** against Linux's 15 GB / 4) and on the `mmproj` being the + largest new buffer in the path. Neither is involved: the request is for + `sizeof(struct ggml_compute_state) * (size_t) -1` bytes, which no allocator can satisfy on any + host with any amount of RAM free. That both hosts failed identically was the clue that memory + pressure could not be the explanation. + + Superseded note (kept because the reasoning was cited earlier): it was NOT certain an `hs_err` + existed at all — `if-no-files-found: warn` and Windows' exit code 1 left that open. It does + exist, on both platforms checked. + +- **[SUPERSEDED — see the entry above] `TtsIntegrationTest` aborts the JVM natively on all 6 test platforms.** + Ubuntu exit 134 (SIGABRT); macOS 14 Metal, macOS 15 Metal, macOS 15 no-Metal; Windows Ninja and + Windows MSVC exit 1. Not an OOM (Linux had ~14 GiB free, Windows ~12.3 GiB of 16 GiB). Not caused by + the bump: `git log b10456..b10618 -- tools/mtmd/mtmd-helper.{cpp,h}` contains only video/webp/ + mergeable/sha256/cmake commits, and grepping that diff for `gen_audio|step_gen|step_prompt| + get_output|GGML_ASSERT|GGML_ABORT|throw` yields zero hits. It is a latent defect in + `tts_engine.cpp`'s drive of `mtmd_helper::gen_audio`, exposed the first time the test ran. + Next step: read `hs_err_pid*.log` from artifact `windows-output` (ID 9583568326) or + `error-log-macos-14-metal` (ID 9583085009) of run 32899147975 for the aborting frame. Because it + takes the whole fork down it also truncates every job's test run, so it blocks seeing the rest of + the suite and should be fixed first. + +- **[FIXED in this PR] `SessionForkRewindIntegrationTest` — empty reply after a slot restore.** + `rewindRestoresTranscriptAndConversationContinues` and + `forkCreatesIndependentSessionWithSameTranscript` failed `assertThat(reply.isEmpty(), is(false))` + after `rewind()` / `fork()`. The diagnosis above was right that it is not a bump regression, but + wrong that it is the KV restore: the model is Qwen3-0.6B, a **reasoning** model, and the tests' + token budget was being spent entirely inside ``, so generation completed normally and + returned no assistant *content*. Fixed in `cc67ea7` by budgeting past the thinking block. The same + root cause resurfaced later in `llama-langchain4j` (`dd07b0e`), where both chat tests now share a + `MAX_OUTPUT_TOKENS = 1500` matching `ReasoningBudgetTest`'s `N_PREDICT`. Green on all six Java + platforms in run 33111759140. + +- **[FIXED in this PR] `NativeServerAttachIntegrationTest.completion_overHttp_served` — HTTP 500.** + `{"error":{"code":500,"message":"The model produced output that does not match the expected + Content-only format"}}`. Correctly identified above as long-standing upstream behaviour rather than + a bump regression. Root cause: `common_peg_until_parser` (`common/peg-parser.cpp`) tolerates an + **incomplete** trailing UTF-8 sequence in lenient mode — the only mode `common_chat_peg_parse` ever + uses — but its **invalid**-byte branch returned `FAIL` unconditionally, so a single malformed byte + anywhere in the output turned a *finished* generation into a 500. Fixed by local patch `0011` + (`ca60947`), which makes the invalid branch honour leniency exactly as the incomplete branch does; + strict mode is unchanged. Guarded by the `ContentOnlyParseUtf8` tests in `test_utils.cpp`, which — + unlike the upstream test the patch also adds — run in CI on every platform. Upstream-submittable; + re-checked at b10679 — the `until` parser's `INVALID` branch in pristine + `b10679:common/peg-parser.cpp` still returns `FAIL` with no `is_lenient()` guard — so the patch stays. + +- **[ANSWERED] Re-check the full suite once the TTS crash is fixed.** Done: Ubuntu on `999034b` + ran **1689 tests, 3 failures, 1 error, 2 skipped**. Exactly one item was new — the router entry + below — and the other three are the already-recorded `SessionForkRewind` pair and + `NativeServerAttach`. So the earlier Windows list was a lower bound by one item, not by many. + +- **[FIXED in this PR] `RouterModeIntegrationTest.setup:114` — the worker JVM could not load its + own main class.** `IllegalState: Router worker for model 'Qwen3-0.6B-Q4_K_M' failed with exit + code 1`; the worker's stderr (visible only because of the section-3.1 print step, in the surefire + `.dumpstream`) said `Could not find or load main class net.ladenthin.llama.server.NativeServer`. + Not a bump regression and not a router defect: `target/classes` carries a `module-info.class`, so + **Surefire auto-detects a named module and runs the main classes on the module path** — + `java.class.path` then holds only `target/test-classes` plus the dependency jars. The test built + the worker command from that property alone, so the spawned JVM had neither `NativeServer` nor the + packaged native library. It had never surfaced because the test self-skipped in CI for as long as + the model paths resolved to the wrong directory (the `TestConstants.resolveModelPath` fix in this + PR is what made it run). Fixed by deriving the main-classes root from + `NativeServer.class.getProtectionDomain().getCodeSource()`, which is correct in either mode and + for a directory or a jar alike. Reproduced and verified locally without a model: with the old + classpath the worker exits 1 on `ClassNotFoundException`; with the fix it loads `libjllama.so` and + reaches llama-server's own argument parser. + + Note for later: the project sets no `false` (srcmorph does, and its + pom explains why classpath mode is the representative test environment). Flipping it would remove + this whole class of surprise, but it changes how all 1689 tests run and does not belong in a + version-bump PR. + +- **[FIXED in this PR] `LlamaQuantizer.quantizeNative` had C++ linkage — the whole class was + unusable in every published jar.** `QuantizerIntegrationTest` failed all 3 tests with + `UnsatisfiedLinkError: 'void net.ladenthin.llama.LlamaQuantizer.quantizeNative(...)'`. Cause: the + C-linkage declarations come from the javac-generated `jllama.h`, which covers **only** + `LlamaModel`; every other class's JNI function must say `extern "C"` itself (`train_engine.cpp` and + `native_server.cpp` do). `quantizeNative` did not, so it was exported as + `_Z54Java_net_ladenthin_llama_LlamaQuantizer_quantizeNativeP7JNIEnv_...` and the JVM could never + resolve it. Reproduced on Linux with `nm -D`, so it was never Windows-specific — the public + `LlamaQuantizer` API has never worked. Fixed, and guarded model-free by + `NativeLibraryLoadSmokeTest.quantizerNativeEntryPointResolves` (`nm -D` on the rebuilt lib now shows + zero mangled `Java_*` exports). **Confirmed on a second toolchain:** at `7be24a6` the macOS 14 job + ran `QuantizerIntegrationTest` at *3 tests, 0 failures, 0 errors* with no `UnsatisfiedLinkError` + anywhere in the run — the same 3 tests that were 2 failures + 1 error before the fix. Worth having, + since symbol export and mangling differ between ELF/gcc and Mach-O/clang. + +- **[FIXED in this PR] `JsonEndpointParametersTest.testDryMultiplierAccepted` sent + `dry_penalty_last_n: -1`.** The one genuine b10456→b10618 regression in the list: b10273 gave the + field hard limits `[0, INT32_MAX]` (0 = disabled) and dropped the old "-1 = context size" sentinel, + so the request now 400s. The `InferenceParameters` / `ModelParameters` setters were already fixed in + this PR; this test builds raw JSON and bypassed them. A repo-wide sweep confirms it was the only + remaining `-1` on either de-sentinelled field. + + ### LlamaLoader extraction-directory isolation (optional follow-up, low priority) Left over from the 2026-06-20 code audit (18/18 findings fixed in PRs #258/#260, regression tests in @@ -34,8 +214,9 @@ round-trips — see CLAUDE.md "Two server modes"). **Owner priority: the native- - **Future *output* modalities (audio / image) — design note, not yet actionable.** llama.cpp's server produces text (plus embeddings/rerank) only; the integration points are isolated (a new `OpenAiBackend.stream*` primitive + `OpenAiSseFormatter.*Chunk` per modality). Two future hooks: - OuteTTS behind an `/v1/audio/speech`-style route; proxying image/audio generation to an external - model. Keep chunk formatters modality-neutral. + the existing `TextToSpeech` (Qwen3-TTS since llama.cpp b10270 — OuteTTS no longer exists upstream) + behind an `/v1/audio/speech`-style route; proxying image/audio generation to an external model. + Keep chunk formatters modality-neutral. - **Incremental tool-call streaming on the alternative surfaces.** Ollama/Anthropic/Responses emit each tool call whole at end-of-stream (`ToolCallDeltaAccumulator`); revisit only if a client needs incremental `input_json_delta` / `function_call_arguments.delta` fidelity. @@ -117,6 +298,50 @@ upstream PR #22393 — it drops automatically when that merges.) These are JNI plumbing items for upstream API additions. Policy: add only after a real user request — they are mostly relevant to specific model families or specialized workflows. +- **Video input (`ContentPart.videoFile(...)`).** `mtmd` has had an end-to-end video path since + llama.cpp **b9562** (#24269) — `mtmd_helper_video_init_params` was already present at the previous + pin, b10456. What **b10647** (#24318, commit `f29551215`) added is the surfacing: a fourth + `mtmd_helper_init_opt` parameter on the bitmap/tokenize helpers and the CLI flags `--video-fps`, + `--video-timestamp-interval`, `--video-ffmpeg-dir`. Older notes cite b10649 for all of it because + that was the *bump step* that carried it; b10647 is the tag that introduced it, and the video path + itself is older still. + + **The three flags are now exposed** as `ModelParameters.setVideoFps` / + `setVideoTimestampInterval` / `setVideoFfmpegDir`. They were initially refused at the b10649 bump + as "inert without a way to submit a video"; a later audit showed that reasoning was wrong on two + counts. First, they are not inert: `server_context::load_model` copies them into its own + `init_opt` when the projector loads, and that `init_opt` is what `server-context.cpp` passes to + `process_mtmd_prompt` on the task path this binding uses — so they take effect for any media the + caller attaches. Second, video decoding is genuinely compiled in: `MTMD_VIDEO` defaults to `ON` + (it needs only `LLAMA_SUBPROCESS`, also `ON`), and the shipped `libjllama.so` carries the ffmpeg + invocation strings. `setVideoFfmpegDir` is the one that matters most, because upstream otherwise + looks the binaries up on `PATH`, which a JVM process frequently does not have them on. + + What is still missing is the content part. Upstream's wire type is + `{"type":"input_video","input_video":{"data":""}}`, handled in + `oaicompat_chat_params_parse` and gated on `allow_video = mtmd_helper_support_video(mctx)`. Note it + calls `handle_media(..., accept_base64_uri = false)`, i.e. **raw base64 only** — unlike `image_url`, + it will not take a `data:` URI, so `ContentPart.videoFile(Path)` must emit the bare base64 payload, + not the `data:video/mp4;base64,...` form the image factories build. + + (An earlier draft of this entry suggested smuggling video through + `ContentPart.imageBytes(bytes, "video/mp4")`, on the reasoning that `mtmd_helper_bitmap_init_from_buf` + sniffs the container. That is plausible — the `image_url` branch does pass + `accept_base64_uri = true` and does not validate the MIME string — but it is **untested here** and + additionally gated on `allow_image`, so it is not documented as a supported route.) + + Remaining work: the factory, a bytes overload, and an integration test. Note the runtime cost: + upstream **shells out to `ffmpeg`/`ffprobe`**, so a consumer needs those binaries, which makes the + feature untestable on a CI runner without them. + +- **`--spec-synth-len` / `--spec-synth-rates` — deliberately NOT exposed, and this should stay that + way.** Added in b10649. Upstream's own help text marks both **"(benchmarking only)"**: they + synthesise fake per-position acceptance probabilities so the speculative-decoding harness can be + measured without a real draft model. They are an instrument for benchmarking llama.cpp itself, not a + knob for an application, and exposing them as library API would invite callers to "tune" numbers + that fabricate rather than measure acceptance. Anyone who genuinely wants them already has them: + `NativeServer` forwards raw llama-server argv verbatim. + - **Expose `--spec-draft-backend-sampling` toggle via `ModelParameters.setSpecDraftBackendSampling(boolean)`.** Added in b9437 (env `LLAMA_ARG_SPEC_DRAFT_BACKEND_SAMPLING`). Backend sampling for the speculative draft is enabled by default upstream but auto-disabled on `LLAMA_SPLIT_MODE_TENSOR` setups; an explicit Java-side setter lets callers force-disable it for benchmarking or for backends with sampler bugs. Speculative-decoding power users. - **Expose runtime reasoning control via `InferenceParameters.setReasoningControl(boolean)` + `LlamaModel.endReasoning(...)`.** Added in b9444–b9490: new `common_params_sampling::reasoning_control` flag arms the budget sampler so reasoning can be ended at runtime, and new `common_sampler_reasoning_budget_force(common_sampler *)` triggers the end-of-thinking token injection on the next sample. Upstream also adds a `POST /v1/chat/completions/control` server endpoint accepting `{"id": "...", "action": "reasoning_end"}`. Java mapping would be: (a) `InferenceParameters.setReasoningControl(boolean)` arms the sampler on the inference run, (b) a new `LlamaModel.endReasoning(int slotId)` (or per-streaming-task-id) JNI method calls the upstream `common_sampler_reasoning_budget_force` against the slot's sampler. Useful for interactive UIs that want a "skip thinking and answer now" button. Relevant only for reasoning-trained models (DeepSeek-R1, Qwen3-Thinking, GPT-OSS-Reasoner, etc.). @@ -222,6 +447,152 @@ the load-time failure class is already covered, and a slow smoke tends to get ma **Not yet observed green in CI** — the job and the two sibling-repo smokes landed in one change set and have only run locally so far. +### Test-coverage gaps found by the b10679 mutation audit (PR #403) + +> **Update.** The `IdleSleepWakeIntegrationTest` added to close the `wake_and_post` gap immediately +> found a real JVM crash (SIGSEGV on all six CI platforms) — see the CHANGELOG "Fixed" entry. Both +> facets are fixed in that PR via the `wake_server()` choke point. This is the clearest evidence for +> the entry below about a floor on executed tests: the defect had been reachable from public API for +> as long as `--sleep-idle-seconds` has existed, and nothing ran that path. + +A mutation pass over the branch applied 27 mutations and 26 went red on the test that claims them, +so no test here passes with its subject deleted. What it did find is code with **no runnable guard**. +Two of the three were closed in that PR (a model-free `jsonSchemaToGrammar` test in +`NativeLibraryLoadSmokeTest`, and `IdleSleepWakeIntegrationTest` for the `wake_and_post` path); +these are what remains. + +- **`patches/0010` has no guard that runs on a model-free host.** Reverting the patch's + `(int)` cast in the fetched `tools/server/server-context.cpp` leaves `ctest` at a clean **520/520** — + the always-run `C++ Tests` job cannot see the regression at all. The only guard is + `NativeServerAttachIntegrationTest.models_reportNumericVocabType`, which is model-gated; it *does* + run on all six CI Java jobs (the full model set is downloaded there), so this is a coverage gap + rather than a shipping risk today. It becomes one the moment a platform stops downloading models. + `CommonJsonEnumTrap` in `test_json_helpers.cpp` cannot help — it builds its own JSON literals and + calls no project code. A direct unit test is impossible as things stand: `get_res_model_info` is + `static` inside `server-context.cpp` and unreachable from `jllama_test`. Cheapest real fix is a + CI assertion in the `C++ Tests` job that the patch is present in the fetched tree + (`grep -c '(int) meta.model_vocab_type'` plus a non-empty `git -C _deps/llama.cpp-src diff`). + +- **`TestConstantsTest.theShippedModelConstantsGoThroughTheResolver` is vacuous when the fixture is + absent.** Mutating `MODEL_PATH = resolveModelPath("models/…")` to the bare literal leaves the test + green with `models/` empty, and only goes red once the GGUF actually exists. In CI that is the + normal case (`validate-models.sh` hard-fails first), so residual risk is low — but the two + `src/test/resources/...` constants resolve from the module basedir either way, so their wrapper is + undetectable **even in CI**. Fix: assert the wiring structurally rather than by value — reflect + over the `String` constants and require each `models/…`-shaped one to equal + `resolveModelPath(literal)` against a `@TempDir` fixture planted at the reactor root, so the + assertion does not depend on a real model being present. + +- **Nothing asserts a floor on the number of tests actually executed.** A class-level `@BeforeAll` + assumption makes Surefire record `tests="0" errors="0" skipped="0"` — the class contributes no + entries at all, so "did the run skip anything?" is structurally blind to it. This is exactly how + the model-gated suite stayed silently muted for months. Summing `tests=` across + `target/surefire-reports/TEST-*.xml` in each `test-java-*` job and failing below a pinned minimum + is the one check that would have caught it directly, and it is cheap. + +### Release/build robustness gaps found by the b10679 audit (PR #403) + +Both are **pre-existing** and orthogonal to a version bump, so they were recorded rather than folded +into that PR. + +- **Two `all-*-aarch64` fat jars are attached to releases with no smoke job.** + `.github/package-fatjars.sh` emits four OS/arch fat jars (`linux-x86-64`, `linux-aarch64`, + `windows-x86-64`, `windows-aarch64`), all uploaded as `llama-fatjars` and attached by + `github-release-signed` / `github-snapshot`. Only the two **x86-64** ones are smoked + (`smoke-fatjar-linux`, `smoke-fatjar-windows`); grepping `publish.yml` for `all-linux-aarch64` or + `all-windows-aarch64` returns nothing, so neither is ever downloaded or launched. + + That directly violates the cross-repo rule in + [`../workspace/policies/fat-jar-release-assets.md`](../workspace/policies/fat-jar-release-assets.md) + — *"No release asset is attached that CI has not run"* — which exists because a corrupt macOS dylib + shipped in three releases under a fully green pipeline. The fix is cheap: the workflow **already** + uses the free ARM runners elsewhere (`ubuntu-24.04-arm` for the aarch64 CPU and Vulkan builds, + `windows-11-arm` for the Windows arm64 build), so `smoke-fatjar-linux-aarch64` and + `smoke-fatjar-windows-arm64` can mirror the existing smoke jobs and join both publish jobs' + `needs:`. Not done in the bump PR because it widens a version bump into CI work and would gate that + PR on a pre-existing defect if either jar turns out to be broken. + +- **The patch applier silently accepts a partially-reverted source tree.** The stamp file records the + checked-out llama.cpp commit plus each patch's SHA-256 — **nothing about the resulting file + contents**. Reverting one patched file after a successful apply leaves the stamp valid and the tree + still dirty (the other patched files are still modified), so the dirty-tree branch reports + "already applied — skipping", exits 0, and the build compiles unpatched code. Reproduction: + + ```bash + # with the tree fully patched and the stamp written: + git -C checkout -- common/peg-parser.cpp # drops patch 0011's fix + cmake -DPATCH_DIR=... -DLLAMA_SRC=... -P llama/cmake/apply-llama-patches.cmake + # -> "8 patch(es) already applied — skipping", exit 0, patch NOT restored + ``` + + Every other path is correctly fail-loud (committed-patch state, stamp/HEAD mismatch on a dirty tree, + and a non-git-worktree re-run all exit 1). The fix is a content oracle in the manifest — cheapest is + to append `git -C diff --no-color | sha256`, or per-patched-file blob hashes — so a reverted or + hand-edited file invalidates the stamp. **CI is unaffected** (every job configures into a fresh build + directory); this only bites a local reconfigure, which is why it was not rushed. Note the stamp + format change will make every existing local build dir abort with the applier's + "configure into a fresh build directory" message — that is the designed fail-loud path, not a + regression. + +### Test-coverage debt found during the b10649 review (PR #403) + +Each item below was verified against pristine upstream tags and is real, but none is a regression +introduced by the version bump — they were deferred to keep that PR landable. + +- **`ModelParameters` emits five CLI flags the server arg parser rejects, so any caller of them + cannot load a model.** `--dump-kv-cache`, `--hf-repo-v` and `--hf-file-v` no longer exist anywhere + in llama.cpp (absent at both b10456 and b10649); `--grp-attn-n` and `--grp-attn-w` still exist but + are `set_examples({LLAMA_EXAMPLE_COMPLETION, ...})`, so `add_opt` never registers them for + `LLAMA_EXAMPLE_SERVER` — the example jllama parses with. An unregistered flag is not ignored: + `arg.cpp` throws, `common_params_parse` returns false, and `load_model_impl` throws + `LlamaException("Failed to parse model parameters")`. Four existing tests pin the dead literals and + would pass forever. Fix: deprecate the five members the way this PR handled + `withTfsZ`/`withPenalizeNl` (keep source compatibility, never write the map), and add a hermetic + `jllama_test` contract test that walks `common_params_parser_init(params, LLAMA_EXAMPLE_SERVER)`'s + `ctx.options` (upstream's own `test-arg-parser` pattern; the symbols already link into + `jllama_test`) and asserts every flag `ModelParameters`/`ModelFlag` can emit is in that set, + excluding only `--vocab-only`, which `strip_flag_from_argv` removes on purpose. A grep-based sweep + is **not** sufficient — it is structurally blind to example scoping, which is exactly how + `--grp-attn-w` hides. + +- **`acquire_jllama_context_impl` / `release_jllama_context_impl` / `jllama_context_guard` have no + model-free unit guard.** These three (`jni_helpers.hpp`) are the whole `close()`-vs-inference + use-after-free defence, and grep finds zero references across all seven `test_*.cpp` files, while + their sibling `get_jllama_context_impl` has three tests. A dropped `fetch_add`, or a guard whose + destructor stops calling release, produces a use-after-free during `close()` or a `close()` that + hangs forever. `LlamaModelTest#testCloseDuringInference` covers the mechanism end to end but only + bluntly. They are absent from `jllama_test` only because they are `inline` and never odr-used + there: `g_ctx_mutex` is `extern` in the header and defined in `jllama.cpp`, which `jllama_test` + does not compile — a test-local definition at global scope unblocks it. + +- **`OSInfo`: the `archMapping` alias branch is untested.** `getArchName()`'s map lookup has no + assertion anywhere — the two test call sites either take the override early-return or only assert + non-empty — so a lost `amd64 -> x86_64` entry would send `LlamaLoader` to a resource directory + that does not exist. Cheap to close: set `os.arch`, assert the non-identity aliases only (identity + entries such as `s390x` are behaviourally redundant with the `\W`-stripping fallback). + +- **`LlamaTrainer`'s end-to-end path runs on no CI platform.** `LlamaTrainerIntegrationTest` + self-skips everywhere: `net.ladenthin.llama.train.model` is set by no job and its model is in no + `.github/models.csv` row, so `validate-models.{sh,bat}` does not treat it as required. The C++ half + is now mitigated (`test_tts_params.cpp`'s `TrainParams` + `ResolveCpuParams` suites), but nothing + exercises the Java → JNI → native trainer round trip. Adding a small training model to `models.csv` + plus the matching property to the Java test jobs would close it. + +- **`LlamaLoader`'s jar-extraction internals need synthetic jar fixtures.** `readBackendManifest`, + `tryLoadBackend`, `extractFile`, `moveIntoPlace`, `cleanPath` and `hasNativeLib` are named in no + test; `BackendManifestLoadTest` and `LlamaLoaderTest` drive the class only from outside via system + properties. Covering the multi-backend fat-jar path (per-backend temp subdir extraction, + manifest-extras-first ordering, `UnsatisfiedLinkError` fallback to the next backend and then to the + default CPU natives) means building jars carrying a `jllama-backends.txt` and dummy payloads. + +- **`Java8CompatibilityHelper` is mostly dead code — decide delete vs. test.** Six of its seven + public methods have zero call sites repo-wide; the only live one is + `toString(ByteArrayOutputStream, Charset)`, used once in `ProcessRunner`. Writing tests for the + rest would pin dead code. + +- **`ContentPart.videoFile(...)` — see the video-input entry above** for the wire shape upstream + expects (`input_video`, raw base64, not a `data:` URI). + ## Open — cross-cutting (slice for this repo) - **jqwik pin policy** — see [`../workspace/policies/jqwik-prompt-injection.md`](../workspace/policies/jqwik-prompt-injection.md). `jqwik.version ≤ 1.9.3` is mandatory. @@ -256,6 +627,41 @@ and have only run locally so far. ## Done (kept for history) +### 2026-08-25 — the five gaps recorded during the b10618 bump, now fixed + +All five were found while bumping llama.cpp to b10618, recorded there as diagnoses rather than fixes +(the bump commit had to stay a bump), and closed in a follow-up. Details in CLAUDE.md; one-liners: + +- **Model-gated Java tests silently self-skipped in CI** — Surefire's working directory is the + module basedir while the GGUF cache is restored to the reactor root, so every `models/…` path + resolved to nothing, every model-gated class aborted in `@BeforeAll`, and the job still went green. + Fixed with `TestConstants.resolveModelPath` / `resolveModelProperty` (accept either layout), routed + through every path constant and every `-Dnet.ladenthin.llama.*` fixture property, plus + `TestConstantsTest` pinning the resolver **and** the wiring. `llama-langchain4j` had the same + defect and got the same resolver as `TestModelPaths`. Verified end-to-end: with a placeholder at + `/models/`, `LlamaModelTest` reports `Skipped: 0` and actually attempts the load. +- **`getMetrics()` payload contract drifted at b10408** — restored in the native layer instead of + bending the Java contract: `handleSlotAction(0, …)` now posts both `SERVER_TASK_TYPE_METRICS` and + `SERVER_TASK_TYPE_SLOT_GET` and merges them via the pure `server_metrics_to_json`. All three + consumers keep working unchanged; `value.ServerMetrics` additionally exposes the cache and + speculative-decoding counters that previously existed only in the Prometheus text. + `LlamaModelTest#testGetMetrics` now asserts the parsed shape rather than substrings (the old + assertion was satisfiable by the slot entries themselves), and `GET /slots` answers `[]` instead of + an empty body when the payload carries no `slots` key. +- **`RouterClient` had no API-key support** — added `RouterClient(port, apiKey)` / + `RouterClient(host, port, apiKey)` sending `Authorization: Bearer `; an empty key behaves like + none, `toString()` never prints it, `equals` includes it. +- **`RouterClient.awaitModelLoaded` vs hidden router models** — the TODO's first suggestion (poll to + the timeout) was wrong: upstream filters hidden models out of `GET /models` permanently, so no + amount of polling observes one. Fixed the honest way — the "not listed" message now names the + `dedup-cache-models` cause and the javadoc documents the direct-request path. +- **`apply-llama-patches.cmake` was not idempotent** — replaced per-patch reverse-checking with a + stamp file (llama.cpp commit + per-patch SHA-256) gated on git's clean/dirty state. A reconfigure + over a patched tree is now a no-op; a genuine mismatch fails with an accurate message instead of a + misleading "does not apply cleanly". Verified against the real build tree and a purpose-built + two-patches-one-file fixture that reproduces the old failure. + + ### 2026-07-05 feature wave (PR #298) + follow-ups One-liners for the sections removed from "Open" (full detail: PR #298, CLAUDE.md, git history): diff --git a/docs/history/llama-cpp-breaking-changes.md b/docs/history/llama-cpp-breaking-changes.md index 718a72d20..d939c8483 100644 --- a/docs/history/llama-cpp-breaking-changes.md +++ b/docs/history/llama-cpp-breaking-changes.md @@ -619,3 +619,67 @@ Used during `llama.cpp` version bumps: when upgrading, scan this file from the r | b10454–b10455 | `ggml/src/ggml-sycl/{ggml-sycl.cpp,opt-step.cpp,opt-step.hpp}` (**SYCL `OPT_STEP_ADAMW` / `OPT_STEP_SGD`, #25268**), `docs/ops/SYCL.csv` + `docs/ops.md` + `examples/sycl/update-ops-doc.sh` (**regenerated op-support table**) | **No project-source change.** A single commit inside the SYCL backend, so only the `sycl-{fp16,fp32}-linux-x86-64` / `sycl-windows-x86-64` classifiers' compiled sources change. Step forced above the 100 KiB threshold (3.2 MiB) — but ~3.2 MiB of that is the regenerated `docs/ops/SYCL.csv` alone (20 646 lines rewritten); the reviewable code is ~150 lines, and upstream has no tag between b10454 and b10455 anyway. | | b10455–b10456 | `ggml/src/ggml-sycl/cpy.cpp` (**thread/block count fixed in the quantized cpy kernel launches, #27160**) | **No project-source change (final step).** A single one-commit fix inside the SYCL backend; no shared header touched, so only the `sycl-{fp16,fp32}-linux-x86-64` / `sycl-windows-x86-64` classifiers' compiled sources change. | | b10423–b10456 | upstream verification (sandbox, final target) | All **6** patches re-verified against a clean b10456 checkout: sequential `git apply` (filename order, mirroring the `PATCH_COMMAND` applier) succeeded at **every** intermediate tag of the walk, and the fail-loud `PATCH_COMMAND` ran clean on a fresh `cmake -B build` at the target — no patch needed refreshing across the whole range. **Full local verification (mandatory for the final target):** fresh configure (resolved `ggml commit f275595dd` = b10456 HEAD; all six patch markers confirmed present in the fetched tree) + full `cmake --build` (`libjllama.so` + `jllama_test` compile and link, `-O3`, no undefined references) + `ctest` **485/485 passing**. **API surface:** of the 22 upstream headers project source `#include`s directly, only three changed across the whole range — `tools/server/server-queue.h` (breaking: `on_new_task`'s callback type became `std::function`, plus the new `yield_to_queue()`; inert here, upstream's own `server-context.cpp` is the sole registrant), `ggml/include/ggml.h` (`ggml_ssm_scan()` gained a trailing `int64_t K`; called only inside llama.cpp's own TUs) and `tools/mtmd/mtmd-helper-common.h` (`size_t` widening, no signature change). `mtmd.h`, `mtmd-helper.h`, `common.h`, `chat.h`, `llama.h`, `arg.h` and every `tools/server/*.h` the project includes are byte-identical b10423→b10456; no files were added or removed under `tools/server`/`tools/mtmd`/`vendor` and no upstream `CMakeLists.txt` changed, so the b10154 `server-mcp.cpp` missing-`target_sources` failure class does not recur. Walk of 8 steps (b10423→b10430→b10436→b10441→b10447→b10448→b10454→b10455→b10456) on branch `claude/update-b10456-8a7j58`. Two steps exceeded the 100 KiB chunking threshold unavoidably — upstream has **no** intermediate tags there (b10449–b10453 and b10456 aside, nothing sits between b10447/b10448 or b10454/b10455) — and the 3.2 MiB b10454→b10455 figure is almost entirely the regenerated `docs/ops/SYCL.csv` (20 646 lines rewritten), leaving ~150 lines of reviewable code. | +| b10456–b10470 | `tools/server/server-tools.cpp` (**built-in `get_datetime` tool removed, −57 lines, plus a "keep this array minimal" contributor note**) + `common/arg.cpp` & `tools/server/README.md` (**matching one-line `--tools` help-text edit**), `common/chat-diff-analyzer.cpp` (**additive Bailing-V3 template-patch lambda**), `common/speculative.cpp` (**`dflash.sample_from_anchor` GGUF knob in the file-local DFlash/DSpark impl**), `tools/mtmd/mtmd-image.cpp` (**two `GGML_ASSERT` bounds checks in the granite preprocessor**), `CMakeLists.txt` + `ggml/CMakeLists.txt` (**`*_VERSION_PATCH` 0→1**) | **No project-source change.** The only change in a file the project compiles is `server-tools.cpp`, but both the deleted `struct server_tool_get_datetime` and `build_tools()` are file-internal (declared in no header) and `server-tools.cpp` is deliberately not linked into `jllama_test` — greps for `get_datetime`/`server_tool`/`build_tools`/`find_tool`/`"/tools"` over `src/main/cpp` + `src/test/cpp` + the Java tree return zero hits; the sole effect is that NativeServer full mode's `/tools` advertises one fewer built-in (attach mode already 403s that route via patch 0007). No public header changed: `common/speculative.h`, `common/chat.h`, `tools/mtmd/mtmd{,-helper}.h` and every `tools/server/*.h` are untouched, so the speculative/chat-analyzer/mtmd edits stay inside upstream-compiled TUs. **Patch context intact:** the `arg.cpp` edit sits at line ~3362, >2000 lines from patch `0001`'s hunks (`@@ -1201` / `@@ -1242`), and `server.cpp`/`server-context.cpp`/`server-models.cpp` are not in the range at all. **Build wiring:** the `tools/server/` **file-name** set is unchanged b10456→b10470 (`git ls-tree --name-only` diff empty; contents did change, see the left column) and `vendor/`+`include/` are untouched, so `target_sources` needs no edit and the b10154 `server-mcp.cpp` missing-source link-failure class does not recur. | +| b10456–b10470 | upstream verification (sandbox) | All **6** patches re-verified against a clean b10470 checkout: sequential `git apply --check` + `git apply` in filename order (mirroring the fail-loud `PATCH_COMMAND` applier) succeeded with zero fuzz. **Compile-verified per step:** a fresh `cmake -B build` against the patched b10470 tree configured clean, and all ten project translation units (`jllama.cpp`, `native_server.cpp`, `tts_engine.cpp`, `train_engine.cpp` + the six `src/test/cpp/test_*.cpp`) compiled with no diagnostics. Full link + `ctest` deferred to the b10618 target. | +| b10470–b10472 | `CMakeLists.txt` + **NEW** `vendor/hash/**` (`CMakeLists.txt`, `rotate-bits/`, `sha1/`, `sha256/`, `xxhash/`) (**#27262: `examples/gguf-hash/deps/` promoted to a first-class `vendor-hash` static library, added to the root `CMakeLists.txt` *unconditionally* — before the `LLAMA_BUILD_COMMON` gate — because "mtmd needs this even when common is not built"**), `ggml/src/ggml-cuda/**` (**#27083, UMA override skipped for HIP**) | **No project-source and no project-CMake change.** `FetchContent_MakeAvailable(llama.cpp)` processes upstream's root `CMakeLists.txt`, so the new `vendor-hash` target is created before the project's own `add_subdirectory(${llama.cpp_SOURCE_DIR}/tools/mtmd)` — including on Android, where the project forces `LLAMA_BUILD_TOOLS OFF`/`LLAMA_BUILD_COMMON` off. Nothing links it yet in this step (mtmd picks it up at b10481). No header, no `tools/server` file and none of the five **core** patched files (`common/arg.cpp`, `server-context.{cpp,h}`, `server.cpp`, `server-models.cpp`) changed, so all six patches keep byte-identical context. | +| b10470–b10472 | upstream verification (sandbox) | All **6** patches re-verified against a clean b10472 checkout (sequential `git apply`, filename order, zero fuzz); fresh configure clean and all ten project + C++-test translation units compiled with no diagnostics. Full link + `ctest` deferred to the b10618 target. | +| b10472–b10481 | `tools/mtmd/{CMakeLists.txt,mtmd.h,mtmd.cpp,mtmd-helper.{h,cpp},mtmd-image.{h,cpp}}` (**#27274: bitmap IDs switch from a private FNV-1a decimal hash to `hash_sha256_hex()` "to prevent cache poisoning"; mtmd now links `vendor-hash`. Additive public API `mtmd_input_chunk_get_placeholder()`, `mtmd_input_chunk_save()` refactored around a shared impl with an unchanged C signature. #27246: LFM2 thumbnail skipped for non-tiled images**), `tools/server/server-common.{h,cpp}` + `tools/server/server-context.cpp` (**#27278: additive `server_tokens::push_back_placeholder()`; already-encoded media chunks are cached as metadata-only placeholders**), `tools/server/server-tools.{h,cpp}` + `tools/server/server.cpp` (**#27271: `server_tool::type()` default renamed `"builtin"` → `"server"`; one help-string line**), `vendor/hash/{CMakeLists.txt,hash.{h,cpp},sha1/**}` (**C++ `hash_sha256_hex()` wrapper; sha1 namespaced `vendor_hash`**), `CMakeLists.txt`, `tools/server/README{,-dev}.md` | **No project-source change.** The mtmd hash switch is invisible here: the project's only bitmap call site is `src/main/cpp/tts_engine.cpp:100` (`mtmd_helper_bitmap_init_from_file`, the TTS speaker-reference clip) — signature unchanged and the ID is never read back (`grep -rn "bitmap" src/main/cpp` shows no ID use). `mtmd_input_chunk_*` and `push_back_placeholder` are additive: `push_back(const mtmd_input_chunk *)` and `push_back(server_tokens &)` both survive, so every TU including `server-common.h` (`utils.hpp`, `jllama.cpp`, `native_server.cpp` + three test files) compiles unchanged, and the project's `server_tokens` tests all construct with `has_mtmd=false`. `server_tool::type()`'s renamed default is a `GET /tools` JSON field upstream marks "do NOT use in a downstream application" — zero hits for `"builtin"` in the project's C++ **and** Java trees, and no C++ test asserts on `server_tool::to_json()`. The vendored sha1 being namespaced `vendor_hash` matters for one project-specific reason: `llama/CMakeLists.txt` forces `LLAMA_BUILD_BORINGSSL ON` on Windows, so BoringSSL and `vendor-hash` land in the same `jllama.dll` — the namespace prevents a `SHA1*` symbol clash, and `vendor-hash` is linked `PRIVATE` into mtmd so its include dir never shadows anything. **Patch context intact:** the two one-line edits in patched files sit far outside every hunk (`server.cpp:346` between patch 0007's `@@ -258,47` and `@@ -556,3`; `server-context.cpp:3416` ~2 260 lines from patch 0002's `@@ -1152,8`). **Build wiring:** no `tools/server` source added/removed/renamed, so `llama/CMakeLists.txt`'s explicit `target_sources` list still matches upstream. | +| b10472–b10481 | upstream verification (sandbox) | All **6** patches re-verified against a clean b10481 checkout (sequential `git apply`, filename order, zero fuzz); fresh configure clean and all ten project + C++-test translation units compiled with no diagnostics. Full link + `ctest` deferred to the b10618 target. | +| b10481–b10488 | `CMakeLists.txt` + `common/CMakeLists.txt` + **NEW** `vendor/CMakeLists.txt` and `vendor/{cpp-httplib,hash,miniaudio,nlohmann,sheredom,stb}/CMakeLists.txt` (**#27304 "build : fix xcframework + cmake clean-up": one `add_subdirectory(vendor)`, per-vendor `vendor::` aliases, header-only deps become INTERFACE targets exporting the `vendor/` root; `common` drops `PUBLIC ../vendor` in favour of `PUBLIC vendor::nlohmann vendor::sheredom`; `LLAMA_VERSION_PATCH` 1→2**), `tools/mtmd/{CMakeLists.txt,mtmd-helper.cpp}` (**mtmd links `vendor::{hash,miniaudio,stb,sheredom}` instead of raw `../..`/`../../vendor` include dirs; `"hash.h"` → `"hash/hash.h"`**), `tools/mtmd/{mtmd-image.{h,cpp},clip-impl.h}` (**#27057: LFM2 tiling threshold reworked into a new `should_tile()`; `#ifndef DIRECTORY_SEPARATOR` guard**) | **No project-source and no project-CMake change** — but this is the range with the one real build-wiring risk, so it was checked rather than assumed. The project compiles `tools/server/server-http.cpp` (whose `#include ` needs the `vendor/` root on the include path) and `vendor/cpp-httplib/httplib.cpp` into `jllama` itself, adds no vendor include directory of its own, and rode on `llama-common`'s `PUBLIC ../vendor` — which b10488 deletes. It still resolves because `vendor::nlohmann` / `vendor::sheredom` are linked **PUBLIC** into `llama-common` and each declares `target_include_directories( INTERFACE ..)` = the `vendor/` root, which CMake propagates transitively to `jllama` (confirmed by the compile of `server-http.cpp` and `httplib.cpp` in this step's verification build). `add_subdirectory(vendor)` is unconditional, so the `vendor::*` ALIAS targets exist before the project's own `add_subdirectory(tools/mtmd)`, Android included. The new bare target names (`nlohmann`, `sheredom`, `miniaudio`, `stb`) do not clash with the project's FetchContent nlohmann/json, whose target is `nlohmann_json`; the vendored copy stays 3.12.0, the same version. mtmd's PUBLIC surface and every `tools/server/*` header are unchanged. **Patches:** none of `common/arg.{h,cpp}`, `tools/server/server.cpp`, `server-context.{cpp,h}`, `server-models.cpp` changed in this range — zero context shift. *Optional hardening, deliberately NOT done in this bump:* adding `${llama.cpp_SOURCE_DIR}/vendor` to `jllama`'s own `target_include_directories` would make the httplib include explicit instead of transitive, and immune to a future upstream `PUBLIC`→`PRIVATE` flip on `llama-common`. | +| b10481–b10488 | upstream verification (sandbox) | All **6** patches re-verified against a clean b10488 checkout (sequential `git apply`, filename order, zero fuzz); fresh configure clean and all ten project + C++-test translation units compiled with no diagnostics. Because this is the range that deletes `llama-common`'s `PUBLIC ../vendor` include directory, the two upstream TUs the project compiles itself that depend on it — `tools/server/server-http.cpp` (`#include `) and `vendor/cpp-httplib/httplib.cpp` — were additionally compiled against the patched b10488 tree, both clean: the include still resolves transitively through the PUBLIC `vendor::nlohmann` / `vendor::sheredom` INTERFACE targets. Full link + `ctest` deferred to the b10618 target. | +| b10488–b10499 | `common/common.cpp` (**#27138: threadpool sharing when only `n_threads` differs — new file-static `can_share_threadpool()`; a `-t`/`-tb` mismatch yields ONE unpaused pool sized `max(n, nb)` instead of two**), `tools/server/server-models.cpp` (**#27347: `CMD_CHILD_TO_ROUTER_STATE` lines demoted `LOG` → `LOG_DBG`, other child output forwarded in a new `else` branch**), `tools/mtmd/models/deepseekocr.cpp` (**#26727: DeepSeek-OCR SAM convolutions re-expressed via a file-static `conv_2d_f32()` = `ggml_im2col` + `ggml_mul_mat` keeping the im2col in F32**), plus out-of-scope `tools/ui/**`, `ggml/src/ggml-{opencl,sycl}/**`, `ggml/include/ggml-rpc.h`, `src/llama-model-loader.h` | **No project-source change.** All three in-scope edits are behavioural/internal with no API surface moved: `common/common.h` is byte-identical across the range (so `struct common_threadpools` and every signature are unchanged) and no project source references `threadpool`/`ggml_threadpool_params_match`; the `server-models.cpp` edit is pure logging (`RouterModeIntegrationTest` drives the typed `RouterClient` over HTTP, not the log); the deepseekocr helper is `static` and compiles against the unchanged 11-parameter `ggml_im2col` declaration, and neither `mtmd.h` nor `mtmd-helper.h` changed. **Patches:** of the five **core** patched files only `server-models.cpp` changed, ~810 lines below patch `0008`'s single `@@ -215 @@` hunk — no context shift. Patch `0001` also touches 37 files outside that core set, and one of them *did* change here: `tools/perplexity/perplexity.cpp` lost a blank line at 2026, two lines from `0001`'s `@@ -2018,7` hunk — the closest call of the whole walk. It still applies (verified by really applying the set at both endpoints) and is applied-but-not-compiled here (`LLAMA_BUILD_TOOLS` is OFF under FetchContent). **Build wiring:** no `CMakeLists.txt` changed except upstream's own `tests/`, and no file was added/removed/renamed under `tools/server`, `tools/mtmd`, `vendor` or `include`. The three headers that changed anywhere (`ggml-rpc.h`, `ggml-sycl/fwht.hpp`, `src/llama-model-loader.h`) are all outside the project's include dependency graph. | +| b10488–b10499 | upstream verification (sandbox) | All **6** patches re-verified against a clean b10499 checkout (sequential `git apply`, filename order, zero fuzz); fresh configure clean and all ten project + C++-test translation units compiled with no diagnostics. Full link + `ctest` deferred to the b10618 target. | +| b10499–b10507 | `common/arg.{h,cpp}` + `common/download.{h,cpp}` + `tools/server/server-models.{h,cpp}` (**#27346 router preset option `dedup-cache-models`** — new `COMMON_ARG_PRESET_DEDUP_CACHE_MODELS` macro, new `common_download_resolve_path()`, new `server_model_meta::hidden` + a skip in the `GET /models` handler), `tools/mtmd/{mtmd.h,mtmd.cpp,mtmd-helper.cpp,CMakeLists.txt}` + **NEW** `tools/mtmd/mtmd-internal.h` (**#27348 `mtmd_bitmap_set_mergeable()`** — Qwen-VL temporal frame merging becomes opt-in: `can_merge_with()` now also requires `mergeable && other.mergeable`, and only `mtmd_helper_video`'s frame reader sets it), `ggml/src/ggml-{webgpu,vulkan,metal}/**`, `gguf-py/**`, `tests/**` | **No project-source change.** Both `common/download.h` and `tools/mtmd/mtmd.h` changes are *purely additive* — no existing declaration moved — and grepping `src/main/cpp` + `src/test/cpp` for `common_download*`, `COMMON_ARG_PRESET*`, `common_preset`, `mtmd_bitmap_*`, `mtmd_input_part`, `mtmd_group_mergeable_bitmaps`, `server_model_meta`, `get_all_meta` returns **zero** hits (the only `server-models` mention is a comment in `native_server.cpp:188`). `server-models.cpp` is compiled into both `jllama` and `jllama_test`, and its new `common_download_resolve_path()` call resolves from the `llama-common` both already link — no CMake change. The new `mtmd-internal.h` is added only to `add_library(mtmd …)`'s **header** list (no new `.cpp`), so it cannot repeat the b10154 `server-mcp.cpp` missing-`target_sources` link failure. Two behaviour notes, neither actionable: the `/models` JSON *shape* is unchanged (hidden entries are omitted, never serialized) and nothing is hidden unless a preset opts in with `dedup-cache-models`, which the project's own code never writes — though `NativeServer` forwards raw llama-server argv verbatim by design, so a *caller* can enable it, and `RouterClient.awaitModelLoaded` then reports a hidden-but-still-loadable model as unknown (recorded in `TODO.md`); and because upstream's server builds bitmaps with `mtmd_helper_bitmap_init_from_buf` without setting `mergeable`, two consecutive same-size images in one request are no longer temporal-merged on Qwen-VL models — upstream's deliberate fix, invisible to SmolVLM (`n_merge_frames == 1`) and to the TTS speaker-reference clip (an audio bitmap, excluded by the pre-existing `!is_audio` guard). **Patch context:** patch `0008`'s `server-models.cpp` hunk keeps the identical offset across the range; patch `0001`'s `common/arg.h` hunk shifts by exactly **+1 line** (the reformatted preset-macro block) with unchanged context and still applies; `server.cpp` and `server-context.{cpp,h}` are untouched. | +| b10499–b10507 | upstream verification (sandbox) | All **6** patches re-verified against a clean b10507 checkout (sequential `git apply`, filename order, zero fuzz); fresh configure clean and all ten project + C++-test translation units compiled with no diagnostics. Full link + `ctest` deferred to the b10618 target. | +| b10507–b10509 | `ggml/include/ggml.h` + `ggml/src/**` + `ggml/src/ggml-metal/**` (**#27120: new `ggml_rope_set_offset()` and its Metal support**), `tools/ui/**` (**#27365: persisted settings read before the API-key probe**) | **No project-source change.** Nothing under `common/`, `tools/server/`, `tools/mtmd/`, `include/`, `vendor/` or the root `CMakeLists.txt` changed at all in this range, so none of the five **core** patched files (`common/arg.cpp`, `server-context.{cpp,h}`, `server.cpp`, `server-models.cpp`) moved and no header in the project's include dependency graph was touched. `ggml_rope_set_offset` is purely additive to `ggml.h` (a header CLAUDE.md's review list marks "safe to skip" — the project never calls a `ggml_rope_*` primitive; `grep -rn "ggml_rope" src/main/cpp src/test/cpp` is empty), and `tools/ui` is the Svelte WebUI, which CI rebuilds from the pinned `GIT_TAG` and which therefore needs no per-bump source review. | +| b10507–b10509 | upstream verification (sandbox) | All **6** patches re-verified against a clean b10509 checkout (sequential `git apply`, filename order, zero fuzz); fresh configure clean and all ten project + C++-test translation units compiled with no diagnostics. Full link + `ctest` deferred to the b10618 target. This is the **last** tag at which patch `0007` applies unmodified — see the next row. | +| b10509–b10519 | `tools/server/server-task.{h,cpp}` + `tools/server/server-context.{h,cpp}` + `tools/server/server-queue.{h,cpp}` (**#27376 "server: refactor sleep handling, allow access /metrics during sleep"**), `tools/server/server.cpp` + `tools/server/server-http.cpp` (**#26347: `/models` + `/v1/models` are no longer API-key-exempt**), `common/common.cpp` (**#27337: revert of the b10499 threadpool sharing when `n_threads` differ**), `ggml/include/ggml.h`, `tools/ui/**`, `src/models/granite-swa.cpp` | **Project source change required (two), plus a patch refresh.** (1) #27376 split `server_task_result_metrics` in two: `n_idle_slots` + `slots_data` moved out into a **new `server_task_result_slots`**, produced by a **new `SERVER_TASK_TYPE_SLOT_GET`** task, and `server_task_result_metrics::to_json()` now just returns `json{}` — the *default-constructed* value, i.e. JSON **null**, not an empty object (`/metrics` renders Prometheus text via the byte-identical `to_metrics()`). Two project impacts, one silent and one hard: `jllama.cpp`'s `handleSlotAction` LIST arm posted `SERVER_TASK_TYPE_METRICS`, which still **compiles** at b10519 but would return `{}` instead of the slot array to `LlamaModel.getMetrics()` — re-pointed at `SERVER_TASK_TYPE_SLOT_GET` (upstream's `/slots` handler builds exactly that task, with no extra fields); and `test_server.cpp` was a **hard compile error** (`make_metrics()` set the removed `n_idle_slots`, `ToJson_ReturnsSlotsArrayVerbatim` set the removed `slots_data`) — the slots assertions moved to `server_task_result_slots` and two tests were added (`ServerTaskResultSlots.ToJson_EmptyByDefault`, `ServerTaskResultMetrics.ToJson_UnusedAndEmpty`), 485 → 487. `to_metrics()` is byte-identical and never read `n_idle_slots`, so every Prometheus assertion survived unchanged. (2) **Patch `0007` stopped applying here** — #26347 deleted the trailing `// public endpoint (no API key check)` comments on the two `/models` route registrations, which sit inside `0007`'s `@@ -258,47 +310,7 @@` removal block (`git apply` → "patch does not apply", `server.cpp:258`); refreshed by dropping that comment from those four lines (2 on the `-` side, 2 on the extracted `llama_server_register_common_routes()` `+` side, where it had also become factually wrong). `0001`/`0002`/`0003`/`0006`/`0008` applied unchanged. No CMake wiring change: no `tools/server` or `vendor` file was added, removed or renamed. **Behavioural note for consumers:** a `NativeServer` started with `--api-key` now returns 401 on `/models` and `/v1/models`; the project's own `RouterModeIntegrationTest` runs without an API key, so CI is unaffected. The new `server_routes` sleep-cache callback registers in the constructor and is inert **by default** — `sleep_idle_seconds` defaults to `-1`. It is **not** true that nothing in the project sets it: `ModelParameters.setSleepIdleSeconds(int)` is public API, so a caller can enable the sleep state. That matters because b10618 also stops `SERVER_TASK_TYPE_METRICS` from resetting the idle timer (`task_resets_idle_timer` in `server-queue.cpp`), which widens the window in which a task posted right before sleep is never processed — see the `post_and_wait` note in `jllama.cpp`. | +| b10509–b10519 | upstream verification (sandbox) | Patch `0007` **refreshed** (see above); all **6** patches then re-verified against a clean b10519 checkout — sequential `git apply --check` + `git apply` in filename order, zero fuzz — and the refreshed `0007` additionally re-checked to apply cleanly at **every** later endpoint of this walk up to b10618, so the refresh is a one-time fix, not a per-chunk one. Fresh configure clean and all ten project + C++-test translation units compiled with no diagnostics against the patched b10519 tree. Full link + `ctest` deferred to the b10618 target. | +| b10519–b10532 | `common/speculative.cpp` (**#27404: avoid binding a reference to a null pointer**), `ggml/**` (backend-split scheduler race #26040, CUDA cuBLAS workspace #26574, Metal FA dequant #27390, Vulkan/OpenCL/Hexagon kernels), `src/llama-graph.cpp` (**#27392: V built as a view of K in `k_iswa build_attn`**), `convert_hf_to_gguf.py` | **No project-source change.** The only in-scope file is `common/speculative.cpp`, and the fix is inside an upstream-compiled TU — `common/speculative.h` (priority 3 on the CLAUDE.md review list) has a zero-line diff across the range, so nothing the project compiles against moved, and `grep -rn "common_speculative" src/main/cpp src/test/cpp` is empty. Nothing under `tools/server/`, `tools/mtmd/`, `include/`, `vendor/` or any `CMakeLists.txt` changed, so no patch context shifted and `llama/CMakeLists.txt`'s `target_sources` list still matches upstream. The remainder is ggml backends and the llama graph builder, both entirely upstream-owned. | +| b10519–b10532 | upstream verification (sandbox) | All **6** patches (with the b10519-refreshed `0007`) re-verified against a clean b10532 checkout — sequential `git apply`, filename order, zero fuzz. Fresh configure clean and all ten project + C++-test translation units compiled with no diagnostics. Full link + `ctest` deferred to the b10618 target. | +| b10532–b10541 | `common/common.h` + `common/arg.cpp` + `tools/mtmd/{mtmd.h,mtmd.cpp,clip.h,clip.cpp}` + `tools/server/server-context.cpp` (**#23255 `--mmproj-device`: new `common_params::mmproj_device` (`ggml_backend_dev_t`) and a new `device` member at position 2 in `mtmd_context_params`/`clip_context_params`; the `MTMD_BACKEND_DEVICE` env lookup moves out of `clip.cpp` into `arg.cpp`**), `common/json-schema-to-grammar.cpp` (**#26939: unsupported regex patterns degrade to accept-any-string with a warning instead of throwing**), `tools/server/{server-models.h,server-models.cpp,server.cpp}` (**#27424: load-on-startup deferred into a new public `server_models::load_startup_models()` called from `llama_server()`'s router branch; reload no longer autoloads. #27416: `is_router_server` also excludes `--docker-repo`**), `common/speculative.cpp` (**`!dp.drafting` guard**) | **No project-source change.** The mid-struct `mtmd_context_params::device` insertion is the one positional-aggregate-init hazard in this range and the project does not trip it: `src/main/cpp/tts_engine.cpp` builds its params via `mtmd_context_params_default()` plus a named `.use_gpu` assignment, so `device` defaults to `nullptr` = the previous auto-GPU behaviour; `ggml_backend_dev_t` is already visible through `mtmd.h` → `llama.h` → `ggml-backend.h`, and every `common_params` in the project is default-constructed. **Left unwired during the bump, wired afterwards:** the new `-mmdev/--mmproj-device` flag deliberately got no `ModelParameters` setter *in the bump commits* — that is new public Java API and the owner's call, not a version bump's. It was added later on the same branch (`ModelParameters.setMmprojDevice(String)` plus `-mmdev`/`--mmproj-device` in `OpenAiServerCli`), once the owner asked for it; before that it was reachable only through `NativeServer`, which forwards raw llama-server argv. Note the flag lands at **b10541**, not at the b10618 target — README/CHANGELOG/javadoc were corrected to say so. `common/json-schema-to-grammar.h` is unchanged, so #26939 is semantic only — it does reach a public Java API (`jllama.cpp`'s `jsonSchemaToGrammarBytes` → `LlamaModel.jsonSchemaToGrammar`), where a schema with an unanchored `pattern` or a `\d`/`\w`/`\s` escape now yields a permissive `string` grammar plus a stderr warning instead of throwing `LlamaException`; no test moves, because `LlamaModelTest#testJsonSchemaToGrammar` asserts only `PRIMITIVE_RULES` output from three pattern-less string properties (zero `"pattern"` hits across `src/test/{java,cpp}`). `server_models` / `load_startup_models` / `docker_repo` are zero-hit greps in the project's C++ and Java. **Patch context:** `server.cpp` gains 13 lines (the `load_startup_models()` block) **below** patch `0007`'s route-table hunk, so only that patch's *last* hunk offset moves (10 → 23) — it still applies, verified by real `git apply` at both endpoints. **Build wiring:** no file added/removed/renamed under `tools/server`/`tools/mtmd`/`vendor`. | +| b10532–b10541 | upstream verification (sandbox) | All **6** patches re-verified against a clean b10541 checkout — sequential `git apply`, filename order, zero fuzz. Fresh configure clean and all ten project + C++-test translation units compiled with no diagnostics. Full link + `ctest` deferred to the b10618 target. | +| b10541–b10545 | `ggml/src/ggml-metal/**` (**#27450: clamp the K extent in the tensor-API mat-mat kernel when K is not a multiple of 32**), `ggml/src/ggml-opencl/**` (**#26476 q6_K flat mul_mat on Adreno A6x/A7x with older E031 compilers; #27339 norm local size**), `tools/ui/**` (**#27240: stores split refactor**) | **No project-source change.** Nothing under `common/`, `tools/server/`, `tools/mtmd/`, `include/`, `vendor/` or any `CMakeLists.txt` changed — the whole range is ggml backend kernels plus the Svelte WebUI (which CI rebuilds from the pinned `GIT_TAG`, so it needs no per-bump source review). No patch context moved and `llama/CMakeLists.txt`'s `target_sources` list still matches upstream. The step's raw `git diff` is large (≈978 KiB) only because of the WebUI refactor; the reviewable code outside `tools/ui` is ≈17 KiB. | +| b10541–b10545 | upstream verification (sandbox) | All **6** patches re-verified against a clean b10545 checkout — sequential `git apply`, filename order, zero fuzz. Fresh configure clean and all ten project + C++-test translation units compiled with no diagnostics. Full link + `ctest` deferred to the b10618 target. | +| b10545–b10549 | `ggml/**` (**#27345: `ggml_rope_set_offset` support added to the OpenCL / SYCL / WebGPU / Hexagon backends; #26993: tensor split enabled for LFM2/LFM2MOE**), `docs/**`, `.github/**` (**#27414: cmake-package check moved to a shell script**) | **No project-source change.** Nothing under `common/`, `tools/server/`, `tools/mtmd/`, `include/`, `vendor/` or any project-consumed `CMakeLists.txt` changed. `ggml_rope_set_offset` was already added to `ggml.h` at b10509 and remains unreferenced by the project (`grep -rn "ggml_rope" src/main/cpp src/test/cpp` is empty); this range only implements it in the non-CPU backends. No patch context moved. | +| b10545–b10549 | upstream verification (sandbox) | All **6** patches re-verified against a clean b10549 checkout — sequential `git apply`, filename order, zero fuzz. Fresh configure clean and all ten project + C++-test translation units compiled with no diagnostics. Full link + `ctest` deferred to the b10618 target. | +| b10549–b10566 | `CMakeLists.txt` (**#27498: llama.cpp semver bumped to 0.2.0; ggml/1597 bumps ggml to 0.21.0**), `ggml/src/ggml-{sycl,opencl,cpu}/**` (SYCL Q2_K/Q5_K ESIMD kernels reverted then re-landed, KleidiAI SME2 F32 GEMV, Adreno A7X lm_head workaround), `scripts/release.sh` + `.github/**` (release tooling), `tools/ui/**` (**#27241: settings-navigation cleanup**), `tests/**` | **No project-source change.** The only in-scope file is the root `CMakeLists.txt`, and only its version numbers moved. That is invisible here: `llama/CMakeLists.txt` already substitutes `"0"` placeholders for `LLAMA_VERSION_BASE` / `LLAMA_VERSION_MAJOR` (upstream `set()`s them in FetchContent's child scope, out of the project's reach) and uses them solely as mtmd's `VERSION`/`SOVERSION`, which are no-ops for a static library under `BUILD_SHARED_LIBS OFF`. The Java-visible pin is `LlamaCppVersion.LLAMA_CPP_VERSION` (the `b` tag), not the semver. Nothing under `common/`, `tools/server/`, `tools/mtmd/`, `include/` or `vendor/` changed, so no patch context moved and the `target_sources` list still matches upstream. | +| b10549–b10566 | upstream verification (sandbox) | All **6** patches re-verified against a clean b10566 checkout — sequential `git apply`, filename order, zero fuzz. Fresh configure clean and all ten project + C++-test translation units compiled with no diagnostics. Full link + `ctest` deferred to the b10618 target. | +| b10566–b10568 | `src/models/**` (**#27382: the model graph builders adopt `ggml_rope_set_offset()`**), `.github/**` (**#27503: `ccache-clear` runs last in the release jobs**) | **No project-source change.** Zero lines changed under `common/`, `tools/server/`, `tools/mtmd/`, `include/`, `vendor/` or any `CMakeLists.txt`. `src/models/**` is llama.cpp's internal graph-builder layer — compiled into the `llama` static library the project links, but it exposes no header the project includes, and `grep -rn "ggml_rope" src/main/cpp src/test/cpp` is empty. No patch context moved. | +| b10566–b10568 | upstream verification (sandbox) | All **6** patches re-verified against a clean b10568 checkout — sequential `git apply`, filename order, zero fuzz. Fresh configure clean and all ten project + C++-test translation units compiled with no diagnostics. Full link + `ctest` deferred to the b10618 target. | +| b10568–b10569 | `src/models/dots3note.cpp` (new) + `src/llama-{model,arch,vocab}.*` + `convert_hf_to_gguf.py` + `gguf-py/**` (**#27060: dots3-note model support**) | **No project-source change.** A new model architecture, entirely inside llama.cpp's own model layer: the new TU is added to upstream's own `src/CMakeLists.txt` (which the project consumes verbatim through `FetchContent_MakeAvailable`, so there is nothing to mirror into `llama/CMakeLists.txt` — unlike the `tools/server/*.cpp` list, which the project enumerates itself). Zero lines changed under `common/`, `tools/server/`, `tools/mtmd/`, `include/` or `vendor/`; no patch context moved. | +| b10568–b10569 | upstream verification (sandbox) | All **6** patches re-verified against a clean b10569 checkout — sequential `git apply`, filename order, zero fuzz. Fresh configure clean and all ten project + C++-test translation units compiled with no diagnostics. Full link + `ctest` deferred to the b10618 target. | +| b10569–b10578 | `tools/mtmd/mtmd-helper.{h,cpp}` (**#27520: webp decoded as a single ffmpeg frame; the header change is one comment line**), `common/speculative.cpp` (**#27400: draft-MTP fixed with embeddings — draft params now reset `embedding`/`pooling_type`**), `ggml/**` (**#24575 row-level `concat`; #26431 gpt-oss MoE bias fused into the OpenCL epilogue; #27490 SYCL Q2_K kernels re-landed**), `README.md` badges, `docs/**` | **No project-source change.** `mtmd-helper.h` changed by exactly one comment line, so `mtmd_helper::gen_audio`, `mtmd_helper_gen_audio_inp` and `mtmd_helper_bitmap_init_from_file` — the surface `src/main/cpp/tts_engine.cpp` uses directly, and the reason `mtmd-helper.h` is on the CLAUDE.md priority review list — are untouched. The new webp path is reachable through upstream's own `server-common.cpp` bitmap loader and is a functional gain requiring `ffmpeg`/`ffprobe` on `PATH` **at runtime only** (no new build dependency). One cosmetic upstream wart: the new `is_webp_file()` sits outside the `MTMD_VIDEO` guard while its only caller is inside it, so an `MTMD_VIDEO=OFF` build emits `-Wunused-function`; harmless here — it is compiled in upstream's own `mtmd` target and `LLAMA_FATAL_WARNINGS` defaults OFF and is never set by this project. The speculative fix is behaviour-only: `common/speculative.h` is unchanged and the only callers are upstream's `server-context.cpp`. **Patches:** none of the five **core** patched files (`common/arg.cpp`, `server-context.{cpp,h}`, `server.cpp`, `server-models.cpp`) changed — zero context shift. | +| b10569–b10578 | upstream verification (sandbox) | All **6** patches re-verified against a clean b10578 checkout — sequential `git apply`, filename order, zero fuzz. Fresh configure clean and all ten project + C++-test translation units compiled with no diagnostics. Full link + `ctest` deferred to the b10618 target. | +| b10578–b10584 | `common/fit.{h,cpp}` + `common/common.cpp` + `common/speculative.cpp` + `tools/{fit-params,llama-bench}` (**draft/MTP context is now fitted together with the target model — new public `struct common_fit_extra_model` and a new 7th `const common_fit_extra_model * extra` parameter on `common_fit_params()`; `common_speculative_init_result` pins the draft `cparams.n_ctx = llama_n_ctx(ctx_tgt)`**), `tools/server/server-context.cpp` (**−56 lines: the hand-rolled draft-model VRAM pre-reservation block is deleted, superseded by the fitter**), `common/arg.cpp` (**`--conversation` dropped `LLAMA_EXAMPLE_CLI` from its `set_examples()`**), `tools/mtmd/{CMakeLists.txt,clip*.h,clip.cpp,models/dots3note.cpp,models/models.h,mtmd-audio.{h,cpp},mtmd.cpp}` (**new `dots3note` multimodal model: two `PROJECTOR_TYPE_DOTS3NOTE_{V,A}` values, a vision MoE FFN via a new `clip_graph::build_moe_ffn()`, and `mtmd_audio_preprocessor_dots3note`**) | **No project-source change; no patch refresh.** `common_fit_params()`'s signature break is upstream-only: `common/fit.h` is not in the project's include graph, `grep -rn "common_fit_params\|common_fit_extra_model\|fit_params\|common_get_device_memory_data" src/main/cpp src/test/cpp` returns zero hits, and the two other callers (`tools/fit-params`, `tools/llama-bench`) are not built here (`LLAMA_BUILD_TOOLS` is OFF under FetchContent; only `tools/mtmd` and an explicit `tools/server/*.cpp` list are added back). The `server-context.cpp` deletion sits directly **above** patch `0002`'s hunk, whose anchor lines (blank / `// attach a progress callback` / `{`) are byte-identical, so that hunk only *shifts* (offset −53 → −108) and still applies cleanly. `--conversation` never reached the project's parser (`jllama.cpp` parses with `LLAMA_EXAMPLE_SERVER`, in neither the old nor the new example set) and patch `0001`'s `arg.cpp` hunks are ~650 lines away. All mtmd edits are internal — **`mtmd.h` and `mtmd-helper.h` are unchanged**, so `tts_engine.cpp`'s `mtmd_helper::gen_audio` surface is untouched, and `models/dots3note.cpp` enters through upstream's own `tools/mtmd/CMakeLists.txt`. The only reachable behaviour delta is a more accurate VRAM budget when `--fit` runs together with a draft model. | +| b10578–b10584 | upstream verification (sandbox) | All **6** patches re-verified against a clean b10584 checkout — sequential `git apply`, filename order, zero fuzz. Fresh configure clean and all ten project + C++-test translation units compiled with no diagnostics. Full link + `ctest` deferred to the b10618 target. | +| b10584–b10585 | **NEW** `common/json.{h,cpp}` + `common/CMakeLists.txt`, `common/chat.h`, `common/chat-auto-parser.h`, `common/chat-peg-parser.h`, `common/json-schema-to-grammar.h`, `common/peg-parser.h`, `common/arg.cpp`, `common/{chat,download,hf-cache,jinja/*,json-schema-to-grammar,peg-parser}.cpp`, `tools/server/{server-common.h,server-common.cpp,server-chat.h,server-chat.cpp,server-context.h,server-context.cpp,server-task.h,server-task.cpp,server-schema.cpp,server-models.cpp,server-tools.cpp}` (**#27511 "common: add json.h abstraction"**) | **BREAKING — the largest project impact of the whole b10456 → b10618 walk.** Upstream introduces the pimpl wrapper class `common_json` and flips `using json = nlohmann::ordered_json` → `using json = common_json` in `server-common.h`, so **every** project TU that includes an upstream server header changes JSON type. `common_json` is a deliberately closed API: no `get_ref`, no `array_t`, no `type_name()`, a braced list in *value* position does not build an array, `at(key)` no longer implicitly converts (needs `.get()`), `get()` is limited to the specialisations in `common/json.cpp`, and errors are `common_json_error`. Project changes, in the order they were found: **(1)** `utils.hpp`'s `str_to_bytes` used `bytes.get_ref().reserve(...)` — dropped (a pure optimisation; `json::array()` + `push_back` is unchanged). **(2)** `jllama.cpp`'s `jsonSchemaToGrammarBytes` parsed with `nlohmann::ordered_json::parse` and handed the result to `json_schema_to_grammar`, which now takes `const common_json &` — there is no conversion, so this is a hard compile error; switched to `json::parse`. **(3) Two silent, compile-clean regressions**, neither of which a build could catch: `jni_helpers.hpp`'s `require_json_field_impl` declared its parameter `const nlohmann::json &`, and a `common_json` **still binds to it** — through `common_json::operator std::string()` feeding nlohmann's string-constructible converting constructor — so the presence check became a `json::type_error 302` ("type must be string, but is object") thrown out of `handleInfill`, whose two call sites sit *outside* its `try` block; the helper is now a template on the JSON type, with two new `common_json` regression tests. And `common_json_value`'s integral constructor template is `std::is_integral`-gated, which **excludes enums**, so `{"vocab_type", }` bound to the `bool` constructor and serialised as `true`/`false`; `ModelMeta.getVocabType()` reads it with Jackson's `asInt(0)`, so every non-SPM model would have reported vocab type 1. Both emit sites now `static_cast(...)`, with a `CommonJsonEnumTrap` C++ pair pinning the trap and a `LlamaModelTest` assertion that the wire value `isIntegralNumber()`. (Both were confirmed empirically with a standalone probe linked against `common/json.cpp`, not inferred.) **(4)** Fourteen C++ test literals across three files now use `json::array({...})`; the produced JSON is identical. Two distinct incompatibilities: a braced array in *value* position (`test_json_helpers.cpp`'s `extract_embedding_prompt` input, `test_server.cpp`'s `dry_sequence_breakers` + five `samplers` cases — seven sites), and direct-initialisation `json{1, 2, 3}` in *constructor* position (`test_utils.cpp`'s seven `JsonArrayChecks` cases), which `common_json` also cannot build because its only initializer-list constructor takes `common_json_item`s, i.e. keyed object entries. **No JSON output shape changed in the project's own emissions**, so no existing assertion moved — but the same enum trap bit **upstream's own** `/models` handler, which the project compiles and serves; that was found by the post-bump review and is carried as `patches/0010` (see the last row of this file); **no CMake change** is needed (`common/json.cpp` rides the `llama-common` target both `jllama` and `jllama_test` already link, and the `nlohmann_json` link stays for `log_helpers.hpp` / `train_engine.cpp`, which keep their own alias); and **no patch context shifted** — `server.cpp` is untouched in this range, and `0001`/`0002`/`0003`/`0008` keep byte-identical anchors. Test total 487 → 491. | +| b10584–b10585 | upstream verification (sandbox) | All **6** patches re-verified against a clean b10585 checkout — sequential `git apply`, filename order, zero fuzz. Fresh configure clean and all ten project + C++-test translation units compiled with no diagnostics **after** the migration above. Full link + `ctest` deferred to the b10618 target. | +| b10585–b10590 | `common/json.{cpp,h}` (**#27575 "fix clang lto": the `common_json_value` set/map/unordered_map/vector constructors and every `common_json::get()` flip from explicit *instantiation* to explicit *specialization* — "an explicit instantiation is a weak symbol, dropped by some LTO builds (clang-cl)" — and `operator std::string()` + `value(key, const char *)` move from inline-in-`json.h` to out-of-line in `json.cpp`; the set of usable types is unchanged**), `vendor/sheredom/subprocess.h` + `scripts/sync_vendor.py` (**#27409 upstream resync, +363/−12: new `SUBPROCESS_SPAWN_VIA_FORK` / `SUBPROCESS_ADDCHDIR_IS_POSIX` macros gating a `fork()`+`execve`/`execvpe` launcher with an errno-relay pipe, new `subprocess_pipe_cloexec()` / `subprocess_fds_above_std()` POSIX helpers, and a Windows `STARTUPINFOEX` + `PROC_THREAD_ATTRIBUTE_HANDLE_LIST` handle-inheritance list**), `tools/mtmd/{clip.cpp,clip-graph.h,models/gemma4v.cpp,models/minimax-m3.cpp}` (**#27521: 2D RoPE rewritten from view/rope/`ggml_concat` onto two in-place `ggml_rope_ext` calls + `ggml_rope_set_offset`**) | **No project-source and no project-CMake change.** The `json` work *is* in the project's compile path (the alias became `common_json` at b10585) and the project calls both members that moved out-of-line, but `common/json.cpp` belongs to the `llama-common` target that `jllama` and `jllama_test` already link — nothing to wire, and no `to_json()` shape moved. The project enables no LTO, so the bug being fixed never bit it; the change is robustness for the clang-cl Windows-arm64 job. **`subprocess.h` re-verified against the dropped patch `0009`:** `SUBPROCESS_HAVE_CWD` and its `#elif !SUBPROCESS_HAVE_CWD → posix_error = ENOSYS;` fallback both survive, and `SUBPROCESS_SPAWN_VIA_FORK` is 0 on glibc/bionic/macOS/Windows (only `_AIX`/`__OpenBSD__`/old NetBSD set it), so manylinux2014 (glibc 2.17) still resolves exactly as before and the new `execvpe` declaration is preprocessed away everywhere the project builds — **`0009` stays dropped.** The new always-compiled POSIX helpers need only `pipe2`/`O_CLOEXEC`/`F_DUPFD` (glibc 2.7+/bionic) and the header already includes ``/``; the Windows attribute-list path adds no new link input. mtmd is internal-only here: `build_rope_2d`'s signature is untouched (the `clip-graph.h` hunk is comment-only) and `mtmd.h`/`mtmd-helper.h`/`clip.h` are byte-identical, so `tts_engine.cpp`'s surface is unaffected. **Patches:** zero changes under `tools/server/` and none to `common/arg.{h,cpp}` — byte-identical context for all six. | +| b10585–b10590 | upstream verification (sandbox) | All **6** patches re-verified against a clean b10590 checkout — sequential `git apply`, filename order, zero fuzz. Fresh configure clean and all ten project + C++-test translation units compiled with no diagnostics. Full link + `ctest` deferred to the b10618 target. | +| b10590–b10593 | `include/llama.h` (**one comment line: a `[TAG_LLAMA_SEQ_ID_NEG]` marker added to `llama_memory_seq_rm`'s doc block**), `src/llama-*.cpp` (**#26756 DeepseekV4 multi-seq rollback fix; #27574 tensor-parallel meta tensor-split state propagation**), `tools/ui/**` (**#27263: tabbed chat-conversation navigation**) | **No project-source change.** `include/llama.h` is priority 7 on the CLAUDE.md review list, so its diff was read in full: it is a single comment line inside an existing doc block — no signature, no enum value, no struct field moved. The rest is llama.cpp's internal KV/tensor-split implementation and the Svelte WebUI. Nothing under `common/`, `tools/server/`, `tools/mtmd/`, `vendor/` or any `CMakeLists.txt` changed. One file patch `0001` touches *did* change, outside those directories: `tests/test-recurrent-state-rollback.cpp` gained 172 lines above `0001`'s hunk, shifting it ~175 lines — it still applies by offset (verified by really applying the set at this endpoint), and it is applied-but-not-compiled here (`LLAMA_BUILD_TESTS` is OFF under FetchContent). | +| b10590–b10593 | upstream verification (sandbox) | All **6** patches re-verified against a clean b10593 checkout — sequential `git apply`, filename order, zero fuzz. Fresh configure clean and all ten project + C++-test translation units compiled with no diagnostics. Full link + `ctest` deferred to the b10618 target. | +| b10593–b10599 | `tools/mtmd/{clip-model.h,clip.cpp,mtmd-image.cpp}` (**#27594 "mtmd: use pillow-accurate algo, correct resize_algo for all models": `enum resize_algo` drops `RESIZE_ALGO_BICUBIC_PILLOW` (plain `BICUBIC` now *means* the Pillow path), the four private `img_tool` statics `resize_{bilinear,bicubic,bicubic_pillow,lanczos_pillow}` and both dispatch switches collapse into one `resize_pillow(..., resize_algo algo)` (−246 net), and ~14 projector types are re-assigned**), `tools/server/server-context.cpp` (**#27600: private `slots_n_diff` member + `LLAMA_SERVER_SLOTS_N_DIFF` getenv; the hardcoded prompt-mismatch debug window `n_past−4..+6` becomes `n_past−slots_n_diff..+slots_n_diff+2`**), `common/common.cpp` (**#26692: `common_params_print_info`'s device-enumeration loop gated on `print_devices && verbosity >= LOG_LEVEL_TRACE`**), `tools/CMakeLists.txt` (**#27548: `add_subdirectory(parser)` dropped — the parser tool moved under `tests/`**) | **No project-source, no project-CMake and no patch change.** The mtmd resize rework is entirely mtmd-internal: `resize_algo`/`clip_hparams` live in `tools/mtmd/clip-model.h`, which the project never includes, and all four deleted functions were `private:` statics inside `struct img_tool`; the public `mtmd.h`/`mtmd-helper.h` are unchanged in this range. The only visible effect is that `MultimodalIntegrationTest`'s SmolVLM preprocessing becomes Pillow-exact — pixel-level, no API or JSON shape. `server-context.cpp`'s new fields are `private:` in `server_context_impl` and **`server-context.h` is unchanged**, so no `to_json()` shape moved and no C++ test assertion is affected; patch `0002`'s and `0003`'s anchors are byte-identical and resolve with a pure offset. `common_params_print_info` keeps its `bool print_devices = true` default, so `jllama.cpp`'s one-argument call still compiles; the behaviour delta is that at the default log threshold libjllama no longer prints `device_info:` (nothing in the project reads it) and no longer triggers early CUDA primary-context creation from that call site. The `tools/CMakeLists.txt` edit sits inside the `LLAMA_BUILD_TOOLS`-gated branch the project never enters. | +| b10593–b10599 | upstream verification (sandbox) | All **6** patches re-verified against a clean b10599 checkout — sequential `git apply`, filename order, zero fuzz. Fresh configure clean and all ten project + C++-test translation units compiled with no diagnostics. Full link + `ctest` deferred to the b10618 target. | +| b10599–b10612 | `tools/mtmd/mtmd-helper.cpp` (**#27596 "video: fix moov atom at the end of file": `subprocess_handle` splits `alive` into `created`+`alive`, `stop()` gains `subprocess_join()` to reap zombies plus a Windows `CloseHandle(proc.hStdInput)`, `start_feeder()` gains `pthread_sigmask(SIG_BLOCK, {SIGPIPE})` and an `#ifdef F_SETNOSIGPIPE fcntl(...)`, and the ffmpeg argv gains `-read_ahead_limit -1`; new `#include //` under `#ifdef MTMD_VIDEO`/`#ifndef _WIN32`**), `ggml/**` (**#27644 `ggml_clamp` fix; #27608 shorter virtual-device naming; #27545 WebGPU include order**), `src/**` (**#26534 MTP in GLM-4.5-Air; #26490 Deepseek 4 `-sm tensor`; #27513 mamba2 GEMM dispatch**), `convert_hf_to_gguf.py`, `tools/ui/**` | **No project-source change.** `mtmd-helper.cpp` **is** compiled into `libjllama` (the project `add_subdirectory`s `tools/mtmd` and upstream defaults `LLAMA_SUBPROCESS` **OFF** when `CMAKE_SYSTEM_NAME`/`ANDROID` says Android, which force-disables `MTMD_VIDEO` — but the dockcross cross-clang sets neither variable, as the Android section of `CLAUDE.md` documents, so `LLAMA_SUBPROCESS`/`MTMD_VIDEO` stay ON on this project's Android build too; a switch to the NDK toolchain would flip that silently), so its three new build-surface dependencies were each checked against the pinned tree rather than assumed: `subprocess_join` is declared in `vendor/sheredom/subprocess.h`, `struct subprocess_s` carries `void *hStdInput` (so the Windows `CloseHandle` compiles), `pthread_sigmask` resolves through the pre-existing `target_link_libraries(mtmd PRIVATE Threads::Threads …)`, and `F_SETNOSIGPIPE` is `#ifdef`-guarded off on Linux/Android. `mtmd.h` and `mtmd-helper.h` are unchanged in this range, so `tts_engine.cpp`'s `mtmd_helper::gen_audio` surface is untouched. Nothing under `common/`, `tools/server/`, `include/` or any project-consumed `CMakeLists.txt` changed — no patch context moved. | +| b10599–b10612 | upstream verification (sandbox) | All **6** patches re-verified against a clean b10612 checkout — sequential `git apply`, filename order, zero fuzz. Fresh configure clean and all ten project + C++-test translation units compiled with no diagnostics. Full link + `ctest` deferred to the b10618 target. | +| b10612–b10614 | `ggml/src/ggml-metal/**` + `ggml/CMakeLists.txt` (**#26561: per-op Metal source split + parallel compile — a large mechanical file split, hence the ~1 MiB raw diff**), `gguf-py/gguf/metadata.py` (**#27659: `repetition_penalty` read from `generation_config.json` at conversion time**) | **No project-source change.** Zero lines changed under `common/`, `tools/server/`, `tools/mtmd/`, `include/`, `vendor/` or any project-consumed `CMakeLists.txt`, so no patch context moved and no header in the project's include graph was touched. The Metal split affects only the `Mac/aarch64` classifier's compiled sources (upstream owns that list entirely); the `generation_config.json` change is a GGUF-conversion-time default with no C++ API surface. | +| b10612–b10614 | upstream verification (sandbox) | All **6** patches re-verified against a clean b10614 checkout — sequential `git apply`, filename order, zero fuzz. Fresh configure clean and all ten project + C++-test translation units compiled with no diagnostics. Full link + `ctest` deferred to the b10618 target. | +| b10614–b10615 | `ggml/src/ggml-metal/**` + `tools/CMakeLists.txt` (**#26570: per-device tuned `(Q, NE)` for the flash-attention vec kernels; a Metal-only `tuning` tool is added under an `if (GGML_METAL)` guard**) | **No project-source change.** The `tools/CMakeLists.txt` addition sits inside the `LLAMA_BUILD_TOOLS`-gated branch the project never enters (`LLAMA_BUILD_TOOLS` defaults to `LLAMA_STANDALONE`, i.e. OFF under FetchContent, and the project forces it OFF on Android), and is additionally Metal-only. Nothing under `common/`, `tools/server/`, `tools/mtmd/`, `include/` or `vendor/` changed — no patch context moved, no header in the project's include graph touched. | +| b10614–b10615 | upstream verification (sandbox) | All **6** patches re-verified against a clean b10615 checkout — sequential `git apply`, filename order, zero fuzz. Fresh configure clean and all ten project + C++-test translation units compiled with no diagnostics. Full link + `ctest` deferred to the b10618 target. | +| b10615–b10618 | `src/llama-grammar.cpp` (**#27591: `parse_char()` accepts `case '-':`, so a `\-` escape produced by `gbnf_escape_char_class()` inside a character class no longer throws**), `ggml/src/ggml-sycl/**` (**#27660: `tq2_0` marked unsupported**), `ggml/src/ggml-webgpu/wgsl-shaders/argsort.wgsl` (**#27538: infinity handling in ARGSORT / TOP_K**) | **No project-source change (final step).** Zero lines changed under `common/`, `tools/server/`, `tools/mtmd/`, `include/`, `vendor/` or any `CMakeLists.txt`, so no patch context moved. The grammar fix is strictly *widening* — an input that previously threw now parses — and reaches the project through both `LlamaModel.jsonSchemaToGrammar` and every grammar-constrained completion, so it is a **positive** for the agentic tool-calling target; no test pinned the old throw (`grep -rn 'gbnf_escape_char_class\|parse_char' src/main/cpp src/test/cpp` is empty). The SYCL and WebGPU edits only affect the `sycl-*` classifiers' compiled sources and a backend the project does not build. | +| b10456–b10618 | upstream verification (sandbox, final target) | **Full local verification, re-run from scratch on the shipped tree.** Fresh `cmake -B build` with the real `FetchContent` path (no source-dir override), so the fail-loud `PATCH_COMMAND` ran for real: it resolved `ggml commit eb25b7263` / `LLAMA_BUILD_NUMBER = 10618` and applied all **7** patches — verified by grepping the fetched tree for each patch's marker (`common_params_parse_main`, `params_base.load_progress_callback == nullptr`, `get_slot_prompt_similarity`, `g_llama_server_embedded`, `llama_server_attach` + `llama_server_register_common_routes`, `LLAMA_SERVER_WORKER_CMD`, `(int) meta.model_vocab_type`). The **7**, not 6, matters: the review-driven `0010` was added after the first verification pass and is the *third* patch to touch `server-context.cpp` (after `0002`/`0003`), a TU compiled into both `jllama` and `jllama_test`, so the whole configure + build + `ctest` cycle was re-run on the final tree rather than assumed to carry over (**491/491**, `mvn test` unchanged at 1405 run / 0 failures / 17 model-gated skips). The patch was subsequently renamed `0009` → `0010` to free the burned number; that is a filename-only change with the same sort position, re-checked by applying the whole set to a clean b10618 checkout. Then a full `cmake --build --config Release` (jllama + jllama_test both link — the `jllama_test` link is what proves `common_json::get()` resolves, since upstream documents un-specialised types as a *link*-time failure) and `ctest`: **491/491 C++ tests pass**. On the Java side: `mvn test-compile` clean, `mvn test` **1405 run / 0 failures / 17 skipped** (only the model-gated integration tests, no GGUF in the sandbox), and `NativeLibraryLoadSmokeTest` green — including `nativeBuildInfoMatchesPinnedVersionConstant`, the end-to-end cross-check that `LlamaCppVersion.LLAMA_CPP_VERSION` ("b10618") matches the `build-info` compiled into the freshly linked `libjllama.so`. `mvn spotless:apply` produced no changes beyond the edits themselves, the pinned **clang-format 22.1.8** reports the whole C++ tree clean, and `mvn clean javadoc:jar` is `BUILD SUCCESS`. **Per-step verification for all 25 chunks:** every intermediate tag additionally had all six patches applied by the applier's own commands and all ten project + C++-test translation units compiled against it, so no commit in this walk is a state that fails to build. **Two notes on how to read the per-chunk rows above.** (1) They say "all **6** patches" because that was the set during the walk; the 7th (`0010`) came out of the post-bump review and exists only from this final commit on. (2) Their prose reasoning about patch risk scans the directories that matter for the project's *compile* surface (`common/`, `tools/server/`, `tools/mtmd/`, `include/`, `vendor/`, `CMakeLists.txt`) and refers to "the five core patched files" — but the patch set actually spans **40** distinct files, because `0001` flips ~34 standalone `main()` call sites under `tools/*`, `examples/*` and `tests/*`. Two ranges did touch one of those and the prose does not say so on its own (b10488–b10499 → `tools/perplexity/perplexity.cpp`, b10590–b10593 → `tests/test-recurrent-state-rollback.cpp`; both now corrected in place). The *verdicts* were never derived from that directory scan: each chunk's patch verdict comes from really running `git apply` for the whole set against a clean checkout of that tag, which covers all 40 files. The reusable check is `git diff --name-only -- $(grep -h '^+++ b/' llama/patches/*.patch | sed 's|^+++ b/||' | sort -u)`. | +| b10456–b10618 | `tools/server/server-context.cpp` — **new local patch `0010-server-cast-vocab-type-for-common-json.patch`** | **Upstream regression found by the post-bump review, fixed downstream.** `get_res_model_info()` builds the `GET /models` + `GET /v1/models` payload and emits `{"vocab_type", meta.model_vocab_type}`, an **unscoped enum**. That was correct while the `json` alias was `nlohmann::ordered_json` (nlohmann serialises an enum as an integer), but `common_json_value`'s integral constructor template is `std::is_integral`-gated, which excludes enums — so from **b10585** the value binds to `common_json_value(bool)` and goes on the wire as `true`/`false`. Upstream regressed it silently in #27511 when they flipped the alias; the same trap hit the project's own two `"vocab_type"` emit sites in `jllama.cpp` (fixed in the b10584–b10585 step). It **ships**: `server-context.cpp` is compiled into `libjllama` and both routes are served by `NativeServer`, the default fat-jar `Main-Class`, in full and in attach mode. The new patch casts the value to `int` at the emit site — one line, upstream-submittable (not yet filed), applied after `0002`/`0003` (same file). Numbered `0010`, not `0009`: that number belongs to the subprocess.h patch dropped at b10280 and is documented as such under the patches table in `CLAUDE.md`, so reusing it would make that note read as if it described this patch. A mechanical sweep confirmed it is the **only** such site in the upstream server sources the project compiles: `server_context_meta` has exactly two enum-typed members, and the other (`pooling_type`) is only ever compared, never serialised. | +| b10456–b10618 | follow-up: `src/main/cpp/{jllama.cpp,json_helpers.hpp}`, `value.ServerMetrics`, `server.OpenAiCompatServer` | **Client-contract restoration, after the walk.** The b10405–b10408 and b10509–b10519 rows above record the upstream metrics refactor correctly but understate its client impact: b10408 (#26920) reduced `server_task_result_metrics::to_json()` to the bare slot array and b10519 (#27376) split the task, so `LlamaModel.getMetrics()` had been returning the slot array — not the object its three Java consumers parse — since **before** this bump range began. Re-pointing `handleSlotAction` at `SERVER_TASK_TYPE_SLOT_GET` (b10519 row) kept the array arriving but did not restore the object. Fixed by posting **both** tasks and merging them in the new pure helper `server_metrics_to_json`, so the documented payload is rebuilt in the JNI layer instead of the Java contract following upstream's transport split; durations are converted µs→ms as the pre-b10408 payload did. Reachable-but-unexposed counters (`n_prompt_cached`, the speculative-decoding tallies) are emitted alongside and surfaced on `ServerMetrics`. `GET /slots` no longer answers 200-with-empty-body when the payload has no `slots` key. Suite 491 → **499**. **This is the class of break a header diff cannot catch**: no signature changed, the project compiled and linked clean at every one of the 25 chunks, and the only guard — `LlamaModelTest#testGetMetrics` — was model-gated and silently skipped in CI (see the same-day CI working-directory fix). | +| b10618–b10631 | `tools/server/server-common.cpp` (**new validation in `oaicompat_chat_params_parse`: continuing a final assistant message that carries `tool_calls` now throws `std::invalid_argument("Cannot continue an assistant message that contains tool calls.")`**), `common/chat.cpp` (**`common_chat_params_init_qwen3_coder` gains `is_qwen3_coder = !supports_reasoning` and gates two things on it: the `` bare-opener alternatives are only added for Coder models, and `tool-call-first` collapses to plain `tool-call` for the reasoning variant**), `CMakeLists.txt` (**`LLAMA_VERSION_MINOR` 2 → 3, cosmetic**), plus `tools/ui/**` (the WebUI, auto-followed) | **No project-source change.** Both server-side edits are behavioural, not structural: no signature, no struct field and no JSON key moved. The `oaicompat_chat_params_parse` addition is a strictly *narrowing* input validation on a combination the project never constructs — `OpenAiRequestMapper` never sets `continue_final_message`, and the C++ guard `test_utils.cpp` exercises the function without it (`ctest` 499/499 unchanged). The `chat.cpp` change is confined to one `static` template initialiser for the Qwen3-Coder family and touches only grammar construction; it is invisible at the API surface and the project pins no Coder model in CI. **The three mechanical contract checks the runbook mandates for a `tools/server/` chunk were not merely run but rendered moot:** `server-schema.cpp`, `server-task.cpp`, `server-context.cpp` and the four `tools/server/*.h` headers are **byte-identical** between the two tags (verified by comparing blob SHAs), so the request-field set, its bounds and the emitted response-key set cannot have moved. `common/common.h`, `include/llama.h` and `tools/mtmd/mtmd-helper.h` are likewise byte-identical, so the `getMetrics()`-class of silent contract break and the `tts_engine.cpp` surface are both out of scope for this range. Sizing note for the reviewer: the raw diff is ~466 KiB, but ~377 KiB of that is `tools/ui`, which CI rebuilds from `GIT_TAG` and which needs no source review — the reviewable remainder is ~89 KiB, under the runbook's 100 KiB single-step threshold. | +| b10618–b10631 | upstream verification (sandbox, target) | **Full local verification on the shipped tree.** Fresh `cmake -B build-b10631 -DBUILD_TESTING=ON` through the real `FetchContent` path, so the fail-loud `PATCH_COMMAND` ran for real against b10631: all **7** patches applied, confirmed by the stamp file (`head 5d5cb4c3a…` plus one SHA-256 line per patch) and by the source tree going dirty in 41 files. A fresh build directory was mandatory here — the applier pins its stamp to the checked-out llama.cpp commit and deliberately aborts rather than guess when an existing build dir's stamp names a different one. Full `cmake --build --config Release` clean (no errors, no new warnings), `ctest` **499/499**. `nm -D` on the freshly linked `libjllama.so` reports **zero** C++-mangled `Java_*` exports, so the `LlamaQuantizer` linkage fix carries forward. `NativeLibraryLoadSmokeTest` **3/3**, including `nativeBuildInfoMatchesPinnedVersionConstant` — the end-to-end cross-check that `LlamaCppVersion.LLAMA_CPP_VERSION` ("b10631") matches the `build-info` compiled into the new binary, which is what proves the four pin sites and the actual build agree. | +| b10631–b10636 | `ggml/src/ggml-cuda/mmq*` (quantised matmul kernel configs for Pascal), `ggml/src/ggml-metal/*` + `kernels/ssm.metal` (SSM/Mamba Metal kernels), `CMakeLists.txt` (**`LLAMA_BUILD_UI` default `ON` → `OFF`; the `LLAMA_USE_PREBUILT_UI` help text drops its `requires LLAMA_BUILD_UI=ON` clause**), `conversion/nemotron.py`, `tests/test-backend-ops.cpp`, nine `.github/workflows/*` files, plus `tools/ui/**` (the WebUI, auto-followed) | **No project-source change, and nothing on the priority review list is touched at all.** The diff contains **zero** files under `common/`, `include/`, `tools/server/` or `tools/mtmd/`, so every row of the API-compatibility table is vacuously satisfied and the three mechanical server-contract greps have no input to compare — the request-field set, its bounds and the emitted response keys cannot have moved. The ggml changes are backend kernel internals behind unchanged public headers (CUDA affects only the `cuda13-*` classifiers, Metal only the macOS default JAR). The upstream `LLAMA_BUILD_UI` flip is inert here: this project never uses upstream's UI build, it compiles its own `webui-generated/ui.cpp` produced by the `build-webui` CI job, and as a `FetchContent` subproject `LLAMA_STANDALONE` is OFF anyway. **All 7 patches apply unchanged** — none of `common/arg.{cpp,h}`, `tools/server/*` or `vendor/*` differs between the tags. Sizing: the reviewable diff (excluding `tools/ui`) is ~85 KiB, under the runbook's 100 KiB single-step threshold. | +| b10631–b10636 | upstream verification (sandbox, target) | **Full local verification on the shipped tree.** Fresh `cmake -B build-b10636 -DBUILD_TESTING=ON` through the real `FetchContent` path, so the fail-loud `PATCH_COMMAND` ran for real against b10636: all **7** patches applied, confirmed by the stamp file (`head 4d19b28769…` plus one SHA-256 line per patch) and by the source tree going dirty in 41 files. Full `cmake --build --config Release` clean, `ctest` **504/504**. `nm -D` on the freshly linked `libjllama.so` reports **zero** C++-mangled `Java_*` exports, so the `LlamaQuantizer` linkage fix carries forward. `NativeLibraryLoadSmokeTest` **3/3, 0 skipped**, including `nativeBuildInfoMatchesPinnedVersionConstant` — the end-to-end cross-check that `LlamaCppVersion.LLAMA_CPP_VERSION` ("b10636") matches the `build-info` compiled into the new binary, which is what proves the four pin sites and the actual build agree. | +| b10636–b10639 | `ggml/src/ggml-rpc/ggml-rpc.cpp` + `ggml/include/ggml-rpc.h` (**#18626: event and async backend APIs for the RPC backend; `RPC_PROTO_{MAJOR,MINOR}_VERSION` 5.1 → 6.0**), `ggml/src/ggml-vulkan/ggml-vulkan.cpp` + two new `vulkan-shaders/cross_entropy_loss{,_back}.comp` + `vulkan-shaders-gen.cpp` (**#27216: Vulkan `cross_entropy_loss` / `_back`; #27726: warptiles clamped because they assume warp sizes ≤ 64**), `docs/ops.md`, `docs/ops/Vulkan.csv` | **No project-source change, and nothing on the priority review list is touched at all.** Eight files, zero of them under `common/`, `include/llama.h`, `tools/server/` or `tools/mtmd/`, so every row of the API-compatibility table is vacuously satisfied and the three mechanical server-contract greps have no input to compare — the request-field set, its bounds and the emitted response keys cannot have moved. Unlike the previous two ranges there is **no `tools/ui` component at all**, so the whole ~70 KiB raw diff is the reviewable diff, comfortably under the runbook's 100 KiB single-step threshold. The one public header in the range is `ggml/include/ggml-rpc.h`, and its entire diff is the two protocol-version macros: **`GGML_RPC` is never enabled anywhere in this project** (`grep -rn GGML_RPC` across the workflows, build scripts and CMake is empty), so `ggml-rpc.cpp` is not compiled into `libjllama` and the protocol bump is a wire-compatibility concern only for operators running upstream's `rpc-server`, which this project does not ship. The Vulkan work is backend kernel internals behind unchanged public headers and reaches only the `vulkan-linux-x86-64` / `vulkan-linux-aarch64` / `vulkan-windows-x86-64` classifiers, whose CI jobs are build-only (GitHub runners have no GPU), so the CI signal for them is compilation. **All 7 patches apply unchanged** — the patch target list and the eight changed files have **zero** overlap (no `common/arg.{cpp,h}`, no `tools/server/*`, no `vendor/*`). | +| b10636–b10639 | upstream verification (sandbox, target) | **Full local verification on the shipped tree.** Fresh `cmake -B build-b10639 -DBUILD_TESTING=ON` through the real `FetchContent` path, so the fail-loud `PATCH_COMMAND` ran for real against b10639: all **7** patches applied, confirmed by the stamp file (`head 5e6a37cb115dc1074e274ac004373f5661909695` plus one SHA-256 line per patch) and by the source tree going dirty in 41 files. A fresh build directory was mandatory here — the applier pins its stamp to the checked-out llama.cpp commit and deliberately aborts rather than guess when an existing build dir's stamp names a different one. Full `cmake --build --config Release` clean, `ctest` **504/504**. `nm -D` on the freshly linked `libjllama.so` reports **40** `Java_*` exports and **zero** C++-mangled ones, so the `LlamaQuantizer` linkage fix carries forward. `NativeLibraryLoadSmokeTest` **3/3, 0 skipped**, including `nativeBuildInfoMatchesPinnedVersionConstant` — the end-to-end cross-check that `LlamaCppVersion.LLAMA_CPP_VERSION` ("b10639") matches the `build-info` compiled into the new binary (`b10639-5e6a37cb1`), which is what proves the four pin sites and the actual build agree. **Note for the next bump:** that last check first reported a *false* drift (`must start with the pinned tag "b10636-"`) even though both the source and `target/classes` already read `b10639`. `LLAMA_CPP_VERSION` is a compile-time constant, so javac had inlined the old value into the already-compiled `NativeLibraryLoadSmokeTest.class` and Maven's incremental compilation cannot see that dependency; `mvn clean test` clears it. Recorded in the runbook so the next bump does not re-diagnose it. **Re-verified after `patches/0011` was added** (a long-standing upstream bug this bump's CI was the first run ever to reach — see the patch table in `CLAUDE.md`; it is *not* a b10636–b10639 regression, the range touches no `common/` file at all): a second fresh build directory, applier run and Release build with **8** patches in the stamp, `ctest` **509/509**, `nm -D` unchanged at 40/0, `NativeLibraryLoadSmokeTest` still 3/3 with 0 skipped. The "all 7 patches" counts in the two rows above are left as written — they are the accurate record of what the *bump* required. | +| b10639–b10644 | `include/llama.h` (**`LLAMA_SESSION_VERSION` 9 → 10 and `LLAMA_STATE_SEQ_VERSION` 2 → 3** — the only two lines in the file that moved), `src/llama-kv-cells.h` + `src/llama-kv-cache.{cpp,h}` (`llama_kv_cell_ext` gains a `tok` field for n-gram input embeddings, plus `has_cell_ext()` / `get_prev_tokens()` / `for_each_token_in()`, and `reset()` swaps a `memset` for value-initialisation), `src/models/nanbeige.cpp` (one line: register `t_layer_inp[il]`), `tests/test-backend-ops.cpp`, the whole `ggml/src/ggml-hexagon/**` + `scripts/snapdragon/**` + `docs/backend/snapdragon/**` Snapdragon/Hexagon backend rework, plus `tools/ui/**` (the WebUI, auto-followed) | **No project-source change, and the only priority-review-list file touched is `include/llama.h`, whose entire diff is the two version macros.** Sizing: the raw diff is ~608 KiB, but it is dominated by two components this project never compiles or reviews — `tools/ui` (auto-followed; the `build-webui` job rebuilds it from the pin) and the Hexagon backend (`grep -rn GGML_HEXAGON` over this repo's workflows, build scripts and CMake is empty, so `ggml-hexagon.cpp` is never in a `libjllama` link line). Excluding both leaves **11 KiB across 6 files**, well under the runbook's 100 KiB single-step threshold, so no chunking was needed. **Zero** files under `common/`, `tools/server/` or `tools/mtmd/` changed, so every remaining row of the API-compatibility table is vacuously satisfied and the three mechanical server-contract greps have no input — the request-field set, its bounds and the emitted response keys cannot have moved. **Compatibility note for consumers, not a compile break:** the two version bumps are a *state-file format* break — a slot state saved by an older build (`LlamaModel.handleSlotAction` save, or the server's `/slots/{id}?action=save`) is rejected by its own version check on b10644 and must be regenerated. Nothing the project calls changed shape. **All 8 patches apply unchanged** — the 6 project-relevant changed files and the 42 files the patches target have **zero** overlap (no `common/arg.{cpp,h}`, no `common/peg-parser.cpp`, no `tools/server/*`; the one changed test, `tests/test-backend-ops.cpp`, is touched by no patch). | +| b10639–b10644 | upstream verification (sandbox, target) | **Full local verification on the shipped tree.** Fresh `cmake -B build-b10644 -DBUILD_TESTING=ON` through the real `FetchContent` path, so the fail-loud `PATCH_COMMAND` ran for real against b10644: all **8** patches applied, confirmed by the stamp file (`head d7a2074112d27649303fa107eb8c94db1ee435f3` plus one SHA-256 line per patch) — including `0011`, whose target `common/peg-parser.cpp` the range does not touch. Full `cmake --build --config Release` clean, `ctest` **509/509**. `nm -D` on the freshly linked `libjllama.so` reports **40** `Java_*` exports and **zero** C++-mangled ones. `NativeLibraryLoadSmokeTest` **3/3, 0 skipped**, including `nativeBuildInfoMatchesPinnedVersionConstant` — the end-to-end cross-check that `LlamaCppVersion.LLAMA_CPP_VERSION` ("b10644") matches the `build-info` compiled into the new binary (`b10644-d7a207411`), which is what proves the four pin sites and the actual build agree. Run with `mvn clean test`, not a bare `mvn test`: `LLAMA_CPP_VERSION` is a compile-time constant that javac inlines into the already-compiled test class, and Maven's incremental compilation cannot see that dependency — the b10636→b10639 row below records the false drift that costs. | +| b10644–b10649 | **`tools/mtmd/mtmd-helper.{h,cpp}` (BREAKING: `mtmd_helper_bitmap_init_from_file` and `_from_buf` gain a 4th `struct mtmd_helper_init_opt` parameter; new `mtmd_helper_video_init_params` + `mtmd_helper_init_opt` structs and their `_default()` factories)**, **`tools/server/server-common.{h,cpp}` (BREAKING: `tokenize_input_prompts`, `process_mtmd_prompt` and `format_prompt_rerank` all gain a trailing `const mtmd_helper_init_opt &`)**, `tools/server/server-context.{h,cpp}` (`handle_count_tokens` gains the same parameter; threading it through), `common/common.h` (**additive**: `common_params_speculative::{synth_len,synth_rates}` + `has_synth()`, `common_params::{video_fps,video_timestamp_interval_ms,video_ffmpeg_bin_dir}`), `common/arg.cpp` (**purely additive**: 6 new flags), `common/speculative.{cpp,h}` (**additive**: `common_speculative_n_max`, `_synth_rates_resolve`, `_get_synth_probs`), `ggml/src/ggml-metal/*`, `src/models/minimax-01.cpp`, `tests/test-arg-parser.cpp`, `tools/{cli,llama-bench,tts}`, `tools/server/tests/*` | **The first range in this whole bump that broke the project's own compile — four call sites, all from one upstream refactor.** llama.cpp b10649 threaded a new `mtmd_helper_init_opt` (video decode settings: fps target, ffmpeg binary dir, timestamp interval) through every helper that can ingest media. `tts_engine.cpp:95` (`mtmd_helper_bitmap_init_from_file`, the speaker-reference clip) and `jllama.cpp` ×3 (`tokenize_input_prompts`) + ×1 (`format_prompt_rerank`) all lost their signatures. Every one of those paths passes `mctx = nullptr` or handles audio, so none wants video settings: each now passes `mtmd_helper_init_opt_default()`, which is upstream's own default. **This is the failure class the priority table's note calls out** — `server-common.h` is a same-repo header the project `#include`s directly rather than one reachable through the documented dependency graph, so only a real compile finds it. **The wire contract did not move:** all three mechanical greps are identical across the range (68 request fields, 23 bounds, 286 response keys over all six server `.cpp`). `common/arg.cpp` removed or renamed **zero** flags. Sizing: 106 KiB reviewable, or **72 KiB excluding `ggml-metal`** (Metal backend internals behind unchanged public headers — same exclusion rationale as Hexagon), under the runbook's 100 KiB single-step threshold. **All 8 patches apply unchanged** despite six patch-target files changing (`common/arg.cpp`, `tests/test-arg-parser.cpp`, `tools/mtmd/mtmd-cli.cpp`, `tools/server/server-context.{cpp,h}`, `tools/tts/tts.cpp`) — upstream's edits fall outside every patched region. | +| b10644–b10649 | upstream verification (sandbox, target) | **Full local verification on the shipped tree.** Fresh `cmake -B build-b10649 -DBUILD_TESTING=ON` through the real `FetchContent` path: all **8** patches applied (stamp `head 2bb9bddafad44ecbb50889644ca47537ec11841b`), which was the first thing checked because six patch-target files changed in the range. The first build failed loud with the four signature breaks above; after adapting the call sites, Release build clean, `ctest` **516/516** (512 at the bump; the second audit added four `TrainParams` cases), `nm -D` **40** `Java_*` exports and **zero** C++-mangled, `NativeLibraryLoadSmokeTest` **3/3, 0 skipped** including the pin cross-check against the linked `build-info`. `ModelParametersTest` 77/77 with the five CPU-offload cases plus the three video-flag cases. **Feature exposure decided deliberately, not by default:** of the 6 new flags, only `--n-cpu-ffn` is genuinely new upstream, and it was added as `ModelParameters.setCpuFfnLayers`; `setCpuMoeLayers` was added alongside it for `--n-cpu-moe`, which has existed upstream since **b6089** (#15077, `ec428b02c`) but had never been exposed here — b10649 only refactored its lambda onto the shared `llm_add_n_cpu_ffn_overrides` helper; the two `--spec-synth-*` flags were refused as upstream marks them "benchmarking only"; and the three `--video-*` flags were initially refused as inert without a `ContentPart` video factory. **That refusal was wrong and was reversed by the follow-up audit:** they are not inert — `server_context::load_model` copies them into its own `init_opt` when the projector loads, and that `init_opt` is what `server-context.cpp` hands to `process_mtmd_prompt` on the very task path this binding drives (`task.cli_files`), so they take effect for any attached media; and video decoding really is compiled in (`MTMD_VIDEO` defaults `ON`, gated only on `LLAMA_SUBPROCESS` which is also `ON`, and the shipped `libjllama.so` carries the ffmpeg invocation strings). They are now `ModelParameters.setVideoFps` / `setVideoTimestampInterval` / `setVideoFfmpegDir`; `setVideoFfmpegDir` matters most, since upstream otherwise resolves `ffmpeg`/`ffprobe` from `PATH`, which a JVM process frequently lacks. Only the ergonomic `ContentPart.videoFile(...)` entry point is still outstanding (raw bytes already work — the decoder sniffs the container rather than trusting the MIME type). The `--spec-synth-*` refusal stands. Both are recorded in `TODO.md` with their reasoning. | +| b10649–b10679 | `common/arg.cpp` (**purely additive: 2 new flags, 0 removed or renamed**), `common/common.h` (**additive**: `common_params::lazy_mode`, `common_params::kv_unified_per_slot`), `common/common.cpp` (one line: `mparams.lazy_mode = params.lazy_mode`), `include/llama.h` (**additive**: new `llama_lazy_mode` enum, `llama_model_params::lazy_mode`, `llama_model_quantize_params::max_buf_size`), `common/speculative.cpp` (implementation only — `speculative.h` byte-identical), `tools/server/server-context.cpp` (**#24124: per-slot context cap**; private `get_slot_n_ctx()` renamed to `n_ctx_slot()` and made a recomputing accessor), `tools/server/server.cpp` (**KV-pool auto-sizing block**), `tools/server/README.md`, plus `tools/ui/**` and backend internals (auto-followed / not compiled here) | **No project-source change; two features deliberately exposed.** The in-scope delta is **8 files, 172 insertions and 15 deletions** out of a 159-file, 8045-insertion range — the remainder is `tools/ui` (rebuilt from `GIT_TAG` by CI, no source review) and backends the project does not build. **The three mechanical contract checks are moot, not merely passed**: `server-schema.cpp`, `server-task.cpp`, `server-common.cpp` and *all twelve* `tools/server/*.h` headers are **byte-identical** across the range (verified by comparing blob SHAs), so no request field, bound or response key can have moved; `common/chat.h` and `tools/mtmd/mtmd-helper.h` are byte-identical too, so the `getMetrics()`-class silent-contract break and the `tts_engine.cpp` surface are both out of scope. `common/common.h` is the only priority-table file that changed, and only by the two additive fields above. **Two additive upstream features, both now exposed** (deliberate decision, not default): `--kv-unified-per-slot` → `ModelParameters.setKvUnifiedPerSlot(int)`, and `--tensor-read-lazy` → `ModelParameters.setTensorReadLazy(TensorReadLazyMode)` over the new `net.ladenthin.llama.args.TensorReadLazyMode` enum, which mirrors `llama_lazy_mode` (`off`/`auto`/`on` = 0/1/2). **Both really reach this binding**, which is why they were exposed rather than refused: `--tensor-read-lazy` has no `set_examples()` restriction and `common_model_params_to_llama` copies `lazy_mode` into `llama_model_params`, so it applies to a plain `LlamaModel` load; `--kv-unified-per-slot` is `set_examples({LLAMA_EXAMPLE_SERVER})` and `jllama.cpp` parses with `LLAMA_EXAMPLE_SERVER`, so it registers, and its **cap** half is applied inside `server-context.cpp`'s `n_ctx_slot()` (`server-context.cpp:4189`), whose value the project reads through `server_context_meta::slot_n_ctx` — it becomes every `slot.n_ctx` and is the context budget `format_prompt_infill` is given (`jllama.cpp:1547`), so lowering it really does change what this binding does. **Not** via `eval_llama_cmpl_schema`: that lost its `n_ctx_slot` parameter at b10275 and takes four arguments at b10679, and `repeat_last_n` / `dry_penalty_last_n` carry `set_hard_limits(0, INT32_MAX)` since b10273, so no sentinel expands to a context size any more. (The first draft of this row said otherwise; it was corrected by the b10679 audit.) Its **pool-sizing** half lives in `llama_server()` and therefore applies only to `NativeServer` full mode; the Javadoc says so explicitly rather than implying the flag is self-sufficient. `llama_model_quantize_params::max_buf_size` needs **no** adaptation: `LlamaQuantizer` builds its params from `llama_model_quantize_default_params()`, so the new field is initialised by upstream. The `get_slot_n_ctx()` → `n_ctx_slot()` rename is a **private** member of the impl and is not called by the project. **Three patch-target files were touched** (`common/arg.cpp`, `server-context.cpp`, `server.cpp`) and **all 8 patches still apply with zero fuzz**; patch `0007`'s standing invariant survives because the new KV-pool-sizing block in `llama_server()` sits *before* the extracted route table, not inside it. **Patch `0001` shrank 37 → 36 files**: upstream rewrote `tests/test-save-load-state.cpp`'s `main()` to strip `--models DIR` into its own `filtered_argv` and call `common_params_parse(fargc, filtered_argv.data(), ...)`. By that patch's own rule — *embedded callers that build their own argv must call `common_params_parse` directly* — that call site no longer wants the `_main()` flip, so the hunk was **dropped, not refreshed**. The patch itself is **still required**: `common_params_parse` at b10679 still carries the count-guarded `GetCommandLineW` override, and `common_params_parse_main` does not exist in `b10679:common/arg.h` — upstream has not adopted either direction proposed in [ggml-org/llama.cpp#26416](https://github.com/ggml-org/llama.cpp/issues/26416). | +| b10649–b10679 | upstream verification (sandbox, target) | **Full local verification on the shipped tree.** Fresh `cmake -B build-b10679 -DBUILD_TESTING=ON` through the real `FetchContent` path, so the fail-loud `PATCH_COMMAND` ran for real against b10679 — a fresh build directory was mandatory, since the applier pins its stamp to the checked-out llama.cpp commit and aborts rather than guess when an existing stamp names a different one. All **8** patches applied (`0001` in its 36-file form). Release build clean, `ctest` **520/520**. Java: `mvn clean test` **1474 run / 0 failures / 17 model-gated skips**, with `NativeLibraryLoadSmokeTest` **3/3 and 0 skipped** — including `nativeBuildInfoMatchesPinnedVersionConstant`, the end-to-end cross-check that `LlamaCppVersion.LLAMA_CPP_VERSION` ("b10679") matches the `build-info` compiled into the freshly linked `libjllama.so`, which is what proves the four pin sites and the actual build agree. `ModelParametersTest` covers both new setters (exact flag spellings, the `<= 0` rejection on `--kv-unified-per-slot`) and `TensorReadLazyModeTest` pins all three enum wire strings under the PIT-gated `net.ladenthin.llama.args.*` package. `mvn spotless:apply` produced no changes beyond the edits themselves. **One documentation defect was found and fixed during this verification** and is worth recording as a class: the first draft of `setKvUnifiedPerSlot`'s Javadoc stated the pool-sizing effect unconditionally. Reading `tools/server/server.cpp` showed that half executes only in `llama_server()`, which a `ModelParameters`-loaded model never enters — an accurate-for-upstream sentence that would have been wrong for the API it documents. | diff --git a/docs/upgrade/llama-cpp-version-bump.md b/docs/upgrade/llama-cpp-version-bump.md index 9be1fe07f..9896d44e9 100644 --- a/docs/upgrade/llama-cpp-version-bump.md +++ b/docs/upgrade/llama-cpp-version-bump.md @@ -110,34 +110,74 @@ Once you have the `b -> b` step, apply it exactly as Concretely: 1. **Edit the pin — four files:** - - `llama/CMakeLists.txt` — the `GIT_TAG b` line **and** the `-DLLAMA_TAG=b` used by the - WebUI/TTS extraction (both must move together). - - `README.md` — the llama.cpp badge and link (version appears twice). + - `llama/CMakeLists.txt` — the `GIT_TAG b` line. (It is the only `b` tag in this file — the other two `GIT_TAG` lines pin nlohmann/json `v3.12.0` and GoogleTest `v1.17.0` and must NOT move with a llama.cpp bump. The + `-DLLAMA_TAG=b` that once fed the build-time TTS extraction was removed with the + Qwen3-TTS rework, and the WebUI auto-follows `GIT_TAG` in CI.) + - `README.md` — the llama.cpp badge and link (the tag appears three times, all on one line: the + badge alt text, the badge URL and the release link). - `CLAUDE.md` — the "Current llama.cpp pinned version" line (and any build-example `b`). - `llama/src/main/java/net/ladenthin/llama/value/LlamaCppVersion.java` — the `LLAMA_CPP_VERSION` constant (the pure-Java pin consumers read for a version badge/log line). It mirrors `GIT_TAG`; if you forget it, `NativeLibraryLoadSmokeTest.nativeBuildInfoMatchesPinnedVersionConstant` fails the build (it cross-checks the constant against `LlamaModel.getLlamaCppBuildInfo()`, which reads llama.cpp's own linked-in `build-info`). -2. **Re-verify `patches/`** — a clean configure re-runs the fail-loud `PATCH_COMMAND`, so every patch - `0001`–`0006` must still apply. Use a **fresh** build dir (a stale one re-applies over an - already-patched tree and reports a false "does not apply"): + + > **Local-only gotcha: that guard can report a *false* drift after a bump.** + > `LLAMA_CPP_VERSION` is a `public static final String`, i.e. a **compile-time constant**, so + > javac inlines its value into every *referencing* class — including + > `NativeLibraryLoadSmokeTest`. Recompiling `LlamaCppVersion.java` alone therefore does **not** + > update the copy baked into the already-compiled test class, and Maven's incremental + > compilation cannot see the dependency (constant inlining is invisible to its change + > analysis). The symptom is a failure that looks alarming but is pure staleness, e.g. + > `Linked build-info "b10639-…" must start with the pinned tag "b10636-"` when both the source + > and `target/classes` already say `b10639`. Run **`mvn clean test`** (not a bare `mvn test`) + > when re-running this check locally after a bump. CI is immune — it always builds from a clean + > checkout. +2. **Re-verify `patches/`** — a clean configure re-runs the fail-loud `PATCH_COMMAND`, so **every + `*.patch` in `llama/patches/`** must still apply. Do not maintain a list of them here or anywhere + else: `apply-llama-patches.cmake` `file(GLOB)`s the directory and applies them in filename order, + so an enumeration can only go stale (it did, one commit after being written). Use a **fresh** + build dir: the applier's stamp file pins the patch set to the *checked-out llama.cpp commit*, so + after a `GIT_TAG` change an existing build dir is exactly the case it refuses to guess at — it + aborts and tells you to configure fresh, which is what actually re-runs the patches against the + new source: ```bash cd llama && mvn -q compile # generates the OSInfo class CMake's OS-detection needs rm -rf build && cmake -B build # fail-loud: aborts here if any patch no longer applies ``` If a patch no longer applies, refresh its diff against the new source and recommit it. -3. **Append the history rows** — add a pair of rows to +3. **Check the server contract mechanically when the chunk touches `tools/server/`.** A header diff + only shows signature changes; it cannot see a *contract* change behind a stable signature. Two + breaks of that class already shipped — `getMetrics()`'s payload shape (b10408/b10519) and the + removal of the `-1` = context-size sentinel for `repeat_last_n`/`dry_penalty_last_n` (b10273) — + and neither was visible to the build. Diff these three sets between the two tags; anything that + changes has to be traced to the Java layer, not just to the C++ tests: + ```bash + # request-field set + git show b:tools/server/server-schema.cpp | grep -oE 'field_[a-z_]+\("[a-z_0-9]+"' | sort -u + # request-field bounds + git show b:tools/server/server-schema.cpp | tr '\n' ' ' \ + | grep -oE 'field_[a-z]+[^(]*\("[a-z_0-9]+"[^;]*?set_(hard_)?limits\([^)]*\)' | sort -u + # response keys + # response keys -- BOTH emit forms: brace-init AND res["k"] = ...; the b10585 migration + # moved `timings`/`prompt_progress` between the two, so a single-form grep reports + # false removals and would silently miss a new operator[] key. + git show b:tools/server/server-task.cpp | { grep -oE '\{ *"[A-Za-z_0-9.]+" *,'; \ + git show b:tools/server/server-task.cpp | grep -oE '\[ *"[A-Za-z_0-9.]+" *\] *='; } | sort -u + ``` + Repeat with `b` and `comm -13` / `comm -23` the two outputs. + +4. **Append the history rows** — add a pair of rows to [`../history/llama-cpp-breaking-changes.md`](../history/llama-cpp-breaking-changes.md) covering the `b -> b` range (what broke / what was new; "no source change" is a valid row). -4. **Commit + push** on the working branch (do not open a new PR if one already tracks the branch): +5. **Commit + push** on the working branch (do not open a new PR if one already tracks the branch): ```bash git add llama/CMakeLists.txt README.md CLAUDE.md docs/history/llama-cpp-breaking-changes.md \ llama/src/main/java/net/ladenthin/llama/value/LlamaCppVersion.java git commit -m "Upgrade llama.cpp from b to b" git push -u origin ``` -5. **Re-run the helper** for the next chunk. Repeat until it reports the **final chunk** (target +6. **Re-run the helper** for the next chunk. Repeat until it reports the **final chunk** (target reached). CI builds every native classifier from the new pin; the full model-backed Java + C++ suites gate the diff --git a/docs/upstream-investigation-win32-argv-substitution.md b/docs/upstream-investigation-win32-argv-substitution.md index 6db2f9a32..3bc07c99d 100644 --- a/docs/upstream-investigation-win32-argv-substitution.md +++ b/docs/upstream-investigation-win32-argv-substitution.md @@ -1,3 +1,9 @@ + + # `common_params_parse` can silently discard the caller's argv on Windows Technical findings for llama.cpp. Everything below was verified against diff --git a/llama-langchain4j/src/main/java/net/ladenthin/llama/langchain4j/LangChain4jMapping.java b/llama-langchain4j/src/main/java/net/ladenthin/llama/langchain4j/LangChain4jMapping.java index c137b51b1..550a6c2c3 100644 --- a/llama-langchain4j/src/main/java/net/ladenthin/llama/langchain4j/LangChain4jMapping.java +++ b/llama-langchain4j/src/main/java/net/ladenthin/llama/langchain4j/LangChain4jMapping.java @@ -96,7 +96,7 @@ static InferenceParameters toStreamingParameters(ChatRequest request) { InferenceParameters.empty().withMessagesJson(jllama.buildMessagesJson()); java.util.Optional toolsJson = jllama.buildToolsJson(); if (toolsJson.isPresent()) { - params = params.withToolsJson(toolsJson.get()).withUseChatTemplate(true); + params = params.withToolsJson(toolsJson.get()); java.util.Optional toolChoice = jllama.getToolChoice(); if (toolChoice.isPresent()) { params = params.withToolChoice(toolChoice.get()); diff --git a/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/JllamaChatModelIntegrationTest.java b/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/JllamaChatModelIntegrationTest.java index ea4b18ece..cc4de0fba 100644 --- a/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/JllamaChatModelIntegrationTest.java +++ b/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/JllamaChatModelIntegrationTest.java @@ -14,7 +14,6 @@ import dev.langchain4j.model.chat.response.StreamingChatResponseHandler; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import net.ladenthin.llama.LlamaModel; @@ -29,10 +28,26 @@ */ class JllamaChatModelIntegrationTest { + /** + * Generation budget for both tests, matching the core module's {@code ReasoningBudgetTest} + * ({@code N_PREDICT = 1500}) for the same model. + * + *

Qwen3-0.6B is a reasoning model: it spends its first few hundred tokens inside + * {@code } and only then emits assistant content, so a budget that does not clear the + * thinking block yields an empty answer rather than a short one. 320 was tried and is + * right on the boundary — in one CI run (33109360197) the blocking test finished thinking at 267 + * tokens and passed while the streaming test consumed all 320 inside {@code } and failed, + * from the same prompt against the same model. The adapter exposes no reasoning-budget knob, so + * the output budget is the only lever. + * + *

This is a cap, not a target: a normal run stops around 270–340 tokens, so raising it costs + * nothing except in the pathological case it exists to absorb. + */ + private static final int MAX_OUTPUT_TOKENS = 1500; + private static Path modelPath() { - String path = System.getProperty("net.ladenthin.llama.model.path"); - Assumptions.assumeTrue(path != null && !path.isEmpty(), "model path property not set"); - Path resolved = Paths.get(path); + Path resolved = TestModelPaths.fromProperty("net.ladenthin.llama.model.path"); + Assumptions.assumeTrue(resolved != null, "model path property not set"); Assumptions.assumeTrue(Files.exists(resolved), "model file not present: " + resolved); return resolved; } @@ -47,11 +62,15 @@ void chatReturnsAssistantText() { chat.chat( ChatRequest.builder() .messages(UserMessage.from("Reply with the single word: ok")) - .maxOutputTokens(8) + // See MAX_OUTPUT_TOKENS: too small a budget yields an EMPTY + // assistant text, which a bare notNullValue() would accept. + .maxOutputTokens(MAX_OUTPUT_TOKENS) .build()); assertThat(response.aiMessage(), is(notNullValue())); assertThat(response.aiMessage().text(), is(notNullValue())); + assertThat("the model must produce assistant text, not only a thinking block", + response.aiMessage().text().trim().isEmpty(), is(false)); } } @@ -66,7 +85,10 @@ void streamingDeliversTokensThenCompletes() throws Exception { streaming.chat( ChatRequest.builder() .messages(UserMessage.from("Reply with the single word: ok")) - .maxOutputTokens(8) + // See MAX_OUTPUT_TOKENS: the budget has to clear the thinking block, or + // the run produces no assistant content at all and the second assertion + // below fails on a healthy model. + .maxOutputTokens(MAX_OUTPUT_TOKENS) .build(), new StreamingChatResponseHandler() { @Override @@ -85,8 +107,23 @@ public void onError(Throwable error) { } }); - ChatResponse complete = done.get(60, TimeUnit.SECONDS); - assertThat(complete.aiMessage().text(), is(streamed.toString())); + ChatResponse complete = done.get(180, TimeUnit.SECONDS); + + // Two independent assertions, both of which must hold. + // + // 1. The concatenated onPartialResponse fragments are exactly the final text. This is the + // streaming contract itself: a regression that misroutes content deltas into + // reasoning_content, or drops a fragment, breaks it. + String finalText = complete.aiMessage().text() == null + ? "" + : complete.aiMessage().text(); + assertThat(finalText, is(streamed.toString())); + + // 2. Actual assistant CONTENT arrived -- not merely "content or thinking". With a budget + // that clears the thinking block this is the real signal; accepting thinking alone + // would let a content-routing regression pass unnoticed, which is what the earlier, + // 8-token version of this test did. + assertThat("stream delivered no assistant content", !streamed.toString().isEmpty(), is(true)); } } } diff --git a/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/JllamaEmbeddingModelIntegrationTest.java b/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/JllamaEmbeddingModelIntegrationTest.java index 79e0384a2..2aede4ebb 100644 --- a/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/JllamaEmbeddingModelIntegrationTest.java +++ b/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/JllamaEmbeddingModelIntegrationTest.java @@ -14,7 +14,6 @@ import dev.langchain4j.model.output.Response; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.Arrays; import java.util.List; import net.ladenthin.llama.LlamaModel; @@ -32,9 +31,8 @@ class JllamaEmbeddingModelIntegrationTest { private static Path modelPath() { - String path = System.getProperty("net.ladenthin.llama.langchain4j.embedding.model"); - Assumptions.assumeTrue(path != null && !path.isEmpty(), "embedding model path property not set"); - Path resolved = Paths.get(path); + Path resolved = TestModelPaths.fromProperty("net.ladenthin.llama.langchain4j.embedding.model"); + Assumptions.assumeTrue(resolved != null, "embedding model path property not set"); Assumptions.assumeTrue(Files.exists(resolved), "embedding model file not present: " + resolved); return resolved; } diff --git a/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/JllamaScoringModelIntegrationTest.java b/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/JllamaScoringModelIntegrationTest.java index 4c61c16d8..db4442dbb 100644 --- a/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/JllamaScoringModelIntegrationTest.java +++ b/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/JllamaScoringModelIntegrationTest.java @@ -12,7 +12,6 @@ import dev.langchain4j.model.output.Response; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.Arrays; import java.util.List; import net.ladenthin.llama.LlamaModel; @@ -30,9 +29,8 @@ class JllamaScoringModelIntegrationTest { private static Path modelPath() { - String path = System.getProperty("net.ladenthin.llama.langchain4j.rerank.model"); - Assumptions.assumeTrue(path != null && !path.isEmpty(), "rerank model path property not set"); - Path resolved = Paths.get(path); + Path resolved = TestModelPaths.fromProperty("net.ladenthin.llama.langchain4j.rerank.model"); + Assumptions.assumeTrue(resolved != null, "rerank model path property not set"); Assumptions.assumeTrue(Files.exists(resolved), "rerank model file not present: " + resolved); return resolved; } diff --git a/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/JllamaToolCallingIntegrationTest.java b/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/JllamaToolCallingIntegrationTest.java index 2f24169c1..5e95cb2f7 100644 --- a/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/JllamaToolCallingIntegrationTest.java +++ b/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/JllamaToolCallingIntegrationTest.java @@ -23,7 +23,7 @@ import dev.langchain4j.model.chat.response.ChatResponse; import dev.langchain4j.model.output.FinishReason; import java.nio.file.Files; -import java.nio.file.Paths; +import java.nio.file.Path; import net.ladenthin.llama.LlamaModel; import net.ladenthin.llama.parameters.ModelParameters; import org.junit.jupiter.api.AfterAll; @@ -49,9 +49,10 @@ class JllamaToolCallingIntegrationTest { @BeforeAll static void loadModel() { - String path = System.getProperty(PROP_TOOL_MODEL); - Assumptions.assumeTrue(path != null && !path.isEmpty(), "tool model path property not set"); - Assumptions.assumeTrue(Files.exists(Paths.get(path)), "model file not present: " + path); + Path resolved = TestModelPaths.fromProperty(PROP_TOOL_MODEL); + Assumptions.assumeTrue(resolved != null, "tool model path property not set"); + Assumptions.assumeTrue(Files.exists(resolved), "model file not present: " + resolved); + String path = resolved.toString(); model = new LlamaModel(new ModelParameters() .setModel(path) .setCtxSize(8192) diff --git a/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/LangChain4jMappingTest.java b/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/LangChain4jMappingTest.java index 40cea27c5..e495e4c8c 100644 --- a/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/LangChain4jMappingTest.java +++ b/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/LangChain4jMappingTest.java @@ -218,6 +218,11 @@ void streamingParametersCarryToolsAndToolChoice() { assertThat(json, containsString("get_weather")); assertThat(json, containsString("\"tool_choice\"")); assertThat(json, containsString("required")); + // Jinja is a load-time option (--jinja). Upstream's request parser never reads a "use_jinja" + // key and silently discards unknown fields, so a re-added withUseChatTemplate(true) here + // would be invisible at runtime and uncatchable by any integration test. Mirrors + // OpenAiRequestMapperTest#toolsEnableChatTemplateAndForwardChoice. + assertThat(json, not(containsString("\"use_jinja\""))); } @Test diff --git a/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/StreamingChunkAssemblerTest.java b/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/StreamingChunkAssemblerTest.java index 1f2a9b2c1..171393316 100644 --- a/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/StreamingChunkAssemblerTest.java +++ b/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/StreamingChunkAssemblerTest.java @@ -6,6 +6,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -151,6 +152,30 @@ void forwardsThinkingDeltasAndKeepsThinkingOnFinalMessage() { assertThat(response.aiMessage().text(), is("42")); } + @Test + void thinkingOnlyStreamLeavesTextNullAndDeliversNoContentFragments() { + // A reasoning model that spends its whole output budget inside streams only + // reasoning_content. complete() then never calls message.text(...), so AiMessage.text() + // is null rather than "" -- which is exactly what made + // JllamaChatModelIntegrationTest#streamingDeliversTokensThenCompletes fail with + // "expected \"\" but was null" the first time it actually ran against Qwen3-0.6B. + // Pinned here, model-free, so the null is a known contract and not a surprise. + RecordingHandler handler = new RecordingHandler(); + StreamingChunkAssembler assembler = new StreamingChunkAssembler(handler); + + assembler.accept("{\"choices\":[{\"index\":0,\"delta\":{\"reasoning_content\":\"still \"}," + + "\"finish_reason\":null}]}"); + assembler.accept("{\"choices\":[{\"index\":0,\"delta\":{\"reasoning_content\":\"thinking\"}," + + "\"finish_reason\":null}]}"); + assembler.accept("{\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"length\"}]}"); + ChatResponse response = assembler.complete(); + + assertThat(response.aiMessage().text(), is(nullValue())); + assertThat(response.aiMessage().thinking(), is("still thinking")); + assertThat(handler.partials, is(empty())); + assertThat(handler.thinking, contains("still ", "thinking")); + } + @Test void noUsageChunkMeansNoTokenUsage() { RecordingHandler handler = new RecordingHandler(); diff --git a/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/TestModelPaths.java b/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/TestModelPaths.java new file mode 100644 index 000000000..bb4a7954d --- /dev/null +++ b/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/TestModelPaths.java @@ -0,0 +1,59 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.langchain4j; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Resolves the GGUF paths the model-backed integration tests are pointed at. + * + *

Surefire's working directory defaults to the module basedir ({@code /llama-langchain4j}), + * while CI restores the shared GGUF cache to {@code /models} and passes the paths as bare + * {@code models/}. Resolving those against the module directory finds nothing, every test + * self-skips on its {@code Assumptions.assumeTrue(exists)}, and the job still reports success. This + * helper accepts either layout — module-relative first, then reactor-root — so the tests + * actually run in CI without the workflow having to know where Surefire stands. + * + *

The core module carries the same resolver as {@code TestConstants.resolveModelPath}; it is + * duplicated rather than shared because test classes are not published between modules. + */ +final class TestModelPaths { + + private TestModelPaths() {} + + /** + * Resolves a configured fixture path against the working directory and then its parent. + * + * @param path the configured path, may be {@code null} or empty + * @return an existing path, or {@code null} when {@code path} is null/empty, or the unresolved + * path itself when it exists in neither location (so skip messages name what was looked for) + */ + static Path resolve(String path) { + if (path == null || path.isEmpty()) { + return null; + } + Path candidate = Paths.get(path); + if (candidate.isAbsolute() || Files.exists(candidate)) { + return candidate; + } + Path fromParent = Paths.get("..").resolve(candidate); + if (Files.exists(fromParent)) { + return fromParent.toAbsolutePath().normalize(); + } + return candidate; + } + + /** + * Resolves the path held by a system property. + * + * @param property the system-property name + * @return the resolved path, or {@code null} when the property is unset or empty + */ + static Path fromProperty(String property) { + return resolve(System.getProperty(property)); + } +} diff --git a/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/TestModelPathsTest.java b/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/TestModelPathsTest.java new file mode 100644 index 000000000..89f4397dc --- /dev/null +++ b/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/TestModelPathsTest.java @@ -0,0 +1,166 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.langchain4j; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Guards {@link TestModelPaths}, which is this module's copy of the core module's model-path + * resolver. + * + *

It exists because of a defect that stayed invisible for months: Surefire's working directory is + * the module basedir while CI restores the GGUF cache to the reactor root, so a bare + * {@code models/} resolved to nothing, every model-gated class aborted in its + * {@code Assumptions.assumeTrue(exists)}, and the job stayed green with the tests reported as + * skipped. The resolver fixes that; until now nothing proved the resolver itself works, and + * nothing stopped a new test class from going back to a bare {@code System.getProperty}. + */ +class TestModelPathsTest { + + @Test + void nullAndEmptyResolveToNull() { + assertThat(TestModelPaths.resolve(null), is(nullValue())); + assertThat(TestModelPaths.resolve(""), is(nullValue())); + } + + @Test + void anExistingModuleRelativePathIsReturnedUnchanged(@TempDir Path tmp) throws IOException { + Path present = Files.createFile(tmp.resolve("present.gguf")); + Path resolved = TestModelPaths.resolve(present.toString()); + assertThat(resolved, is(notNullValue())); + assertThat(Files.exists(resolved), is(true)); + } + + @Test + void anAbsolutePathIsReturnedEvenWhenItDoesNotExist(@TempDir Path tmp) { + Path missing = tmp.resolve("missing.gguf").toAbsolutePath(); + assertThat(TestModelPaths.resolve(missing.toString()), is(missing)); + } + + @Test + void anUnresolvablePathComesBackUnchangedSoTheSkipMessageNamesIt() { + // Deliberately NOT null: the caller puts this in its assumption message, and "models/x.gguf + // not found" is a far more useful CI line than "null". + String wanted = "models/definitely-not-here-" + TestModelPathsTest.class.getSimpleName() + ".gguf"; + Path resolved = TestModelPaths.resolve(wanted); + assertThat(resolved, is(Paths.get(wanted))); + } + + @Test + void aPathOnlyPresentFromTheReactorRootResolvesToAnExistingFile() { + // The branch this resolver exists for, and the only one the other cases never reach: + // Surefire's working directory is this module's basedir, so "llama-langchain4j/pom.xml" + // exists only when looked up from the parent. Exactly the models/ situation. + Path resolved = TestModelPaths.resolve("llama-langchain4j/pom.xml"); + assertThat(resolved, is(notNullValue())); + assertThat("a path that only exists from the reactor root must resolve: " + resolved, + Files.exists(resolved), is(true)); + assertThat(resolved.isAbsolute(), is(true)); + } + + @Test + void fromPropertyIsNullWhenThePropertyIsUnset() { + assertThat(TestModelPaths.fromProperty("net.ladenthin.llama.langchain4j.definitely.unset"), is(nullValue())); + } + + /** + * The rule, not a snapshot: no test in this module may read a model path with a bare + * {@code System.getProperty}, because that bypasses the resolver and silently re-mutes that class + * in CI. {@link TestModelPaths} itself is where the raw read legitimately lives. + */ + @Test + void noTestReadsAModelPropertyWithoutTheResolver() throws IOException { + Path testSources = Paths.get("src/test/java"); + assertTrue(Files.isDirectory(testSources), "test sources not on disk: " + testSources.toAbsolutePath()); + + List offenders = new ArrayList<>(); + try (Stream paths = Files.walk(testSources)) { + List javaFiles = paths.filter(Files::isRegularFile) + .filter(f -> f.getFileName().toString().endsWith(".java")) + .filter(f -> !f.getFileName().toString().equals("TestModelPaths.java")) + .filter(f -> !f.getFileName().toString().equals("TestModelPathsTest.java")) + .collect(Collectors.toList()); + assertFalse(javaFiles.isEmpty(), "the scan matched no sources under " + testSources.toAbsolutePath()); + for (Path file : javaFiles) { + String source = new String(Files.readAllBytes(file), StandardCharsets.UTF_8); + // Inspect each call's own argument list rather than the surrounding line: a property + // is normally named through a constant, either form may be wrapped across lines, and + // scoping to the arguments keeps a neighbouring comment mentioning a + // net.ladenthin.llama class from reading as a violation. + offenders.addAll(rawModelPropertyReads(file, source)); + } + } + + assertTrue( + offenders.isEmpty(), + "Read net.ladenthin.llama.* path properties through TestModelPaths, not System.getProperty " + + "- a bare read resolves against Surefire's module-basedir CWD and silently self-skips " + + "in CI. Offending sites:\n" + String.join("\n", offenders)); + } + + /** + * Finds every {@code System.getProperty(...)} / {@code System.getenv(...)} call in {@code source} + * whose own argument list names a {@code net.ladenthin.llama.*} property, either as a literal or + * through a {@code PROP_*} constant. + * + * @param file the file being scanned, used only to label a finding + * @param source the file's full text + * @return one entry per offending call site; empty when the file is clean + */ + private static List rawModelPropertyReads(Path file, String source) { + List found = new ArrayList<>(); + for (String call : new String[] {"System.getProperty(", "System.getenv("}) { + int from = 0; + while (true) { + int start = source.indexOf(call, from); + if (start < 0) { + break; + } + int open = start + call.length() - 1; + int depth = 0; + int end = open; + while (end < source.length()) { + char c = source.charAt(end); + if (c == '(') { + depth++; + } else if (c == ')') { + depth--; + if (depth == 0) { + break; + } + } + end++; + } + String arguments = source.substring(open, Math.min(end + 1, source.length())); + if (arguments.contains("net.ladenthin.llama") + || arguments.contains("PROP_") + || arguments.contains("LlamaSystemProperties.PREFIX")) { + found.add(file + " " + source.substring(start, Math.min(end + 1, source.length())) + .replaceAll("\\s+", " ")); + } + from = start + call.length(); + } + } + return found; + } + +} diff --git a/llama/CMakeLists.txt b/llama/CMakeLists.txt index 2197709ac..3c34c60c6 100644 --- a/llama/CMakeLists.txt +++ b/llama/CMakeLists.txt @@ -173,7 +173,7 @@ set(LLAMA_BUILD_APP OFF CACHE BOOL "" FORCE) FetchContent_Declare( llama.cpp GIT_REPOSITORY https://github.com/ggerganov/llama.cpp.git - GIT_TAG b10456 + GIT_TAG b10679 PATCH_COMMAND ${CMAKE_COMMAND} -DPATCH_DIR=${CMAKE_CURRENT_SOURCE_DIR}/patches -DLLAMA_SRC= @@ -451,9 +451,12 @@ endif() # shared to every native build as a generated, platform-independent ui.cpp/ui.h; # the "WebUI assets" block below compiles it in when present and otherwise falls # back to the empty-asset stub (src/main/cpp/webui_stub/ui.h). -# already resolves via llama-common's vendor/ include dir, -# whose bundled nlohmann/json is the same 3.12.0 as our FetchContent copy, so -# adding nothing there shadows it. +# already resolves through llama-common: upstream #27304 +# (b10488) replaced its `PUBLIC . ../vendor` include dir with PUBLIC links to the +# vendor::nlohmann / vendor::sheredom INTERFACE targets, and each of those exports +# the vendor/ root (`target_include_directories( INTERFACE ..)`), so the same +# directory still reaches jllama transitively. Its bundled nlohmann/json is the +# same 3.12.0 as our FetchContent copy, so nothing added here shadows it. target_sources(jllama PRIVATE ${llama.cpp_SOURCE_DIR}/tools/server/server-http.cpp ${llama.cpp_SOURCE_DIR}/vendor/cpp-httplib/httplib.cpp @@ -564,6 +567,7 @@ if(BUILD_TESTING) src/test/cpp/test_json_helpers.cpp src/test/cpp/test_log_helpers.cpp src/test/cpp/test_tts_wav.cpp + src/test/cpp/test_tts_params.cpp ${llama.cpp_SOURCE_DIR}/tools/server/server-common.cpp ${llama.cpp_SOURCE_DIR}/tools/server/server-chat.cpp ${llama.cpp_SOURCE_DIR}/tools/server/server-context.cpp diff --git a/llama/cmake/apply-llama-patches.cmake b/llama/cmake/apply-llama-patches.cmake index 6c8c1293e..ddc92cb2b 100644 --- a/llama/cmake/apply-llama-patches.cmake +++ b/llama/cmake/apply-llama-patches.cmake @@ -12,11 +12,24 @@ # Windows (the dockcross/native/MSVC jobs all call the same code path). # * Every `patches/*.patch` and `patches/*.diff` is applied, sorted by filename (so a numeric # prefix like 0001-, 0002- defines a deterministic order). -# * Idempotent: `git apply --reverse --check` detects an already-applied patch and skips it, so -# a CMake reconfigure over an already-patched source tree does not fail. +# * Idempotent, via a stamp file rather than per-patch probing. The stamp +# (`${LLAMA_SRC}/.jllama-patches-applied`) records the checked-out llama.cpp commit plus the +# SHA-256 of every patch, and the decision is driven by whether the source tree is pristine: +# - clean tree -> nothing is applied yet (a fresh fetch, or a re-checkout after a version +# bump), so apply all patches forward and write the stamp; +# - dirty tree -> already patched; skip when the stamp matches this exact commit + patch +# set, and fail loudly when it does not. +# A per-patch `git apply --reverse --check` cannot do this: `--check` never mutates the tree, +# so an earlier patch whose region a later patch rewrote (today 0001 vs 0006/0007 in +# tools/server/server.cpp) always reverse-checks as "not applied" and the forward re-apply +# then aborts an otherwise harmless reconfigure. The stamp is state, but it is state derived +# from — and invalidated by — both inputs that matter. # * Fail-loud: a patch that no longer applies (e.g. after a llama.cpp version bump shifts the # context) aborts the configure with a clear message, so a stale patch can never be silently # dropped from a release build. +# * A source tree that is not a git work tree (e.g. supplied via +# `-DFETCHCONTENT_SOURCE_DIR_LLAMA.CPP=`) has no clean/dirty oracle and no HEAD, so it +# falls back to the legacy per-patch reverse-check path, with the caveat described above. # # Invoked as: # cmake -DPATCH_DIR=/patches -DLLAMA_SRC= -P cmake/apply-llama-patches.cmake @@ -38,20 +51,12 @@ if(NOT patch_files) return() endif() -foreach(patch IN LISTS patch_files) - get_filename_component(patch_name "${patch}" NAME) - - # Already applied? A successful reverse-apply check means the change is present already. - execute_process( - COMMAND "${GIT_EXECUTABLE}" -C "${LLAMA_SRC}" apply --reverse --check "${patch}" - RESULT_VARIABLE reverse_rc - OUTPUT_QUIET ERROR_QUIET) - if(reverse_rc EQUAL 0) - message(STATUS "apply-llama-patches: ${patch_name} already applied — skipping") - continue() - endif() +set(STAMP_NAME ".jllama-patches-applied") +set(stamp_file "${LLAMA_SRC}/${STAMP_NAME}") - # Not applied yet — confirm it applies cleanly before touching the tree. +# Applies one patch, aborting the configure when it no longer fits the source tree. +function(apply_one_patch patch) + get_filename_component(patch_name "${patch}" NAME) execute_process( COMMAND "${GIT_EXECUTABLE}" -C "${LLAMA_SRC}" apply --check "${patch}" RESULT_VARIABLE check_rc @@ -62,7 +67,6 @@ foreach(patch IN LISTS patch_files) " A llama.cpp version bump probably shifted the patched code — refresh the patch " "against the new source and recommit it.") endif() - execute_process( COMMAND "${GIT_EXECUTABLE}" -C "${LLAMA_SRC}" apply "${patch}" RESULT_VARIABLE apply_rc) @@ -70,4 +74,106 @@ foreach(patch IN LISTS patch_files) message(FATAL_ERROR "apply-llama-patches: failed to apply ${patch_name}") endif() message(STATUS "apply-llama-patches: applied ${patch_name}") +endfunction() + +# --------------------------------------------------------------------------- +# Is the source tree a git work tree? Without one there is no HEAD to pin the +# stamp to and no clean/dirty oracle, so fall back to the legacy behaviour. +# --------------------------------------------------------------------------- +execute_process( + COMMAND "${GIT_EXECUTABLE}" -C "${LLAMA_SRC}" rev-parse HEAD + RESULT_VARIABLE head_rc + OUTPUT_VARIABLE llama_head + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET) + +if(NOT head_rc EQUAL 0) + message(STATUS "apply-llama-patches: ${LLAMA_SRC} is not a git work tree — " + "using per-patch detection (a reconfigure over a patched tree may fail)") + foreach(patch IN LISTS patch_files) + get_filename_component(patch_name "${patch}" NAME) + execute_process( + COMMAND "${GIT_EXECUTABLE}" -C "${LLAMA_SRC}" apply --reverse --check "${patch}" + RESULT_VARIABLE reverse_rc + OUTPUT_QUIET ERROR_QUIET) + if(reverse_rc EQUAL 0) + message(STATUS "apply-llama-patches: ${patch_name} already applied — skipping") + continue() + endif() + apply_one_patch("${patch}") + endforeach() + return() +endif() + +# --------------------------------------------------------------------------- +# Build the manifest: the checked-out commit plus every patch's content hash. +# Any llama.cpp version bump changes HEAD; any patch edit changes a hash. +# --------------------------------------------------------------------------- +set(manifest "head ${llama_head}\n") +foreach(patch IN LISTS patch_files) + get_filename_component(patch_name "${patch}" NAME) + file(SHA256 "${patch}" patch_hash) + string(APPEND manifest "${patch_name} ${patch_hash}\n") endforeach() + +# --------------------------------------------------------------------------- +# Clean tree => nothing applied yet. Untracked files count as dirty (a future +# patch may add a file), except the stamp itself, which we write ourselves. +# --------------------------------------------------------------------------- +execute_process( + COMMAND "${GIT_EXECUTABLE}" -C "${LLAMA_SRC}" status --porcelain + RESULT_VARIABLE status_rc + OUTPUT_VARIABLE status_out + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET) +if(NOT status_rc EQUAL 0) + message(FATAL_ERROR "apply-llama-patches: 'git status' failed in ${LLAMA_SRC}") +endif() + +set(tree_is_dirty FALSE) +if(NOT status_out STREQUAL "") + string(REPLACE "\n" ";" status_lines "${status_out}") + foreach(line IN LISTS status_lines) + string(STRIP "${line}" line) + if(line STREQUAL "" OR line MATCHES "${STAMP_NAME}$") + continue() + endif() + set(tree_is_dirty TRUE) + break() + endforeach() +endif() + +if(NOT tree_is_dirty) + foreach(patch IN LISTS patch_files) + apply_one_patch("${patch}") + endforeach() + file(WRITE "${stamp_file}" "${manifest}") + return() +endif() + +# --------------------------------------------------------------------------- +# Dirty tree: already patched. Only a stamp matching this exact commit + patch +# set proves the modifications are ours and complete. +# --------------------------------------------------------------------------- +set(stamp_matches FALSE) +if(EXISTS "${stamp_file}") + file(READ "${stamp_file}" stamp_content) + if(stamp_content STREQUAL manifest) + set(stamp_matches TRUE) + endif() +endif() + +if(stamp_matches) + list(LENGTH patch_files patch_count) + message(STATUS "apply-llama-patches: ${patch_count} patch(es) already applied — skipping") + return() +endif() + +message(FATAL_ERROR + "apply-llama-patches: ${LLAMA_SRC} has local modifications that do not match the current " + "patch set.\n" + " Patches cannot be applied on top of an already-patched tree, and the previous state is " + "unknown (the tree was patched with a different patch set or llama.cpp commit, or edited by " + "hand).\n" + " Configure into a FRESH build directory so FetchContent re-checks-out a pristine " + "llama.cpp, then build again.") diff --git a/llama/patches/0001-win32-arg-parse-embed-guard.patch b/llama/patches/0001-win32-arg-parse-embed-guard.patch index 54a779f41..8527089ad 100644 --- a/llama/patches/0001-win32-arg-parse-embed-guard.patch +++ b/llama/patches/0001-win32-arg-parse-embed-guard.patch @@ -313,19 +313,6 @@ index 8e2eace6a..d7a7afed9 100644 return 1; } -diff --git a/tests/test-save-load-state.cpp b/tests/test-save-load-state.cpp -index 6e93ce6fb..9caab1653 100644 ---- a/tests/test-save-load-state.cpp -+++ b/tests/test-save-load-state.cpp -@@ -358,7 +358,7 @@ int main(int argc, char ** argv) { - - common_init(); - -- if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) { -+ if (!common_params_parse_main(argc, argv, params, LLAMA_EXAMPLE_COMMON)) { - return 1; - } - diff --git a/tests/test-state-restore-fragmented.cpp b/tests/test-state-restore-fragmented.cpp index d5548afba..95ba2b67b 100644 --- a/tests/test-state-restore-fragmented.cpp @@ -508,6 +495,19 @@ index a3b2a8b0f..80d6a3ff6 100644 return 1; } +diff --git a/tools/tokenize/tokenize.cpp b/tools/tokenize/tokenize.cpp +index 77b33c4a4..834a410c9 100644 +--- a/tools/tokenize/tokenize.cpp ++++ b/tools/tokenize/tokenize.cpp +@@ -99,7 +99,7 @@ int main(int argc, char ** argv) { + + common_init(); + +- if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_TOKENIZE, print_usage)) { ++ if (!common_params_parse_main(argc, argv, params, LLAMA_EXAMPLE_TOKENIZE, print_usage)) { + return 1; + } + diff --git a/tools/tts/tts.cpp b/tools/tts/tts.cpp index 2a1bdccc9..3bd25a246 100644 --- a/tools/tts/tts.cpp diff --git a/llama/patches/0002-server-preserve-caller-load-progress-callback.patch b/llama/patches/0002-server-preserve-caller-load-progress-callback.patch index 4ff5fc5c1..ef6b9206c 100644 --- a/llama/patches/0002-server-preserve-caller-load-progress-callback.patch +++ b/llama/patches/0002-server-preserve-caller-load-progress-callback.patch @@ -2,7 +2,7 @@ diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 39b7eb2..bc73429 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp -@@ -1152,8 +1152,16 @@ private: +@@ -1152,8 +1152,26 @@ private: // attach a progress callback { @@ -14,7 +14,17 @@ index 39b7eb2..bc73429 100644 + // common_params before server_context::load_model). Only install the server's + // reporter when the caller has not supplied one, so a caller-provided callback + // survives and fires during model load. -+ if (params_base.load_progress_callback == nullptr) { ++ // ++ // The second disjunct is load-bearing and must not be simplified to a nullptr check: ++ // `load_progress_text` is a LOCAL of this function, and upstream re-assigns both fields ++ // on every call, so the user_data always points at the current frame. load_model() runs ++ // a SECOND time when resuming from the sleeping state, and by then params_base holds ++ // OUR OWN callback from the first load -- a bare nullptr check skips the re-assignment ++ // and leaves user_data pointing into a dead stack frame, which segfaults in ++ // load_progress_callback() on the first request after an idle window. So: install ours ++ // unless the caller installed something that is not ours. ++ if (params_base.load_progress_callback == nullptr || ++ params_base.load_progress_callback == load_progress_callback) { + params_base.load_progress_callback = load_progress_callback; + params_base.load_progress_callback_user_data = &load_progress_text; + } diff --git a/llama/patches/0007-server-attach-http-frontend.patch b/llama/patches/0007-server-attach-http-frontend.patch index ebc03b7d8..ec80ef913 100644 --- a/llama/patches/0007-server-attach-http-frontend.patch +++ b/llama/patches/0007-server-attach-http-frontend.patch @@ -1,5 +1,5 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp -index ce1d239c3..b242fc48a 100644 +index 102692c74..8389fe6bf 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -63,6 +63,7 @@ int llama_server(int argc, char ** argv); @@ -27,8 +27,8 @@ index ce1d239c3..b242fc48a 100644 + ctx_http.get ("/metrics", ex_wrapper(routes.get_metrics)); + ctx_http.get ("/props", ex_wrapper(routes.get_props)); + ctx_http.post("/props", ex_wrapper(routes.post_props)); -+ ctx_http.get ("/models", ex_wrapper(routes.get_models)); // public endpoint (no API key check) -+ ctx_http.get ("/v1/models", ex_wrapper(routes.get_models)); // public endpoint (no API key check) ++ ctx_http.get ("/models", ex_wrapper(routes.get_models)); ++ ctx_http.get ("/v1/models", ex_wrapper(routes.get_models)); + ctx_http.post("/completion", ex_wrapper(routes.post_completions)); // legacy + ctx_http.post("/completions", ex_wrapper(routes.post_completions)); + ctx_http.post("/v1/completions", ex_wrapper(routes.post_completions_oai)); @@ -68,7 +68,7 @@ index ce1d239c3..b242fc48a 100644 int llama_server(int argc, char ** argv) { std::setlocale(LC_NUMERIC, "C"); -@@ -258,47 +310,7 @@ int llama_server(common_params & params, int argc, char ** argv) { +@@ -259,47 +311,7 @@ int llama_server(common_params & params, int argc, char ** argv) { ctx_http.del ("/models", ex_wrapper(models_routes->del_router_models)); } @@ -77,8 +77,8 @@ index ce1d239c3..b242fc48a 100644 - ctx_http.get ("/metrics", ex_wrapper(routes.get_metrics)); - ctx_http.get ("/props", ex_wrapper(routes.get_props)); - ctx_http.post("/props", ex_wrapper(routes.post_props)); -- ctx_http.get ("/models", ex_wrapper(routes.get_models)); // public endpoint (no API key check) -- ctx_http.get ("/v1/models", ex_wrapper(routes.get_models)); // public endpoint (no API key check) +- ctx_http.get ("/models", ex_wrapper(routes.get_models)); +- ctx_http.get ("/v1/models", ex_wrapper(routes.get_models)); - ctx_http.post("/completion", ex_wrapper(routes.post_completions)); // legacy - ctx_http.post("/completions", ex_wrapper(routes.post_completions)); - ctx_http.post("/v1/completions", ex_wrapper(routes.post_completions_oai)); @@ -117,7 +117,7 @@ index ce1d239c3..b242fc48a 100644 // resumable streaming: a child binds the local session factories, the router binds // proxies that resolve the owning child, see server-stream.h -@@ -556,3 +568,89 @@ int llama_server(common_params & params, int argc, char ** argv) { +@@ -579,3 +591,89 @@ int llama_server(common_params & params, int argc, char ** argv) { return 0; } diff --git a/llama/patches/0010-server-cast-vocab-type-for-common-json.patch b/llama/patches/0010-server-cast-vocab-type-for-common-json.patch new file mode 100644 index 000000000..24a8b4f90 --- /dev/null +++ b/llama/patches/0010-server-cast-vocab-type-for-common-json.patch @@ -0,0 +1,16 @@ +diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp +index 15fa8a498..f2fbe2be8 100644 +--- a/tools/server/server-context.cpp ++++ b/tools/server/server-context.cpp +@@ -4439,7 +4439,10 @@ static json get_res_model_info(const server_context_meta & meta) { + {"created", std::time(0)}, + {"owned_by", "llamacpp"}, + {"meta", { +- {"vocab_type", meta.model_vocab_type}, ++ // an unscoped enum has no common_json_value ctor of its own (the integral one is ++ // is_integral-gated, which excludes enums), so it binds to common_json_value(bool) ++ // and serialises as true/false -- cast it to keep the numeric vocab type on the wire ++ {"vocab_type", (int) meta.model_vocab_type}, + {"n_vocab", meta.model_vocab_n_tokens}, + {"n_ctx", meta.slot_n_ctx}, + {"n_ctx_train", meta.model_n_ctx_train}, diff --git a/llama/patches/0011-peg-parser-lenient-invalid-utf8.patch b/llama/patches/0011-peg-parser-lenient-invalid-utf8.patch new file mode 100644 index 000000000..0f2d96d03 --- /dev/null +++ b/llama/patches/0011-peg-parser-lenient-invalid-utf8.patch @@ -0,0 +1,74 @@ +diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp +index 46fc29bf2..3f2008217 100644 +--- a/common/peg-parser.cpp ++++ b/common/peg-parser.cpp +@@ -680,7 +680,16 @@ struct parser_executor { + + if (utf8_result.status == utf8_parse_result::INVALID) { + // Malformed UTF-8 +- return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); ++ if (!ctx.is_lenient()) { ++ return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); ++ } ++ // Lenient: keep what was scanned before the malformed byte instead of failing the ++ // whole parse, mirroring the INCOMPLETE branch above. Failing here loses a result ++ // that was produced successfully: common_chat_peg_parse() always parses in lenient ++ // mode, and the server runs it over every completion (content-only when the request ++ // configures no chat parser), so a single stray byte anywhere in the generated text ++ // turns a finished generation into an HTTP 500 instead of a response. ++ return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, last_valid_pos); + } + + // Check if a delimiter starts at this position +diff --git a/tests/peg-parser/test-unicode.cpp b/tests/peg-parser/test-unicode.cpp +index 24663d701..9ba64b519 100644 +--- a/tests/peg-parser/test-unicode.cpp ++++ b/tests/peg-parser/test-unicode.cpp +@@ -238,6 +238,47 @@ void test_unicode(testing &t) { + } + }); + ++ t.test("invalid UTF-8 is tolerated when lenient", [](testing &t) { ++ // A malformed byte must not fail the whole parse in lenient mode: common_chat_peg_parse() ++ // always parses leniently and the server runs it over every completion, so failing here ++ // would turn a finished generation into an error response. Keep what was scanned before ++ // the bad byte instead, exactly like the incomplete-sequence case above. ++ std::vector test_cases { ++ // Lone continuation byte in the middle ++ {std::string("Hello\x80World"), "Hello", COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT}, ++ ++ // Truncated CJK sequence followed by more bytes ++ {std::string("ab\xE4\xB8cd"), "ab", COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT}, ++ ++ // Invalid lead byte ++ {std::string("abc\xFF" "d"), "abc", COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT}, ++ }; ++ ++ auto parser = build_peg_parser([](common_peg_parser_builder& p) { ++ return p.until(""); ++ }); ++ ++ for (size_t i = 0; i < test_cases.size(); i++) { ++ const auto & tc = test_cases[i]; ++ std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); ++ ++ t.test(test_name, [&](testing &t) { ++ common_peg_parse_context lenient(tc.input, COMMON_PEG_PARSE_FLAG_LENIENT); ++ auto result = parser.parse(lenient); ++ ++ assert_result_equal(t, tc.expected_result, result.type); ++ ++ std::string matched = tc.input.substr(result.start, result.end - result.start); ++ t.assert_equal(tc.expected_text, matched); ++ ++ // Strict mode still rejects malformed input. ++ common_peg_parse_context strict(tc.input); ++ auto strict_result = parser.parse(strict); ++ assert_result_equal(t, COMMON_PEG_PARSE_RESULT_FAIL, strict_result.type); ++ }); ++ } ++ }); ++ + t.test("incomplete UTF-8 at end", [](testing &t) { + std::vector test_cases { + // Incomplete emoji at end, no delimiter diff --git a/llama/spotbugs-exclude.xml b/llama/spotbugs-exclude.xml index 02e3dcbe6..95d2e2756 100644 --- a/llama/spotbugs-exclude.xml +++ b/llama/spotbugs-exclude.xml @@ -60,9 +60,9 @@ SPDX-License-Identifier: MIT