From ef3bcfd5eabd1ece281dafa61b8d8fc65c4cc501 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 18 Aug 2026 07:38:29 +0000 Subject: [PATCH 1/7] diag(#1157): instrument the NemotronH recurrent carry and add a fresh-prefill discriminator The A3 gate diverges on real weights while the CPU gate is 12/12 green, so neither arm can say whether the decode step reads the state the previous step wrote. Two instruments, both off by default. `VT_NEMOTRON_H_DIAG` prints, per step, the decode/prefill split, the recurrent slot indices, the has-initial mask, and per Mamba2 layer the L2 of the state gathered in and the state written out. On the CPU fixture it reads a healthy carry, which is what makes it usable as a negative control on the device. `nemotron-h-gen --fresh-prefill` generates the same stream one token per completion from a growing prompt, so every token comes out of a prefill and nothing is carried. Same engine, same weights, same public entry point, so a stream that is right this way and wrong the normal way names the carry. This is scaffolding for the #1157 measurement, not the repair. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: AGENT:claude-opus-5 [claude-code] --- examples/nemotron_h_gen/main.cpp | 47 ++++++++++++- .../models/nemotron_h_device.cpp | 69 +++++++++++++++++++ 2 files changed, 113 insertions(+), 3 deletions(-) diff --git a/examples/nemotron_h_gen/main.cpp b/examples/nemotron_h_gen/main.cpp index 605f29bca..fb08d43b2 100644 --- a/examples/nemotron_h_gen/main.cpp +++ b/examples/nemotron_h_gen/main.cpp @@ -190,6 +190,18 @@ int main(int argc, char** argv) { // entry count and per-entry widths — is the number every count assertion // below is measured against. bool golden_info = false; + // ── #1157 DISCRIMINATOR ──────────────────────────────────────────────────── + // Generate the SAME token stream WITHOUT ever taking a decode step: ask for + // one token at a time, each from a FRESH request whose prompt is the original + // prompt plus every token generated so far. Every token then comes out of a + // PREFILL, and the recurrent state and paged KV each start empty. + // + // It is the same engine, the same weights and the same public entry point, so + // the two runs differ in exactly one thing: whether a token was produced by + // continuing a sequence or by recomputing it. A stream that is right this way + // and wrong the normal way localises the defect to the CARRY and rules out + // the tower; a stream that is wrong BOTH ways rules the carry out instead. + bool fresh_prefill = false; for (int i = 1; i < argc; ++i) { const std::string a = argv[i]; auto next = [&]() -> const char* { return (i + 1 < argc) ? argv[++i] : ""; }; @@ -200,6 +212,7 @@ int main(int argc, char** argv) { else if (a == "--max-model-len") max_model_len = std::atoi(next()); else if (a == "--load-only") load_only = true; else if (a == "--golden-info") golden_info = true; + else if (a == "--fresh-prefill") fresh_prefill = true; else { std::fprintf(stderr, "unknown arg %s\n", a.c_str()); return 2; } } if (model.empty() && !golden_info) { @@ -296,17 +309,44 @@ int main(int argc, char** argv) { std::vector gen(static_cast(steps), 0); int32_t n_gen = 0; const auto ts = std::chrono::steady_clock::now(); + if (fresh_prefill) { + // One token per completion, from a prompt that grows by the token the + // previous completion returned. `n_gen` still counts what THIS driver + // produced, so every count assertion below is unchanged. + std::vector ctx = e.prompt_token_ids; + vllm_sampling_params one = sp; + one.max_tokens = 1; + for (int t = 0; t < steps; ++t) { + int32_t got = 0, n_one = 0; + const vllm_status s1 = vllm_complete_tokens( + eng, ctx.data(), static_cast(ctx.size()), &one, &got, 1, + &n_one, nullptr); + if (s1 != VLLM_OK || n_one != 1) { + std::fprintf(stderr, + "[nemotron-h] prompt %d fresh-prefill step %d FAILED " + "(status=%d n=%d): %s\n", + pi, t, static_cast(s1), static_cast(n_one), + vllm_last_error()); + vllm_engine_free(eng); + return 1; + } + gen[static_cast(t)] = got; + ctx.push_back(got); + ++n_gen; + } + } else { const vllm_status st = vllm_complete_tokens( eng, e.prompt_token_ids.data(), static_cast(e.prompt_token_ids.size()), &sp, gen.data(), static_cast(gen.size()), &n_gen, nullptr); - const auto te = std::chrono::steady_clock::now(); if (st != VLLM_OK) { std::fprintf(stderr, "[nemotron-h] prompt %d FAILED: %s\n", pi, vllm_last_error()); vllm_engine_free(eng); return 1; } + } + const auto te = std::chrono::steady_clock::now(); const int expected = static_cast(e.token_ids.size()); const int n = std::min(expected, static_cast(n_gen)); @@ -336,8 +376,9 @@ int main(int argc, char** argv) { std::fprintf(stderr, "\n[nemotron-h] TOKEN MATCH: %d/%d over %d prompt(s) " - "(full rows=%d, short rows=%d)\n", - total_matched, total_compared, n_prompts, rows_full, rows_short); + "(full rows=%d, short rows=%d, mode=%s)\n", + total_matched, total_compared, n_prompts, rows_full, rows_short, + fresh_prefill ? "fresh-prefill" : "decode"); vllm_engine_free(eng); // A pass needs three things to be true at once, and each is checked here diff --git a/src/vllm/model_executor/models/nemotron_h_device.cpp b/src/vllm/model_executor/models/nemotron_h_device.cpp index 4bd6ad1cd..3651ab004 100644 --- a/src/vllm/model_executor/models/nemotron_h_device.cpp +++ b/src/vllm/model_executor/models/nemotron_h_device.cpp @@ -79,6 +79,7 @@ #include #include +#include #include #include #include @@ -1111,6 +1112,30 @@ std::vector OwnedToF32(const NemotronHOwned& w) { return out; } +// ─── #1157 DIAGNOSTIC SCAFFOLD (VT_NEMOTRON_H_DIAG) ───────────────────────── +// +// TEMPORARY. Off unless `VT_NEMOTRON_H_DIAG` is set to something other than +// "0", and every download it does is inside that guard, so a production step +// pays nothing. It exists to answer ONE question the CPU gate cannot: on the +// real checkpoint, is the recurrent state the decode step READS the state the +// previous step WROTE. +bool NemotronHDiagEnabled() { + static const bool on = [] { + const char* e = std::getenv("VT_NEMOTRON_H_DIAG"); + return e != nullptr && e[0] != '0'; + }(); + return on; +} + +double DiagL2(const std::vector& v, int64_t off, int64_t n) { + double acc = 0.0; + for (int64_t i = 0; i < n; ++i) { + const double x = v[static_cast(off + i)]; + acc += x * x; + } + return std::sqrt(acc); +} + // ─── the per-step device inputs ───────────────────────────────────────────── // // Uploaded ONCE per step and shared by all 6 attention layers and all 23 @@ -1213,6 +1238,24 @@ NemotronHPagedStep BuildNemotronHPagedStep(Dev d, const ModelForwardInput& input // for the same hazard and the same remedy). Waiting here costs nothing this // unit measures: A2-P records no throughput number on any axis (spec §5). d.b.Synchronize(d.q); + if (NemotronHDiagEnabled()) { + std::fprintf(stderr, "[NH-DIAG] step T=%lld R=%lld nd=%lld np=%lld idx=[", + static_cast(T), static_cast(R), + static_cast(nd), static_cast(np)); + for (int64_t r = 0; r < R; ++r) + std::fprintf(stderr, "%d%s", idx[static_cast(r)], r + 1 < R ? "," : ""); + std::fprintf(stderr, "] init=["); + for (int64_t r = 0; r < R; ++r) + std::fprintf(stderr, "%d%s", init[static_cast(r)], r + 1 < R ? "," : ""); + std::fprintf(stderr, "] qsl_attn=["); + for (size_t i = 0; i < am.query_start_loc.size(); ++i) + std::fprintf(stderr, "%d%s", am.query_start_loc[i], + i + 1 < am.query_start_loc.size() ? "," : ""); + std::fprintf(stderr, "] seq_lens=["); + for (size_t i = 0; i < am.seq_lens.size(); ++i) + std::fprintf(stderr, "%d%s", am.seq_lens[i], i + 1 < am.seq_lens.size() ? "," : ""); + std::fprintf(stderr, "]\n"); + } return sdi; } @@ -1540,6 +1583,12 @@ ForwardLogits NemotronHPagedForward(const NemotronHHostWeights& host, params.mamba_num_heads * params.mamba_head_dim * params.ssm_state_size; std::vector conv_all = DownloadF32(d, io.conv, DType::kF32, R * conv_row); std::vector ssm_all = DownloadF32(d, io.ssm, DType::kF32, R * ssm_row); + if (NemotronHDiagEnabled()) { + std::fprintf(stderr, + "[NH-DIAG] L%lld mamba GATHERED |conv|=%.6g |ssm|=%.6g\n", + static_cast(l), DiagL2(conv_all, 0, conv_row), + DiagL2(ssm_all, 0, ssm_row)); + } // At `num_reqs == 1` this loop runs once, and it is written as a loop for // the reason §4.1 gives: the indexing machinery lands here, only the @@ -1587,6 +1636,14 @@ ForwardLogits NemotronHPagedForward(const NemotronHHostWeights& host, ssm_all.begin() + static_cast(r * ssm_row)); } + if (NemotronHDiagEnabled()) { + std::fprintf(stderr, + "[NH-DIAG] L%lld mamba WROTE |conv|=%.6g |ssm|=%.6g " + "|out|=%.6g\n", + static_cast(l), DiagL2(conv_all, 0, conv_row), + DiagL2(ssm_all, 0, ssm_row), + DiagL2(mvec, (T - 1) * H, H)); + } io.conv = UploadAs(d, conv_all, DType::kF32, {R, params.conv_dim(), params.conv_kernel - 1}); io.ssm = UploadAs(d, ssm_all, DType::kF32, @@ -1610,6 +1667,18 @@ ForwardLogits NemotronHPagedForward(const NemotronHHostWeights& host, carry = UploadAs(d, mvec, adt, {T, H}); } + if (NemotronHDiagEnabled()) { + const std::vector cv = DownloadF32(d, carry, adt, T * H); + const std::vector rs = DownloadF32(d, residual, adt, T * H); + const char* kind = lw.block == NemotronHBlock::kAttention ? "attn" + : lw.block == NemotronHBlock::kMamba ? "mamba" + : lw.block == NemotronHBlock::kMoe ? "moe" + : "mlp"; + std::fprintf(stderr, + "[NH-DIAG] L%lld %-5s |mixer_last|=%.6g |resid_last|=%.6g\n", + static_cast(l), kind, DiagL2(cv, (T - 1) * H, H), + DiagL2(rs, (T - 1) * H, H)); + } if (trace != nullptr && trace->capture) { trace->normed[static_cast(l)] = std::move(nvec); std::vector h = DownloadF32(d, residual, adt, T * H); From ed049a7d490f615a621e88a96b724464dec1b1f8 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 18 Aug 2026 08:20:55 +0000 Subject: [PATCH 2/7] diag(#1157): a device-MoE bypass and a both-modes driver, so one GPU slot can bisect The CPU arm now decodes this checkpoint token-exact against the oracle golden, so the divergence is on the device side and the question is which device arm. The MoE block is 23 of the model's 52 layers and its own gate exercises T=4 and T=2 only, never the T=1 a decode step is, so it is the arm with the least coverage at the shape that fails. `VT_NEMOTRON_H_DEVICE_MOE=0` routes those layers back through the host reference the CPU run proves correct, and `--both-modes` runs the decode and fresh-prefill streams over ONE engine load, which is the only affordable shape when a load is minutes long. Both default to today's behaviour, so nothing moves unless a diagnostic asks it to. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: AGENT:claude-opus-5 [claude-code] --- examples/nemotron_h_gen/main.cpp | 14 ++++++++++++++ .../model_executor/models/nemotron_h_device.cpp | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/examples/nemotron_h_gen/main.cpp b/examples/nemotron_h_gen/main.cpp index fb08d43b2..448c972f9 100644 --- a/examples/nemotron_h_gen/main.cpp +++ b/examples/nemotron_h_gen/main.cpp @@ -202,6 +202,10 @@ int main(int argc, char** argv) { // and wrong the normal way localises the defect to the CARRY and rules out // the tower; a stream that is wrong BOTH ways rules the carry out instead. bool fresh_prefill = false; + // Run BOTH modes over one engine load, which is the only affordable shape + // when a load is minutes long: the two streams then differ in nothing except + // whether a token came from a decode step or from a re-prefill. + bool both_modes = false; for (int i = 1; i < argc; ++i) { const std::string a = argv[i]; auto next = [&]() -> const char* { return (i + 1 < argc) ? argv[++i] : ""; }; @@ -213,6 +217,7 @@ int main(int argc, char** argv) { else if (a == "--load-only") load_only = true; else if (a == "--golden-info") golden_info = true; else if (a == "--fresh-prefill") fresh_prefill = true; + else if (a == "--both-modes") both_modes = true; else { std::fprintf(stderr, "unknown arg %s\n", a.c_str()); return 2; } } if (model.empty() && !golden_info) { @@ -304,6 +309,14 @@ int main(int argc, char** argv) { : static_cast(gold.entries.size()); int total_compared = 0, total_matched = 0, rows_full = 0, rows_short = 0; + const int n_modes = both_modes ? 2 : 1; + for (int mi = 0; mi < n_modes; ++mi) { + if (both_modes) { + fresh_prefill = (mi == 1); + total_compared = 0; total_matched = 0; rows_full = 0; rows_short = 0; + std::fprintf(stderr, "\n[nemotron-h] ===== MODE %s =====\n", + fresh_prefill ? "fresh-prefill" : "decode"); + } for (int pi = 0; pi < n_prompts; ++pi) { const GoldenEntry& e = gold.entries[static_cast(pi)]; std::vector gen(static_cast(steps), 0); @@ -379,6 +392,7 @@ int main(int argc, char** argv) { "(full rows=%d, short rows=%d, mode=%s)\n", total_matched, total_compared, n_prompts, rows_full, rows_short, fresh_prefill ? "fresh-prefill" : "decode"); + } vllm_engine_free(eng); // A pass needs three things to be true at once, and each is checked here diff --git a/src/vllm/model_executor/models/nemotron_h_device.cpp b/src/vllm/model_executor/models/nemotron_h_device.cpp index 3651ab004..d991b70b7 100644 --- a/src/vllm/model_executor/models/nemotron_h_device.cpp +++ b/src/vllm/model_executor/models/nemotron_h_device.cpp @@ -1127,6 +1127,19 @@ bool NemotronHDiagEnabled() { return on; } +// #1157 BISECT SWITCH. The device MoE arm is 23 of this model's 52 layers and +// has never run at T=1 anywhere: its own gate (test_nemotron_h_moe_device.cpp) +// exercises T=4 and T=2. Setting `VT_NEMOTRON_H_DEVICE_MOE=0` routes those +// layers back through the host reference the CPU arm already proves token-exact +// on this checkpoint, so one run says whether the device MoE is the difference. +bool NemotronHDeviceMoeEnabled() { + static const bool on = [] { + const char* e = std::getenv("VT_NEMOTRON_H_DEVICE_MOE"); + return e == nullptr || e[0] != '0'; + }(); + return on; +} + double DiagL2(const std::vector& v, int64_t off, int64_t n) { double acc = 0.0; for (int64_t i = 0; i < n; ++i) { @@ -1555,6 +1568,7 @@ ForwardLogits NemotronHPagedForward(const NemotronHHostWeights& host, const bool moe_on_device = lw.block == NemotronHBlock::kMoe && adt == DType::kBF16 && MoeIsNvfp4(lw.moe) && + NemotronHDeviceMoeEnabled() && vt::OpRegistered(vt::OpId::kMoeGroupedGemmNvfp4Marlin, d.q.device.type); const bool needs_host = lw.block != NemotronHBlock::kAttention && !moe_on_device; std::vector nvec; From a75ab96822636f1a4cc6da706afa72219cee6db9 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 18 Aug 2026 08:43:17 +0000 Subject: [PATCH 3/7] test(#1157): the device MoE gate covered two PREFILL widths and never the decode one `NemotronHMoeBlockDevice` runs on 23 of this model's 52 layers, and its gate measured T=4 and T=2. Both are prefill shapes. Every token after the first comes out of a step carrying exactly one token, so the width the model spends its entire decode in was the width nothing measured. It is not a cosmetic gap. `MarlinMoeAlignBlockSizeSelect` and `MarlinMoeAlignSizes` branch on the token count relative to the expert count, and T=1 against 128 experts is on the other side of that branch from T=4. A width loop rather than a third copy, so the three cannot drift, with the covered count asserted afterwards: a loop that ran over nothing would otherwise report a clean pass, which is the shape this tree keeps finding. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: AGENT:claude-opus-5 [claude-code] --- .../vllm/models/test_nemotron_h_moe_device.cpp | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/vllm/models/test_nemotron_h_moe_device.cpp b/tests/vllm/models/test_nemotron_h_moe_device.cpp index 95fcce4c5..181f070cb 100644 --- a/tests/vllm/models/test_nemotron_h_moe_device.cpp +++ b/tests/vllm/models/test_nemotron_h_moe_device.cpp @@ -227,10 +227,22 @@ TEST_CASE("NemotronH A2-Q2a: the device MoE block matches the host reference on const NemotronHParams p = MoeParams(); Queue hq{Device{DeviceType::kCPU, 0}, nullptr}; const DType dt = DType::kBF16; // Marlin's a/c contract (ops.cpp:879) - const int64_t T = 4; const int64_t H = p.hidden_size; const NemotronHMoeWeights w = MakeNvfp4Moe(p, dt); + + // ★ T == 1 IS THE DECODE SHAPE, AND UNTIL #1157 THIS CASE NEVER RAN IT. + // The widths here were 4 and 2, both of them PREFILL shapes. Every token + // after the first comes out of a step with exactly one token, so the arm this + // model spends its whole decode in was the one width the gate did not cover — + // and `MarlinMoeAlignBlockSizeSelect` / `MarlinMoeAlignSizes` take different + // branches at a token count below the expert count, which is what T=1 with + // 128 experts is. A width loop rather than a third copy, so the three cannot + // drift apart, and the count is asserted afterwards: a loop that ran over + // nothing would otherwise report a clean pass. + int64_t widths_covered = 0; + for (const int64_t T : {int64_t{1}, int64_t{2}, int64_t{4}}) { + INFO("token count T=" << T); const std::vector x = SynthVec(static_cast(T * H), 77, 0.5F); // The HOST arm dequantizes each touched expert to bf16 and runs the per-pair @@ -301,6 +313,10 @@ TEST_CASE("NemotronH A2-Q2a: the device MoE block matches the host reference on REQUIRE(guard_examined == examined); INFO("does the band " << band << " REJECT a routed-scale defect?"); CHECK(guard >= band); + ++widths_covered; + } + // Three widths, or the loop did not run the one this case was extended for. + REQUIRE(widths_covered == 3); } // A separate case so a `-tc` run can select it alone. Same no-comma rule. From 65b0759b0dff12ea497d0e3d1b07f185e0ab7ca3 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 18 Aug 2026 08:56:00 +0000 Subject: [PATCH 4/7] test(#1157): the FA-2 d128 decode gate measured two GQA ratios, and the failing model is a third MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `fa2_decode_qwen3` is DEFAULT ON for any bf16 causal pure-decode at head_dim 128. Its own comment scopes it to "only Qwen3-dense hits this", and that stopped being true when NemotronH-3.5-Lightning landed: it is head_dim 128 as well, and its decode goes through the same launcher. Every case in this file measured 16/8 and 32/8 — ngroups 2 and 4, the two Qwen3-dense gate configs. NemotronH is 32 query heads over 2 KV heads, ngroups 16, four times the widest group count the swapped presentation was ever measured at. The launcher packs ngroups as seqlen_q, so the group count is not a detail of the geometry, it is the grid. The ratio is added to all five d128 cases rather than to one, so the prefill arm, the plain-varlen decode, the group-swap decode, the swap-vs-plain near-tie and the num_splits cap all see it. Nothing else moves. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: AGENT:claude-opus-5 [claude-code] --- tests/vt/test_ops_paged_attn.cpp | 60 +++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 5 deletions(-) diff --git a/tests/vt/test_ops_paged_attn.cpp b/tests/vt/test_ops_paged_attn.cpp index f29deed4f..3fa0ab395 100644 --- a/tests/vt/test_ops_paged_attn.cpp +++ b/tests/vt/test_ops_paged_attn.cpp @@ -1196,8 +1196,18 @@ TEST_CASE("paged_attention CUDA FA-2 prefill (bf16 q/kv/out) matches f32 ref at // kBlockN=128 for d128: covers up to ceil(140/128)*128 = 256 keys = 16 pages; // size the block table to that so no column is read past its row (memcheck). const int64_t max_blocks = 16, num_blocks = 64; + // ★ {32, 2} IS NEMOTRON-3.5-LIGHTNING, AND IT WAS NOT HERE (#1157). + // Every case in this file measured ngroups 2 (16/8) and 4 (32/8), the two + // Qwen3-dense gate configs the varlen d128 decode path was written for. The + // path is DEFAULT ON for ANY bf16 causal pure-decode at head_dim 128, so the + // first model to arrive with a different ratio reaches an untested grid — and + // NemotronH-3.5-Lightning-30B is 32 query heads over 2 KV heads, ngroups 16, + // four times the widest group count ever measured here. Its A3 token gate + // failed 6/96 on the device while the same binary decodes the same checkpoint + // token-exact on CPU, which is what sent the search here. for (const auto& ratio : {std::pair{16, 8}, - std::pair{32, 8}}) { + std::pair{32, 8}, + std::pair{32, 2}}) { const int64_t Hq = ratio.first, Hk = ratio.second, page = Hk * D; CAPTURE(Hq); auto qf = RandF32(static_cast(num_tokens * Hq * D), @@ -1569,8 +1579,18 @@ TEST_CASE("paged_attention CUDA FA-2 varlen d128 decode matches composed referen MESSAGE("no CUDA backend; skipping FA-2 varlen d128 decode parity (dgx-pending)"); return; } + // ★ {32, 2} IS NEMOTRON-3.5-LIGHTNING, AND IT WAS NOT HERE (#1157). + // Every case in this file measured ngroups 2 (16/8) and 4 (32/8), the two + // Qwen3-dense gate configs the varlen d128 decode path was written for. The + // path is DEFAULT ON for ANY bf16 causal pure-decode at head_dim 128, so the + // first model to arrive with a different ratio reaches an untested grid — and + // NemotronH-3.5-Lightning-30B is 32 query heads over 2 KV heads, ngroups 16, + // four times the widest group count ever measured here. Its A3 token gate + // failed 6/96 on the device while the same binary decodes the same checkpoint + // token-exact on CPU, which is what sent the search here. for (const auto& ratio : {std::pair{16, 8}, - std::pair{32, 8}}) { + std::pair{32, 8}, + std::pair{32, 2}}) { for (const int batch : {1, 2, 4, 8}) { for (const int base_len : {5, 21, 1024}) { // short => num_splits==1; long => split CAPTURE(ratio.first); @@ -1615,8 +1635,18 @@ TEST_CASE("paged_attention CUDA FA-2 varlen d128 decode GQA group-swap matches c MESSAGE("no CUDA backend; skipping FA-2 varlen d128 group-swap parity (dgx-pending)"); return; } + // ★ {32, 2} IS NEMOTRON-3.5-LIGHTNING, AND IT WAS NOT HERE (#1157). + // Every case in this file measured ngroups 2 (16/8) and 4 (32/8), the two + // Qwen3-dense gate configs the varlen d128 decode path was written for. The + // path is DEFAULT ON for ANY bf16 causal pure-decode at head_dim 128, so the + // first model to arrive with a different ratio reaches an untested grid — and + // NemotronH-3.5-Lightning-30B is 32 query heads over 2 KV heads, ngroups 16, + // four times the widest group count ever measured here. Its A3 token gate + // failed 6/96 on the device while the same binary decodes the same checkpoint + // token-exact on CPU, which is what sent the search here. for (const auto& ratio : {std::pair{16, 8}, - std::pair{32, 8}}) { + std::pair{32, 8}, + std::pair{32, 2}}) { for (const int batch : {1, 2, 4, 8}) { for (const int base_len : {5, 21, 1024}) { // short => num_splits==1; long => split CAPTURE(ratio.first); @@ -1645,8 +1675,18 @@ TEST_CASE("paged_attention CUDA FA-2 varlen d128 decode swap near-ties the plain MESSAGE("no CUDA backend; skipping FA-2 varlen d128 swap-vs-plain near-tie (dgx-pending)"); return; } + // ★ {32, 2} IS NEMOTRON-3.5-LIGHTNING, AND IT WAS NOT HERE (#1157). + // Every case in this file measured ngroups 2 (16/8) and 4 (32/8), the two + // Qwen3-dense gate configs the varlen d128 decode path was written for. The + // path is DEFAULT ON for ANY bf16 causal pure-decode at head_dim 128, so the + // first model to arrive with a different ratio reaches an untested grid — and + // NemotronH-3.5-Lightning-30B is 32 query heads over 2 KV heads, ngroups 16, + // four times the widest group count ever measured here. Its A3 token gate + // failed 6/96 on the device while the same binary decodes the same checkpoint + // token-exact on CPU, which is what sent the search here. for (const auto& ratio : {std::pair{16, 8}, - std::pair{32, 8}}) { + std::pair{32, 8}, + std::pair{32, 2}}) { for (const int batch : {2, 8}) { for (const int base_len : {21, 1024}) { CAPTURE(ratio.first); @@ -1691,8 +1731,18 @@ TEST_CASE("paged_attention CUDA FA-2 varlen d128 decode num_splits cap engages a MESSAGE("no CUDA backend; skipping FA-2 varlen d128 num_splits-cap check (dgx-pending)"); return; } + // ★ {32, 2} IS NEMOTRON-3.5-LIGHTNING, AND IT WAS NOT HERE (#1157). + // Every case in this file measured ngroups 2 (16/8) and 4 (32/8), the two + // Qwen3-dense gate configs the varlen d128 decode path was written for. The + // path is DEFAULT ON for ANY bf16 causal pure-decode at head_dim 128, so the + // first model to arrive with a different ratio reaches an untested grid — and + // NemotronH-3.5-Lightning-30B is 32 query heads over 2 KV heads, ngroups 16, + // four times the widest group count ever measured here. Its A3 token gate + // failed 6/96 on the device while the same binary decodes the same checkpoint + // token-exact on CPU, which is what sent the search here. for (const auto& ratio : {std::pair{16, 8}, - std::pair{32, 8}}) { + std::pair{32, 8}, + std::pair{32, 2}}) { for (const int batch : {1, 2, 4}) { CAPTURE(ratio.first); CAPTURE(batch); From 8b566b68b74b8cf0e660da168f11c33209a8f048 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 18 Aug 2026 09:20:26 +0000 Subject: [PATCH 5/7] fix(#1157): the paged forward embedded the STALE host token ids, so every decode step saw the same token MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ModelForwardInput::device_token_ids` is non-null exactly when the async runner's device combine has already spliced each DECODE row's sampled token into its own device buffer and left the host `token_ids` stale on purpose — not materializing it on the host is the synchronize ENG-ASYNC-SCHED W4 exists to remove. `NemotronHPagedForward` uploaded the host vector anyway, so on the default CUDA path every decode step embedded the same placeholder id. The measurement, all on the released 30B NVFP4 checkpoint and the committed oracle golden, through `include/vllm.h` and nothing else: CPU, decode 96/96, STRICT PASS, full rows 3 GB10, decode 4/24 GB10, fresh-prefill 24/24 (one token per completion: no decode step) and the per-layer trace names the mechanism rather than inferring it. At the prefill step CPU and GB10 agree to six digits on every one of the 52 layers. At the first decode step the gathered conv/SSM state is IDENTICAL on the two — the recurrent carry is exact — while layer 0's embedding row differs, and it reads 0.228135 on GB10 at BOTH decode steps although they consume different tokens. A constant embedding is a constant input id. This refutes the cause on record. #1157 reasoned that `gm.num_decodes` might classify a decode as a prefill and hand the mixer zeros; the trace reports `nd=1 np=0 init=[1]` on every decode step, and mutating that mask to 0 turns the existing A2-P CPU gate RED (1 case, 6 assertions), so the gate was never blind to it. Kimi-Linear was cut from this same divergence (kimi_linear_device.cpp:2270) and every other registered forward already honours the field. Nothing could see that this one did not: the runner sets the pointer only under VLLM_CPP_CUDA with a live device mirror, so no CPU gate can reach the branch. The host-side id range check stays on the host branch only. Validating device ids would need the D2H synchronize this path exists to delete. Closes #1157 FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: AGENT:claude-opus-5 [claude-code] --- .../models/nemotron_h_device.cpp | 47 ++++++++++++++++--- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/src/vllm/model_executor/models/nemotron_h_device.cpp b/src/vllm/model_executor/models/nemotron_h_device.cpp index d991b70b7..54e243b3a 100644 --- a/src/vllm/model_executor/models/nemotron_h_device.cpp +++ b/src/vllm/model_executor/models/nemotron_h_device.cpp @@ -1523,14 +1523,47 @@ ForwardLogits NemotronHPagedForward(const NemotronHHostWeights& host, RequireDeviceWeight(host.norm_f, "backbone.norm_f.weight", adt, {H}); DBuf residual(d, adt, {T, H}); { - std::vector ids = input.token_ids; - for (int32_t id : ids) { - VT_CHECK(id >= 0 && id < V, "NemotronH paged forward: token id out of range"); - } - DBuf it(d, DType::kI32, {T}, ids.data()); - d.b.Synchronize(d.q); // `ids` is a local; see UploadAs. Tensor tab = ResidentWeight(d, host.embeddings); - vt::Embedding(d.q, residual.t(), tab, it.t()); + Tensor rt = residual.t(); + if (input.device_token_ids != nullptr) { + // ★ ENG-ASYNC-SCHED W4 (#1157). `ModelForwardInput::device_token_ids` is + // non-null exactly when the async runner's device combine has already + // spliced each DECODE row's sampled token into ITS device buffer and left + // the host `token_ids` STALE on purpose — materializing it on the host is + // the synchronize W4 exists to remove (model_registry.h:314-324, + // runner.cpp:1175-1194). A forward that embeds the host vector therefore + // embeds the same placeholder id on every decode step. + // + // That is not a hypothesis. With the host vector, this model's A3 token + // gate read 4/24 on GB10 while the SAME binary read 24/24 in + // fresh-prefill mode (no decode step is ever taken) and 96/96 on CPU + // (where this pointer is always null), and the per-layer trace showed the + // layer-0 embedding row identical across two consecutive decode steps + // that consumed different tokens. + // + // Kimi-Linear was cut from this same divergence + // (kimi_linear_device.cpp:2270-2280) and every other registered forward + // already honours the field. This one did not, and nothing could see it: + // the runner sets the pointer only under VLLM_CPP_CUDA with a live device + // mirror, so no CPU gate can reach the branch at all. + // + // The host-side range check below is deliberately NOT repeated here. The + // ids live on the device and validating them would need the D2H + // synchronize this path exists to delete; `LaunchCombineSampledAndDraft + // Tokens` produces them from the sampler's own output, and vt::Embedding + // bounds-checks the gather. + Tensor ids = MakeTensor(const_cast(input.device_token_ids), + DType::kI32, d.q.device, {T}); + vt::Embedding(d.q, rt, tab, ids); + } else { + std::vector ids = input.token_ids; + for (int32_t id : ids) { + VT_CHECK(id >= 0 && id < V, "NemotronH paged forward: token id out of range"); + } + DBuf it(d, DType::kI32, {T}, ids.data()); + d.b.Synchronize(d.q); // `ids` is a local; see UploadAs. + vt::Embedding(d.q, rt, tab, it.t()); + } } vt::RmsNormArgs nargs; From 0fba1af882d79ae63db880a9cc703c1a6723ca83 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 18 Aug 2026 09:29:27 +0000 Subject: [PATCH 6/7] record(#1157): the A3 gate has a host PASS, a named device cause, and the seam that allowed it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The row's public record still said the A3 gate was pending on a toolchain, and `#1157` still carried a cause that the measurement refutes. Both are corrected against numbers rather than re-quoted. `benchmark-record.md` carries the three-arm table the diagnosis rests on — host 96/96, GB10 decode 4/24, GB10 fresh-prefill 24/24, one binary and one checkpoint — plus the per-layer trace that puts the divergence at layer 0's embedding row while the recurrent state gathered identical on both sides. `docs/STATUS.md` and `docs/BENCHMARKS.md` move the row from "gate pending" to "host gate passes, device fixed, sm_121a re-run pending a lease", inside the STATUS size ratchet. [#1217](https://github.com/mudler/vllm.cpp/issues/1217) is filed and listed under the spec's `## Owed`: the runner hands `device_token_ids` to whatever model the step routes to, its own comment claims a model that ignores it is never given one, and nothing enforces that. Two models have now been cut from the identical divergence. Not fixed here because both closes change a shared seam or checker semantics. The two diagnostic knobs are documented in `docs/ENVIRONMENT.md` rather than allowlisted as kernel-internal: `VT_NEMOTRON_H_DIAG` is how this bug was separated from the carry, and the next reader of this model should find it. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: AGENT:claude-opus-5 [claude-code] --- .agents/benchmark-record.md | 64 +++++++++++++++++++ .agents/issue-index.md | 1 + .agents/specs/nemotron-h-a2p-paged-forward.md | 47 ++++++++++++++ docs/BENCHMARKS.md | 2 +- docs/ENVIRONMENT.md | 2 + docs/STATUS.md | 2 +- 6 files changed, 116 insertions(+), 2 deletions(-) diff --git a/.agents/benchmark-record.md b/.agents/benchmark-record.md index d3f285af6..be0145366 100644 --- a/.agents/benchmark-record.md +++ b/.agents/benchmark-record.md @@ -19,6 +19,70 @@ from relative link targets repointed for this file's location. # Benchmarks +## MODEL-NEMOTRON-H-ABI-A2P — the A3 gate PASSES on the host, and the device divergence was the STALE input ids (2026-08-18, `row/MODEL-NEMOTRON-H-ABI-A2P-1157`, #1157, #1217, #810) + +**This supersedes the entry that recorded the A3 gate as a 6/96 device failure +with an unknown cause. The cause is known, it is not the recurrent carry, and +the host leg of the gate is a PASS.** + +**Host leg, `STRICT PASS`.** `TOKEN MATCH: 96/96 over 3 prompts (full rows=3, +short rows=0)`, driven through `include/vllm.h` alone by +`examples/nemotron_h_gen` against the committed oracle golden, on the released +`nemotron-3.5-lightning-30b-nvfp4` at revision `29f2d174`. Engine load 209.0 s; +peak RSS 20 142 392 KB; per-prompt wall 928.93 / 839.42 / 1081.16 s. +`--max-model-len 512`, greedy, `ignore_eos`. **No number on any speed axis is +claimed or implied** — this is a correctness result on a host that is not the +performance target. + +**Device leg, GB10 sm_121a, and the three measurements that name the mechanism.** +The build was not degraded: CUDA 13.x from the `ubuntu2404/sbsa` lane, +`CFG_RC=0`, `cutlass-nvfp4` / `cutlass-fp8` / `marlin-nvfp4` / `fa2` all +`ENABLED for [121a]`, `BUILD_RC=0`, `compile_errors=0`, binary sha256 +`b4677cdb7cf521250c5325fa10e5eadc80134763621d187af1f9b380c7d70140`. + +| arm | same binary, same weights, same golden | result | +|---|---|---| +| decode | the shipped path | **4/24** over the first 8 tokens of 3 prompts | +| fresh-prefill | one token per completion, so no decode step is ever taken | **24/24** | +| host | the same driver on a CPU queue, where `device_token_ids` is always null | **96/96** | + +The `got` streams in the decode arm are byte-identical to the earlier recorded +run, so the failure reproduced on a fresh build rather than drifting. + +**The per-layer trace (`VT_NEMOTRON_H_DIAG`) localises it to the first +operation of the decode step.** At the prefill step CPU and GB10 agree to six +digits on every one of the 52 layers, including bit-identical layer-0 numbers. +At the first decode step the gathered conv/SSM state is IDENTICAL on the two +(`|conv|=310.374`, `|ssm|=3985.8` on both), so the recurrent carry is exact — +and layer 0's embedding row differs. It reads `0.228135` on GB10 at BOTH decode +steps, which consumed different tokens. A constant embedding is a constant +input id. + +**Cause:** `NemotronHPagedForward` embedded the host `input.token_ids` while +`ModelForwardInput::device_token_ids` was non-null. That field's contract is +that the host vector is STALE for decode rows (`model_registry.h:314-324`), +because not materialising it on the host is the synchronize ENG-ASYNC-SCHED W4 +exists to remove. Kimi-Linear was cut from the same divergence +(`kimi_linear_device.cpp:2270-2280`); the seam that allows a third is +[#1217](https://github.com/mudler/vllm.cpp/issues/1217). + +**What this REFUTES, recorded because the wrong cause was on the record for a +day.** [#1157](https://github.com/mudler/vllm.cpp/issues/1157) reasoned that +`gm.num_decodes` might classify a decode as a prefill so the gather would hand +the mixer zeros. On real weights the trace reports `nd=1 np=0 init=[1]` on every +decode step, and mutating that mask to 0 turns the A2-P CPU gate RED (1 case, +6 assertions) — so that gate was never blind to that defect. It was blind to the +real one for a structural reason: the runner sets `device_token_ids` only under +`VLLM_CPP_CUDA` with a live device mirror, so no CPU gate reaches the branch. + +**Two things this run established about the environment, both cheap to lose.** +The released checkpoint LOADS AND DECODES ON A CPU-ONLY BOX — 20.1 GB peak RSS, +209-304 s from a CIFS mount — which is what made a same-binary host/device A/B +affordable at all and should be the first instrument reached for the next +device-only divergence on this model. And `/workspace` on the `rc` worker +persists between runs, so a cloned source tree and a CMake build directory under +`/root` survive long enough for an incremental rebuild between arms. + ## MODEL-NEMOTRON-H-ABI-A3-E2E — the A3 token gate did NOT run, and the cause on record was NOT the cause (2026-08-17, `row/MODEL-NEMOTRON-H-ABI-A3-E2E`, base `origin/main` `a6df72777`, #810) **No number is recorded, on any axis. This entry exists so the pending cause is diff --git a/.agents/issue-index.md b/.agents/issue-index.md index 83876d612..ad7433710 100644 --- a/.agents/issue-index.md +++ b/.agents/issue-index.md @@ -366,3 +366,4 @@ rather than merged. `scripts/check-agent-record.py` gates both. | [#1179](https://github.com/mudler/vllm.cpp/issues/1179) | `ENG-CUDAGRAPH-BREAK` | The hand-rolled decode-graph driver count recorded in `9bc4d7f44` is **eight** and is actually **nine**, and the row it feeds was framed as coverage-only when it is also correctness. The ninth is the DFlash draft graph, file-local with no header declaration, at `src/vllm/model_executor/models/qwen3_dflash.cpp:771,870,1038,1091,1095,1106` — its own `int g_state = 0` three-state machine (`:771`), its own `VT_DFLASH_GRAPH` kill switch (`:870`) instead of the `VLLM_CPP_CUDAGRAPH` the six batched drivers read, its own invalidate-on-block-width-change (`:1038-1047`) and its own `try { EndCaptureGraph(); } catch (...) {}` drain (`:1106`). The eight-count is stated in four places, all corrected here: [`sglang-breakable-cuda-graph.md`](specs/sglang-breakable-cuda-graph.md) §4 and `## Owed`, [`.agents/engine-matrix.md`](engine-matrix.md) rows `ENG-CUDAGRAPH-BREAK` and `ENG-CUDAGRAPH-DEDUP` ("times eight drivers", which sizes #1162's signature table), and [`.agents/roadmap_v1.md`](roadmap_v1.md) track `C12`. The reframing is the substantive half: `ENG-CUDAGRAPH-BREAK` was recorded as a COVERAGE row, and the duplication has already cost a SHIPPED model its decode graph. `src/vllm/model_executor/models/qwen3.cpp:961-986` declines the decode graph outright whenever the asynchronous device-token mirror is live, on its own measured battery — `depth-1, graph ON PASS 78/78`; `depth-2, graph OFF PASS 82/82`; `depth-2, graph ON FAIL, slots 1-3 degenerate` — because `Step()` replays against the HOST `input.token_ids` and the combine has patched the DEVICE ids. The comment names the real fix as reading the identifiers at replay time from a stable device buffer, and that fix exists, in exactly one sibling driver, as `StepDevInputs` (`src/vllm/model_executor/models/qwen3_5.cpp:3894`): `grep -c StepDevInputs` returns 41 lines there and 0 in each of `qwen3_moe.cpp`, `qwen3.cpp`, `deepseek_v2.cpp` and `voxtral.cpp`. One capability, written once, unavailable to four models, with a live mitigation standing in its place. This does NOT weaken the framing rule that `ENG-CUDAGRAPH` established: the row still makes no throughput claim, and the prefill refutation (GB10 3.8% host-idle between launches, GPU-busy >96%, 27B prefill gap 92.5% non-GEMM glue) stands unchanged. Coverage AND correctness, never speed. Fixed in flow with the [`eng-cudagraph-break.md`](specs/eng-cudagraph-break.md) review repair ([#1163](https://github.com/mudler/vllm.cpp/issues/1163)) | record | | [#1181](https://github.com/mudler/vllm.cpp/issues/1181) | `FIX-READ-F32-SCALAR-GUARD` | `ReadF32Scalar` (`src/vllm/model_executor/models/qwen3_5_weights.cpp:312-318` @ `ab6e65216`) bounds its input with `t.data != nullptr && t.nbytes >= sizeof(float)`, a LOWER bound, and then `memcpy`s four bytes into a `float`. Two silent wrong-value paths follow and neither fails: an ARRAY is reduced to element 0, so a block-wise FP8 scale grid of shape `[ceil(N/128), ceil(K/128)]` passes and stands in for the whole weight (measured under [#1166](https://github.com/mudler/vllm.cpp/issues/1166) on `Qwen/Qwen3.8-27B-FP8` @ `017b9c7af6b5689d5dd426a76e0bc077eb5ca20a`, `q_proj.weight_scale_inv` is `[96, 40]`), and ANY dtype is reinterpreted, since that same tensor is `BF16` and its four bytes are two bf16 values read as one float. Both return a finite plausible float, so the output is fluent, plausible and wrong, which is what a token gate cannot see. Upstream makes both facts structural rather than optional: a per-tensor scale is a distinct parameter TYPE that asserts `loaded_weight.shape[0] == 1` (`vllm/model_executor/parameter.py:260-272,304-309` @ `555967922`, plus the `_assert_and_load` shape assert at `:93-96`), the slot is allocated `torch.float32` so a narrow on-disk dtype is VALUE-converted rather than reinterpreted (`utils/fp8_utils.py:1276`), and the declared strategy TENSOR/CHANNEL/BLOCK picks the parameter type before a byte is read (`compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py:63,128`). The AUDIT corrects the issue's own framing twice. The 27 grep hits across five files are 5 definitions, 20 call sites and 2 comment references, and both counts are short: `ReadCtF32Scalar` (`include/vllm/model_executor/models/dense_weight_loaders.h:376`) is a SIXTH copy of the same defect under another name, reached from a SIXTH model file (`src/vllm/model_executor/models/qwen3_weights.cpp:100,126-128` through `LoadCtNvfp4W4A16`). Of the six, three check nothing, `LnReadF32Scalar`/`ShReadF32Scalar` check dtype but not count, and only `nemotron_h_weights.cpp:557-573` is correct, which makes it the model the shared guard generalizes. No call site legitimately passes a multi-element or non-F32 tensor, and every existing fixture emits rank-0 or `{1}` `F32`, so nothing in the tree needed the leniency. It is NOT merely latent: `dense_weight_loaders.h:73-74` and `docs/BENCHMARKS.md:52` both record `unsloth/Qwen3.6-27B-NVFP4` @ `ccdaab7e` as FP8 W8A8 throughout with BF16 PER-OUTPUT-CHANNEL scales, and `LoadAttnDense` branches on the weight dtype alone (`qwen3_5_dense_weights.cpp:478-480`), so those projections enter the per-tensor arm and hit both defects at once under the tensor name the loader actually asked for, with no misspelling to stop them. Fixed in flow by one `dense_loaders::ReadF32Scalar(get, name)` that refuses `numel != 1` naming the shape, refuses a non-`F32` dtype naming the dtype, and requires exactly four readable bytes, with the other five copies deleted onto it and `nemotron_h`'s `Loader`-based twin kept as the one tracked exception. A narrow dtype is refused rather than converted, because a one-element BF16 scale has never been read correctly here and the BF16 layout that IS shipped is per-channel, which the count check refuses first. Per-channel FP8, block-wise FP8 and any explicit narrow-dtype conversion stay owed. Spec [`read-f32-scalar-guard.md`](specs/read-f32-scalar-guard.md) | bug | | [#1185](https://github.com/mudler/vllm.cpp/issues/1185) | `ENV-ORACLE-WHEEL-IN-LEASE` | The pinned vLLM oracle BUILDS, installs, imports and sees the GPU inside an `rc` lease on `dgx:gpu0`, measured 2026-08-18, which falsifies the `nvcc` clause four records carried. [`lease-runtime-staging.md`](specs/lease-runtime-staging.md) said the oracle "needs `nvcc`, which the worker still lacks", and `.agents/environment.md`, [`mtp-k-gt-1.md`](specs/mtp-k-gt-1.md) and [`gpu-lease-methodology.md`](specs/gpu-lease-methodology.md) each derived a blocker from it. The build job (`buildvllm.sh`, staged sha256 prefix `15e140d41f44e7c2`) asserted the checkout against the pin BEFORE compiling, printing `PIN CONFIRMED` at `5559679229bc961848b121ccdeaa8fa5d79bec98` and aborting otherwise, took `nvcc` from the toolkit row `MODEL-NEMOTRON-H-ABI-A3-E2E` staged (`NVCC_RC=0`, CUDA `release 13.3, V13.3.73`) and produced `WHEEL_RC=0`, `PERSIST_RC=0` and a 434 MiB `vllm-0.1.dev1+g555967922.cu133-cp312-cp312-linux_aarch64.whl`, sha256 `7c58b339741a288fbb313f4f5196c9c92a9e3b3c3ebe2ea970b0ff50bb9bcba4`. The identity job (`oracleenv.sh`, prefix `6119f5223f5d818c`) asserted from `cd /`, outside any source tree: `vllm.__version__ = 0.1.dev1+g555967922`, `IDENTITY_RC=0`, `cuda True NVIDIA GB10`, `CUDA_RC=0`. SCOPE, and it carries the same weight as the result: RUNNING A MODEL IS UNTESTED. Only build, install, import and `torch.cuda.is_available()` are measured, and [`mtp-k-gt-1.md`](specs/mtp-k-gt-1.md) records that the last time an oracle reached this far it consumed the host in the step AFTER `torch.compile` and REBOOTED the box, at `gpu_memory_utilization` 0.75 and again at 0.30, so the fraction is not the lever. The version string is an OPEN discrepancy: `.agents/upstream-sync.md` records `vllm_runtime_version = 0.23.1rc1.dev1511+g555967922`, the commit segment matches and satisfies the pin's binding `+g` rule, and the prefix differs because a shallow fetch stops `setuptools_scm` counting commits since the last tag, so a full-string gate needs a deeper fetch or a recorded pretend-version. The venv is NOT staged, because that job was killed at a 90-minute ceiling mid-copy and its partial tree was removed, so only the WHEEL is durable. Four staging walls, all artifacts of the NAS rather than of CUDA: `cp -a` preserves `file_mode=0664` so `nvcc` exited 126. CIFS `nounix` stores no symlink so `include` and `lib64` vanished and CMake reported `Could NOT find CUDA (missing: CUDA_INCLUDE_DIRS CUDA_CUDART_LIBRARY) (found version "13.3")`, naming the version and denying the toolkit in one line. 32 library links `libfoo.so` and `libfoo.so.MAJOR` had to be rebuilt because only the `libfoo.so.X.Y.Z` real files survived. And `markupsafe` existed as a dist-info with NO package files from a `pip --target` killed at a 35-minute ceiling, so Marlin codegen died on `ModuleNotFoundError`. The `rc` worker container is REUSED between jobs, so a repair inside a staging branch is skipped on the next run (`nvcc already in place`) and an environment repair must be unconditional and assert its postcondition. CONSEQUENCE for the rows #1129 blocked, [#1003](https://github.com/mudler/vllm.cpp/issues/1003), [#915](https://github.com/mudler/vllm.cpp/issues/915), [#821](https://github.com/mudler/vllm.cpp/issues/821) and [#81](https://github.com/mudler/vllm.cpp/issues/81): UNBLOCKED FOR THE BUILD STEP and STILL BLOCKED FOR A MODEL RUN. None can take a measurement until a model run is demonstrated. Job details, walls and non-claims in [`oracle-wheel-in-lease.md`](specs/oracle-wheel-in-lease.md) | verification | +| [#1217](https://github.com/mudler/vllm.cpp/issues/1217) | `MODEL-NEMOTRON-H-ABI-A2P` | `ModelForwardInput::device_token_ids` carries the async runner's device-combined ids and its contract is that `token_ids` is STALE for decode rows whenever the pointer is non-null (`model_registry.h:314-324`). A registered forward that embeds the host vector then embeds the same placeholder id on EVERY decode step. The field's own comment says a model that ignores it "is simply never given one", but `runner.cpp:1408` sets the pointer for whatever model the step routes to, with no per-model opt-in and no check \| two models have now been cut from the identical divergence: Kimi-Linear (`kimi_linear_device.cpp:2270-2280`, the GB10 9/128 case) and NemotronH's paged forward under [#1157](https://github.com/mudler/vllm.cpp/issues/1157), whose A3 gate read 4/24 on GB10 against 96/96 for the same binary on CPU where the pointer is always null \| invisible because the runner sets it only under `VLLM_CPP_CUDA` with a live device mirror, so no CPU gate reaches the branch, and the failure is fluent wrong tokens rather than an error \| two closes: give `ModelFactory` an explicit `honors_device_token_ids` and have the runner fall back to the synchronous host path for a forward that has not declared it, or add a checker over the registered `.forward` entry points (a file-level grep flags ~25 false positives because several models delegate through `detail::DeviceTokenIdsScope` or the shared dense block) \| NOT fixed in the #1157 flow because one close changes a shared seam and every model factory and the other changes checker semantics, which is the "needs its own spec" case rather than the in-flow case. Listed under `## Owed` in [`nemotron-h-a2p-paged-forward.md`](specs/nemotron-h-a2p-paged-forward.md) | bug | diff --git a/.agents/specs/nemotron-h-a2p-paged-forward.md b/.agents/specs/nemotron-h-a2p-paged-forward.md index de0198134..e2cb1ab57 100644 --- a/.agents/specs/nemotron-h-a2p-paged-forward.md +++ b/.agents/specs/nemotron-h-a2p-paged-forward.md @@ -794,6 +794,41 @@ lifecycle write. ## 10. Now +**State at this commit: A2-P's PRODUCT CODE HAS LANDED, its A3 end-to-end token +gate HAS RUN on the released checkpoint, and the divergence it found has a +measured cause and a fix (#1157).** + +**The A3 gate PASSES on the host: `TOKEN MATCH: 96/96 over 3 prompts, full +rows=3, short rows=0`, `STRICT PASS`** (2026-08-18, the released +`nemotron-3.5-lightning-30b-nvfp4` at revision `29f2d174`, the committed oracle +golden, through `include/vllm.h` alone). Engine load 209.0 s, peak RSS +20 142 392 KB, per-prompt wall 928.93 / 839.42 / 1081.16 s. That is the whole +paged forward — the recurrent carry, the paged KV, the FP8 Mamba2 projections, +the NVFP4 MoE and `lm_head` — token-exact against the pinned oracle. + +**On GB10 the same binary read 4/24, and the cause was NOT the recurrent +carry.** `NemotronHPagedForward` embedded the HOST `input.token_ids` while +`ModelForwardInput::device_token_ids` was non-null, and that field's contract is +that the host vector is STALE for decode rows (`model_registry.h:314-324`). Every +decode step therefore embedded the same placeholder id. Three measurements name +it rather than infer it: fresh-prefill mode on the SAME GB10 binary, which takes +no decode step at all, read 24/24; the per-layer trace agrees to six digits +between CPU and GB10 at the prefill step across all 52 layers; and at the first +decode step the gathered conv/SSM state is IDENTICAL on the two while layer 0's +embedding row reads 0.228135 on GB10 at BOTH decode steps, which consumed +different tokens. + +**The cause recorded on #1157 was wrong, and this is the part worth keeping.** +It reasoned that `gm.num_decodes` might classify a decode as a prefill so the +gather would hand the mixer zeros. The trace reports `nd=1 np=0 init=[1]` on +every decode step on real weights, and mutating that mask to 0 turns this row's +CPU gate RED (1 case, 6 assertions) — so the gate was never blind to that +defect. It was blind to this one for a structural reason: the runner sets +`device_token_ids` only under `VLLM_CPP_CUDA` with a live device mirror, so no +CPU gate can reach the branch at all. + +**Superseded record below, kept because its corrections are still useful.** + **State at this commit: A2-P's PRODUCT CODE HAS LANDED, and its A3 end-to-end token gate has NOT RUN.** `ForwardNemotronHForCausalLM` selects `NemotronHPagedForward` whenever the runner supplies paged KV and recurrent @@ -863,8 +898,20 @@ nobody routes this architecture through a block that ropes. ## 11. Owed +- **[#1217](https://github.com/mudler/vllm.cpp/issues/1217) — the seam that let + #1157 land.** `ModelForwardInput::device_token_ids` says a forward that ignores + it "is simply never given one", but `runner.cpp:1408` sets the pointer for + whatever model the step routes to, with no per-model opt-in and no check. Two + models have now been cut from the identical divergence: Kimi-Linear + (`kimi_linear_device.cpp:2270-2280`) and this one. Not fixed in the #1157 flow + because both closes — an explicit `ModelFactory::honors_device_token_ids` with + a runner fallback, or a checker over the registered `.forward` entry points — + change a shared seam or checker semantics, which is the case AGENTS.md sends + through its own spec rather than in flow. - **The §5.4 A3 end-to-end token gate**, and the §5.7 sm_121a leg with it. Owned by this row, tracked on [#810](https://github.com/mudler/vllm.cpp/issues/810). + **The HOST leg is now GREEN (96/96, `STRICT PASS`, §10); the sm_121a leg's + green-after re-run under the #1157 fix is what remains.** Nothing about the released checkpoint is claimed until it runs. **The recorded PENDING CAUSE IS NO LONGER TRUE and was re-measured rather than inherited** (2026-08-17): §10 records contention — `dgx.casa` at loadavg 211 with 3 of diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 79ee28b93..f8bcb61c7 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -10,7 +10,7 @@ | **Binary release (ACTIVE; Windows pre-alpha pending)** | v0.0.2 shipped eight primary archive/checksum/provenance triplets + two indexes (26 assets) from source SHA `7020de93652ca920424a10ac5255b34810dd2f24`, run `31466516224` | Windows W14-W16 implemented. **PENDING:** native hosted gates, merged-SHA ten-tuple dry run, matching-hardware evidence, v0.0.3-pre.1 publication, 32-asset audit | W12 optional/non-primary | | **Container images (ACTIVE; arm64 cuda verified on GB10 + Orin 2026-08-11)** | `ENG-RELEASE-CONTAINERS` ([spec](../.agents/specs/container-images.md)) | cpu amd64 783 MB; cuda arm64 **1.71 GB**. GB10 `sm_121a`: `/health`+`/version`+SIGTERM on `--gpus all`. Orin `sm_87` (Tegra): Qwen3-0.6B **generates**, GPU **GR3D 95-97%** | n/a | | **Developer/row protocol** | Contribution entry point; `ENG-NOW-DERIVED` #374 @`dbd0d51c` | Entry-point gates retained. #374 W1-W5 DONE; benchmark/runtime/parity `VOID`; row specs now carry `## Now` | n/a | -| **NemotronH paged forward** (`MODEL-NEMOTRON-H-ABI-A2P`, [#810](https://github.com/mudler/vllm.cpp/issues/810)) | **No number on any axis, by the unit's own rule** ([spec](../.agents/specs/nemotron-h-a2p-paged-forward.md) §5) | **A3 gate PENDING: `nvcc` + checkpoint visibility in the rc container.** Two earlier causes here (contention, then "cannot build") were measured FALSE, see [benchmark-record](../.agents/benchmark-record.md) | CPU gate 12/12, 9/9 mutations RED. A3 driver `examples/nemotron_h_gen` exists; guards armed on a real engine (pass 0, divergence 1, short 4) | +| **NemotronH paged forward** (`MODEL-NEMOTRON-H-ABI-A2P`, [#810](https://github.com/mudler/vllm.cpp/issues/810)) | **No speed number, by the unit's own rule** ([spec](../.agents/specs/nemotron-h-a2p-paged-forward.md) §5) | **A3 host gate PASSES 96/96 `STRICT PASS`.** GB10 read 4/24; cause and fix [#1157](https://github.com/mudler/vllm.cpp/issues/1157), sm_121a re-run pending a lease | CPU gate 12/12. Load 209.0 s, peak RSS 20 142 392 KB | | **LoRA runtime W2** (`LORA-RUNTIME`, #278) | **No number owed:** correctness-only; a grid PENDS the W7 model gate | | **ARCH audit: ABI is text-only** | 4 capabilities (H3 video, Laguna, Kimi-Linear, DeepSeek-V4) reachable only from `examples/`, none registry-backed. No gate asks whether a CONSUMER can reach a capability. Documentation only | | **DSR fix: server TU profiler guards (2026-08-09)** | **No number owed:** comments only. #189 moved the server body into the shared layer with its 5 `VT_BENCH_PROFILE_CONTROL` guards, taking DSR 32 -> 37; they are `DSR-ALLOW`'d per site, baseline unchanged at 32 | diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md index 6857087fa..ffa6a3128 100644 --- a/docs/ENVIRONMENT.md +++ b/docs/ENVIRONMENT.md @@ -226,6 +226,8 @@ Read-only observability; none change output. | `VT_TT_TRACE_DEBUG` | unset | `=1` prints the Tenstorrent capture bisection traces to stderr: op entries (`[TT-OP]`), host readbacks (`to_vector`/`EnsureHostBytes`), device->device copies/zero-fills, and rope cos/sin cache lookups — all gated to fire only while a mesh-trace capture is active. Read-only diagnostics for the host-free decode investigation; byte-identical output when unset | | `VT_OP_PROVIDER_DISABLE` | (none) | Comma-separated provider names to disable, forcing fallback (diagnostic) | | `VT_SERVER_PREFILL_PROGRESS` | off | `=1` prints chunked-prefill progress to stderr, rate-limited to roughly 2 Hz per request. `=0` explicitly disables it even when `VT_SERVER_VERBOSE=1` | +| `VT_NEMOTRON_H_DIAG` | off | `=1` prints one line per step and per layer of `NemotronHPagedForward` to stderr: the decode/prefill split, the recurrent slot indices and the has-initial mask, then per Mamba2 layer the L2 norm of the state gathered IN and the state written OUT, and per layer the last row's mixer and residual norms. It is what separated a broken recurrent carry from a stale input id on this model ([#1157](https://github.com/mudler/vllm.cpp/issues/1157)): the carry read identical on host and GB10 while layer 0's embedding row did not. Every download it does is inside the guard, so an unset run pays nothing | +| `VT_NEMOTRON_H_DEVICE_MOE` | on | `=0` routes NemotronH's NVFP4 MoE layers back through the host reference instead of the device Marlin arm. A bisect lever for a device-vs-host divergence on this model, not a configuration: it trades the whole MoE tower's throughput for the arm the CPU gate proves token-exact | | `VT_GDN_VALIDATE` | off | Run the GDN validation/cross-check path (slower; for kernel debugging) | | `VT_FP4_AUTOTUNE_VERBOSE` | off | Log the NVFP4 GEMM autotuner's tactic selection | | `VT_H3_PROGRESS` | unset | Trace the MiniMax-H3 denoise loop's phases to stderr: which forward path was taken (device vs the CPU reference), how long the ONE-TIME device weight staging took, and per-step forward seconds with the sequence length. A real-checkpoint run spends its minutes in exactly one of those phases, and this says which without guessing — it was added after GPU-utilization counters proved unreliable on Tegra-class boards | diff --git a/docs/STATUS.md b/docs/STATUS.md index a8bc9ec89..5b9a0b3f3 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -131,7 +131,7 @@ token-for-token correctness against the pinned oracle. | GLM-4 dense (sandwich norms, partial rope) | Correctness-complete, speed-pending | Token-exact 16/16 (GLM-4-9B-0414); first GLM-family model; partial interleaved RoPE + Gemma2 sandwich norms + biased qkv | | GLM-4.7-Flash (MLA + GLM MoE) | Correctness-complete, speed-pending | Token-exact 8/8 (GLM-4.7-Flash, 31.2B); reuses the DeepSeek-V2 MLA stack; first e2e coverage of the q_lora query branch + noaux_tc sigmoid router with routed-scaling | | Kimi-Linear-48B-A3B (KDA + NoPE-MLA + MoE hybrid) | **RUNNER FOLD LANDS (ROW 7 §21, #122): engine==CLI 128/128 byte-identical; golden 122/128 (near-tie profile); FA2 MLA default-ON; `vllm_complete_tokens` (ABI v13).** Grouped-router top-k block-parallel (byte-identical); no binding speed number: ckpt is tiktoken-only, so no warm-server harness. STRICT stays CLOSED. Server 19.0 tok/s wall (~0.90× vLLM floor) = speed open | paged suite 8/8·206; SACRED post-fold 35B 315/315 + 27B 235/235; thin ABI client (ratchet 8) | -| Nemotron-3.5-Lightning-30B-A3B (Mamba2 + GQA + relu2 MoE) | **Paged forward + ABI driver land (#810 A2-P, A3); e2e token gate PENDING on `nvcc` + checkpoint visibility in the rc container** | `examples/nemotron_h_gen` reaches it through `include/vllm.h` alone; G-SAFE narrows to `num_reqs <= 1`. Two earlier pending causes on this row were measured false; the box is idle and the checkpoint is revision-verified | +| Nemotron-3.5-Lightning-30B-A3B (Mamba2 + GQA + relu2 MoE) | **Host gate PASSES 96/96 `STRICT PASS`; GB10 fixed ([#1157](https://github.com/mudler/vllm.cpp/issues/1157)), sm_121a re-run pending** | ABI-only driver; G-SAFE `num_reqs <= 1`. The paged forward now honours `device_token_ids`; seam [#1217](https://github.com/mudler/vllm.cpp/issues/1217) | | Gemma-3 dense (GeGLU, dual rope, sandwich norms) | Correctness-complete, speed-pending | STRICT token-exact 48/48 greedy (gemma-3-1b-it); first Gemma-family model; GeGLU (gelu_pytorch_tanh) + dual per-layer RoPE theta + Gemma-RMSNorm sandwich norms + sqrt(hidden) embed-scale + query_pre_attn_scalar scaling | | Gemma-2 dense (attn + final logit soft-cap) | Correctness-complete, speed-pending | Near-tie-band 48/48 (gemma-2-2b-it): 44/48 strict on vLLM's greedy + 4/48 at 0.0-nat ties in vLLM's own logits; proves the attention + final logit soft-cap primitives (attn_logit_softcapping 50 + final 30); the inverse of Gemma-3 (both soft-caps, no QK-norm) | | Gemma-1 dense (the original Gemma) | Correctness-complete, speed-pending | STRICT token-exact 48/48 greedy (gemma-2b); two fused norms/layer, head_dim scale, GeGLU + sqrt(hidden) embed-scale, tied lm_head; no soft-cap/QK-norm/sliding. **D1 (2026-07-31): the whole Gemma family (1/2/3/4) folded to the default-ON bf16 merged-QKV descriptor (`MergedQkvEnabled`); re-gated Gemma-2 SACRED 48/48 (global+sliding) + Gemma-4 STRICT 32/32 — its existing gate held** | From 6e9e8955955426e5fb5995f10a701f39888c7205 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 18 Aug 2026 09:53:43 +0000 Subject: [PATCH 7/7] docs(#1157): the diagnostic is not scaffolding, so its comment should not call itself temporary The block header said TEMPORARY while `docs/ENVIRONMENT.md` documents the knob and the next divergence on this model will be diagnosed with it. It now says what it is for instead: the runner hands a decode step a device-resident input id AND a recurrent page, and when the tokens come out wrong only the per-layer numbers say which of the two the step actually read. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: AGENT:claude-opus-5 [claude-code] --- .../models/nemotron_h_device.cpp | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/vllm/model_executor/models/nemotron_h_device.cpp b/src/vllm/model_executor/models/nemotron_h_device.cpp index 54e243b3a..4455998d9 100644 --- a/src/vllm/model_executor/models/nemotron_h_device.cpp +++ b/src/vllm/model_executor/models/nemotron_h_device.cpp @@ -1112,13 +1112,19 @@ std::vector OwnedToF32(const NemotronHOwned& w) { return out; } -// ─── #1157 DIAGNOSTIC SCAFFOLD (VT_NEMOTRON_H_DIAG) ───────────────────────── +// ─── #1157 DIAGNOSTIC (VT_NEMOTRON_H_DIAG, documented in ENVIRONMENT.md) ──── // -// TEMPORARY. Off unless `VT_NEMOTRON_H_DIAG` is set to something other than -// "0", and every download it does is inside that guard, so a production step -// pays nothing. It exists to answer ONE question the CPU gate cannot: on the -// real checkpoint, is the recurrent state the decode step READS the state the -// previous step WROTE. +// Off unless `VT_NEMOTRON_H_DIAG` is set to something other than "0", and every +// download it does is inside that guard, so a production step pays nothing. +// +// It exists to answer the question no CPU gate on this model can: the runner +// hands a decode step a device-resident input id and a recurrent page, and when +// the tokens come out wrong, only the per-layer numbers say WHICH of the two the +// step actually read. On #1157 they said the carry was exact — the state +// gathered at step k+1 equalled the state written at step k, on host and on +// GB10 alike — and that layer 0's embedding row was constant across two decode +// steps that consumed different tokens. It stays for the next reader of this +// model, because the next divergence here will be diagnosed the same way. bool NemotronHDiagEnabled() { static const bool on = [] { const char* e = std::getenv("VT_NEMOTRON_H_DIAG");