diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml index d40942f792..7bb82d2b5b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml @@ -16,7 +16,7 @@ model: dynamo: install: true - wheel: "1.3.0.dev1" + wheel: "1.3.0.dev20260718" slurm: time_limit: "8:00:00" @@ -44,11 +44,10 @@ infra: frontend: type: dynamo nginx_session_affinity: true - nginx_session_affinity_header: X-Correlation-ID + nginx_session_affinity_header: X-Dynamo-Session-ID enable_multiple_frontends: true num_additional_frontends: 4 env: - DYN_ROUTER_TEMPERATURE: "10000000" # The sglang image is PEP 668 externally-managed; the runtime dynamo # install (dynamo_wheels.py / source build) runs pip and fails with # "externally-managed-environment" without this. Lets pip install into @@ -56,7 +55,7 @@ frontend: PIP_BREAK_SYSTEM_PACKAGES: "1" args: router-mode: "kv" - router-reset-states: true + router-session-affinity-ttl-secs: "3600" active-decode-blocks-threshold: "None" active-prefill-tokens-threshold: "None" active-prefill-tokens-threshold-frac: "None" @@ -66,6 +65,9 @@ backend: aggregated_environment: SGLANG_DEFAULT_THINKING: '1' + # TP4/bs128 can spend over an hour capturing the DSV4 EAGLE draft-extend + # graph. Keep the target/decode graphs and use the upstream eager fallback. + SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH: "1" SGLANG_DSV4_REASONING_EFFORT: high SGLANG_SIMULATE_ACC_LEN: '2.49' SGLANG_SIMULATE_ACC_METHOD: match-expected @@ -140,7 +142,8 @@ benchmark: # in process_agentic_result.py instead. TP must match sglang_config tp-size. IS_MULTINODE: "false" TP: "4" - # Bind each conversation through the supported Dynamo session header. + # Stamp X-Dynamo-Session-ID on every turn so both nginx and Dynamo's + # router keep the session on the worker that owns its KV prefix. AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml similarity index 96% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml index 7ed2dd5d2b..709962936f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload" +name: "disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload" # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (12P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 1024. +# (12P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 1536. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized @@ -15,7 +15,7 @@ model: dynamo: install: true - wheel: "1.3.0.dev1" + wheel: "1.3.0.dev20260718" slurm: time_limit: "8:00:00" @@ -41,18 +41,17 @@ infra: frontend: type: dynamo nginx_session_affinity: true - nginx_session_affinity_header: X-Correlation-ID + nginx_session_affinity_header: X-Dynamo-Session-ID enable_multiple_frontends: true num_additional_frontends: 4 env: - DYN_ROUTER_TEMPERATURE: "10000000" # AgentX warmup can legitimately keep the single wide decode worker busy # for longer than Dynamo's 10-second TCP request-plane default. DYN_TCP_REQUEST_TIMEOUT: "60" PIP_BREAK_SYSTEM_PACKAGES: "1" args: router-mode: "kv" - router-reset-states: true + router-session-affinity-ttl-secs: "3600" active-decode-blocks-threshold: "None" active-prefill-tokens-threshold: "None" active-prefill-tokens-threshold-frac: "None" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml similarity index 94% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml index 2ac8d01b52..e414387655 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload" +name: "disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload" # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (2P x DEP8 / 1D x TP4, MTP + hierarchical-cache KV offload), tuned for concurrency 8. +# (2P x DEP8 / 1D x TP4, MTP + hierarchical-cache KV offload), tuned for concurrency 80. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized @@ -15,7 +15,7 @@ model: dynamo: install: true - wheel: "1.3.0.dev1" + wheel: "1.3.0.dev20260718" slurm: time_limit: "8:00:00" @@ -41,15 +41,14 @@ infra: frontend: type: dynamo nginx_session_affinity: true - nginx_session_affinity_header: X-Correlation-ID + nginx_session_affinity_header: X-Dynamo-Session-ID enable_multiple_frontends: true num_additional_frontends: 4 env: - DYN_ROUTER_TEMPERATURE: "10000000" PIP_BREAK_SYSTEM_PACKAGES: "1" args: router-mode: "kv" - router-reset-states: true + router-session-affinity-ttl-secs: "3600" active-decode-blocks-threshold: "None" active-prefill-tokens-threshold: "None" active-prefill-tokens-threshold-frac: "None" @@ -92,6 +91,9 @@ backend: decode_environment: SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + # Match the TP4 aggregate worker: avoid the pathological DSV4 EAGLE + # draft-extend graph capture while retaining the target/decode graphs. + SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH: "1" SGLANG_SIMULATE_ACC_LEN: '2.49' SGLANG_SIMULATE_ACC_METHOD: match-expected SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml index 60d0fadda1..74b371e94e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml @@ -15,7 +15,7 @@ model: dynamo: install: true - wheel: "1.3.0.dev1" + wheel: "1.3.0.dev20260718" slurm: time_limit: "8:00:00" @@ -41,15 +41,14 @@ infra: frontend: type: dynamo nginx_session_affinity: true - nginx_session_affinity_header: X-Correlation-ID + nginx_session_affinity_header: X-Dynamo-Session-ID enable_multiple_frontends: true num_additional_frontends: 4 env: - DYN_ROUTER_TEMPERATURE: "10000000" PIP_BREAK_SYSTEM_PACKAGES: "1" args: router-mode: "kv" - router-reset-states: true + router-session-affinity-ttl-secs: "3600" active-decode-blocks-threshold: "None" active-prefill-tokens-threshold: "None" active-prefill-tokens-threshold-frac: "None" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml similarity index 96% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml index 25bdf62713..6b585fba80 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload" +name: "disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload" # Agentic-coding SGLang disaggregated recipe for DeepSeek-V4-Pro on GB300 -# (4P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 256. +# (4P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 512. # # DEP8-prefill variant aligned with the measured Pareto point: prefill uses # tp/dp/ep 8, four nodes, and SGLANG_DSV4_MHC_PREWARM=1. @@ -15,7 +15,7 @@ model: dynamo: install: true - wheel: "1.3.0.dev1" + wheel: "1.3.0.dev20260718" slurm: time_limit: "8:00:00" @@ -40,15 +40,14 @@ infra: frontend: type: dynamo nginx_session_affinity: true - nginx_session_affinity_header: X-Correlation-ID + nginx_session_affinity_header: X-Dynamo-Session-ID enable_multiple_frontends: true num_additional_frontends: 4 env: - DYN_ROUTER_TEMPERATURE: "10000000" PIP_BREAK_SYSTEM_PACKAGES: "1" args: router-mode: "kv" - router-reset-states: true + router-session-affinity-ttl-secs: "3600" active-decode-blocks-threshold: "None" active-prefill-tokens-threshold: "None" active-prefill-tokens-threshold-frac: "None" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml similarity index 96% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml index 3cb306c684..b2cac21c78 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload" +name: "disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload" # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (6P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 512. +# (6P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 768. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized @@ -15,7 +15,7 @@ model: dynamo: install: true - wheel: "1.3.0.dev1" + wheel: "1.3.0.dev20260718" slurm: time_limit: "8:00:00" @@ -41,15 +41,14 @@ infra: frontend: type: dynamo nginx_session_affinity: true - nginx_session_affinity_header: X-Correlation-ID + nginx_session_affinity_header: X-Dynamo-Session-ID enable_multiple_frontends: true num_additional_frontends: 4 env: - DYN_ROUTER_TEMPERATURE: "10000000" PIP_BREAK_SYSTEM_PACKAGES: "1" args: router-mode: "kv" - router-reset-states: true + router-session-affinity-ttl-secs: "3600" active-decode-blocks-threshold: "None" active-prefill-tokens-threshold: "None" active-prefill-tokens-threshold-frac: "None" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 393babf91b..3c07c4384a 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7419,13 +7419,13 @@ kimik3-fp4-gb200-dynamo-vllm-agentic: dp-attn: true dsv4-fp4-gb300-dynamo-sglang-agentic-agg: - image: lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8 + image: lmsysorg/sglang:nightly-dev-cu13-20260719-99f5a6f4 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.3.0.dev1" } + router: { name: dynamo-router, version: "1.3.0.dev20260718" } multinode: true disagg: false scenarios: @@ -7433,7 +7433,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg: - dram-utilization: 0.80 search-space: - spec-decoding: mtp - conc-list: [1, 4, 8, 16] + conc-list: [2, 4, 8, 16] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -7453,13 +7453,13 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg: dp-attn: false dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: - image: lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8 + image: lmsysorg/sglang:nightly-dev-cu13-20260719-99f5a6f4 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.3.0.dev1" } + router: { name: dynamo-router, version: "1.3.0.dev20260718" } kv-p2p-transfer: mooncake multinode: true disagg: true @@ -7468,7 +7468,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: - dram-utilization: 0.80 search-space: - spec-decoding: mtp - conc-list: [8] + conc-list: [80] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -7477,7 +7477,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 4 @@ -7500,7 +7500,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 16 dp-attn: true - spec-decoding: mtp - conc-list: [256] + conc-list: [512] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -7509,14 +7509,14 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - spec-decoding: mtp - conc-list: [512] + conc-list: [768] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -7525,14 +7525,14 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - spec-decoding: mtp - conc-list: [1024] + conc-list: [1536] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -7541,7 +7541,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 471981f60c..96052484e7 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5709,3 +5709,14 @@ - "Enable SGLang metrics on every aggregate, prefill, and decode engine." - "Use supported header-based Dynamo session routing with the in-repo AIPerf build." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2520 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-agg + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + description: + - "Upgrade Dynamo from 1.3.0.dev1 to 1.3.0.dev20260718 so router queueing is disabled by default at high concurrency" + - "Migrate AgentX session affinity from nvext.session_control to X-Dynamo-Session-ID headers with a 3600-second router TTL" + - "Remove the DYN_ROUTER_TEMPERATURE=10000000 override and restore Dynamo's deterministic 0.0 default" + - "Upgrade SGLang from nightly-dev-cu13-20260711-7de33ce8 to nightly-dev-cu13-20260719-99f5a6f4, the latest nightly before the scheduler WAR-barrier regression in SGLang #31687" + - "Restore the pre-watchdog AgentX concurrency grid and use AIPerf's default 300-second per-trajectory idle-gap cap" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2319