diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/glm5.2-agentx-1p2d.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/glm5.2-agentx-1p2d.yaml new file mode 100644 index 0000000000..dfd5c50fd0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/glm5.2-agentx-1p2d.yaml @@ -0,0 +1,174 @@ +# Agentic-coding SGLang disaggregated 1P2D recipe for GLM-5.2-NVFP4 on B200. +# Expanded from NVIDIA/srt-slurm#314 at 2f43c324211022d367d8130e42499d579ecc90d9. +# Golden synthetic acceptance is injected only for throughput jobs; EVAL_ONLY +# retains real target-model verification. +name: agentx-1p2d_dtp8-c113 +model: + path: glm-5.2-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 + precision: fp4 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 + frameworks: + dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + sglang: nightly-dev-cu13-20260805-211ee642 +resources: + gpu_type: b200 + gpus_per_node: 8 + decode_nodes: 2 + decode_workers: 2 + gpus_per_decode: 8 + gpus_per_prefill: 8 + prefill_nodes: 1 + prefill_workers: 1 +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +dynamo: + hash: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + install: true +backend: + type: sglang + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + sglang_config: + prefill: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + chunked-prefill-size: 65536 + max-prefill-tokens: 8192 + max-running-requests: 16 + cuda-graph-max-bs: 16 + disable-cuda-graph: true + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_cutlass + fp4-gemm-backend: flashinfer_cutlass + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + context-length: 1048576 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 270 + hicache-io-backend: direct + speculative-algorithm: EAGLE + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 2 + enable-metrics: true + enable-cache-report: true + decode: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + disable-radix-cache: true + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 16 + cuda-graph-max-bs: 16 + chunked-prefill-size: 64 + context-length: 1048576 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_cutlass + skip-tokenizer-init: true + stream-interval: 30 + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.9 + disaggregation-decode-extra-slots: 0 + enable-metrics: true + enable-cache-report: true +health_check: + max_attempts: 1440 + interval_seconds: 10 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 64 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/glm5.2-agentx-1p3d.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/glm5.2-agentx-1p3d.yaml new file mode 100644 index 0000000000..be53227105 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/glm5.2-agentx-1p3d.yaml @@ -0,0 +1,174 @@ +# Agentic-coding SGLang disaggregated 1P3D recipe for GLM-5.2-NVFP4 on B200. +# Expanded from NVIDIA/srt-slurm#314 at 2f43c324211022d367d8130e42499d579ecc90d9. +# Golden synthetic acceptance is injected only for throughput jobs; EVAL_ONLY +# retains real target-model verification. +name: agentx-1p3d_dtp8-c113 +model: + path: glm-5.2-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 + precision: fp4 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 + frameworks: + dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + sglang: nightly-dev-cu13-20260805-211ee642 +resources: + gpu_type: b200 + gpus_per_node: 8 + decode_nodes: 3 + decode_workers: 3 + gpus_per_decode: 8 + gpus_per_prefill: 8 + prefill_nodes: 1 + prefill_workers: 1 +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +dynamo: + hash: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + install: true +backend: + type: sglang + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + sglang_config: + prefill: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + chunked-prefill-size: 65536 + max-prefill-tokens: 8192 + max-running-requests: 16 + cuda-graph-max-bs: 16 + disable-cuda-graph: true + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_cutlass + fp4-gemm-backend: flashinfer_cutlass + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + context-length: 1048576 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 270 + hicache-io-backend: direct + speculative-algorithm: EAGLE + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 2 + enable-metrics: true + enable-cache-report: true + decode: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + disable-radix-cache: true + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 16 + cuda-graph-max-bs: 16 + chunked-prefill-size: 64 + context-length: 1048576 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_cutlass + skip-tokenizer-init: true + stream-interval: 30 + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.9 + disaggregation-decode-extra-slots: 0 + enable-metrics: true + enable-cache-report: true +health_check: + max_attempts: 1440 + interval_seconds: 10 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 64 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/glm5.2-agentx-agg.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/glm5.2-agentx-agg.yaml new file mode 100644 index 0000000000..a4aafe4d78 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/glm5.2-agentx-agg.yaml @@ -0,0 +1,117 @@ +# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B200, +# ported from NVIDIA/srt-slurm#314 at 2f43c324211022d367d8130e42499d579ecc90d9. +# +# The InferenceX matrix supplies concurrency and runs this flat recipe through +# agentic_srt.sh. Synthetic acceptance is injected only for throughput jobs; +# EVAL_ONLY jobs retain real target-model verification. +name: b200-fp4-glm5.2-agentx-agg +model: + path: glm-5.2-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 + precision: fp4 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 + frameworks: + dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + sglang: nightly-dev-cu13-20260805-211ee642 +resources: + gpu_type: b200 + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 8 +frontend: + type: dynamo + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + router-session-affinity-ttl-secs: 3600 +dynamo: + hash: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + install: true +backend: + type: sglang + sglang_config: + aggregated: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 10 + cuda-graph-max-bs: 10 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + context-length: 1048576 + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_trtllm + disable-shared-experts-fusion: true + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 270 + hicache-io-backend: direct + enable-metrics: true + enable-cache-report: true + aggregated_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + PIP_BREAK_SYSTEM_PACKAGES: '1' +health_check: + max_attempts: 1440 + interval_seconds: 10 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '8' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 64 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 8719e6ff83..de2de91340 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8443,6 +8443,93 @@ glm5.2-fp4-b200-sglang-agentic-mtp: search-space: - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16] } +glm5.2-fp4-b200-dynamo-sglang-agentic-agg: + image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 + model: nvidia/GLM-5.2-NVFP4 + model-prefix: glm5.2 + runner: cluster:b200-nscale + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "71eb001e17fa73c742f0afe1a6ed96836cb135fd" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: mtp + conc-list: [2, 4, 8, 12] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.99" + - "CONFIG_FILE=recipes/sglang/glm5.2/b200-fp4/agentic/glm5.2-agentx-agg.yaml" + # Aggregated serving runs prefill and decode on the same TP8 worker. + # Keep decode at zero so matrix metadata does not double-count GPUs. + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false + +glm5.2-fp4-b200-dynamo-sglang-agentic-disagg: + image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 + model: nvidia/GLM-5.2-NVFP4 + model-prefix: glm5.2 + runner: cluster:b200-nscale + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "71eb001e17fa73c742f0afe1a6ed96836cb135fd" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: mtp + conc-list: [113] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.5" + - "CONFIG_FILE=recipes/sglang/glm5.2/b200-fp4/agentic/glm5.2-agentx-1p2d.yaml" + decode: + num-worker: 2 + tp: 8 + ep: 1 + dp-attn: true + - spec-decoding: mtp + conc-list: [113] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.5" + - "CONFIG_FILE=recipes/sglang/glm5.2/b200-fp4/agentic/glm5.2-agentx-1p3d.yaml" + decode: + num-worker: 3 + tp: 8 + ep: 1 + dp-attn: true + glm5.2-fp4-gb200-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:v0.5.17-cu130 model: nvidia/GLM-5.2-NVFP4 diff --git a/configs/runners.yaml b/configs/runners.yaml index 6b6fdd0c71..91184b9def 100644 --- a/configs/runners.yaml +++ b/configs/runners.yaml @@ -253,6 +253,16 @@ labels: - b200-dgxc_07 - b200-dgxc_08 - b200-dgxc_09 + cluster:b200-nscale: + - b200-nscale-slurm_0 + - b200-nscale-slurm_1 + - b200-nscale-slurm_2 + - b200-nscale-slurm_3 + - b200-nscale-slurm_4 + - b200-nscale-slurm_5 + - b200-nscale-slurm_6 + - b200-nscale-slurm_7 + - b200-nscale-slurm_8 cluster:b300-cw: - b300-cw_0 - b300-cw_01 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 55fc4f8671..fafd07a17c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6225,3 +6225,15 @@ - "Add a TP2/EP2 HiCache branch and update the resident recipes with extra_buffer_lazy, ReplaySSM speculative verification, and checkpoint page-cache release." - "Select TP4 C1/C2/C4/C8/C12/C20/C24, TP2/EP2 resident C4/C8/C12/C20, and TP2/EP2 HiCache C22/C24/C28/C32/C40 with golden synthetic acceptance length 3.39." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2660 + +- config-keys: + - glm5.2-fp4-b200-dynamo-sglang-agentic-agg + - glm5.2-fp4-b200-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Add GLM-5.2 NVFP4 B200 Dynamo-SGLang AgentX aggregate and disaggregated coverage." + - "Cover aggregate TP8 at concurrency 2, 4, 8, and 12, plus disaggregated 1P2D and 1P3D DEP8 at concurrency 113." + - "Use EAGLE MTP, golden synthetic acceptance for throughput only, HiCache DRAM offload, NIXL KV transfer for disaggregated serving, and KV-aware session affinity." + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642; recipe ported from NVIDIA/srt-slurm v1.0.53 (217f94387abeddfed7149a71955dc523e07cd765)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2673 diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index 4bf9531d0f..db729b6cc3 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -9,8 +9,9 @@ # dgxc-only change silently alter nscale runs. # # Scope: multi-node Dynamo-vLLM DeepSeek-V4-Pro and Kimi K2.6 FP4 runs, plus -# DeepSeek-V4-Pro FP4 Dynamo-SGLang MTP, on the b200-nscale/b200-new runner -# labels. Anything else exits non-zero. +# DeepSeek-V4-Pro FP4 Dynamo-SGLang MTP and GLM-5.2-NVFP4 Dynamo-SGLang +# AgentX MTP, on the b200-nscale/b200-new runner labels. Anything else exits +# non-zero. SLURM_PARTITION="batch_1" SLURM_ACCOUNT="benchmark" @@ -46,6 +47,9 @@ elif [[ $MODEL_PREFIX == "kimik2.6" && $PRECISION == "fp4" ]]; then elif [[ $MODEL_PREFIX == "kimik3" && $PRECISION == "fp4" ]]; then export MODEL_PATH="${MODEL_PATH:-$NSCALE_MODEL_ROOT/Kimi-K3}" export SRT_SLURM_MODEL_PREFIX="kimik3" +elif [[ $MODEL_PREFIX == "glm5.2" && $PRECISION == "fp4" ]]; then + export MODEL_PATH="${MODEL_PATH:-$NSCALE_MODEL_ROOT/GLM-5.2-NVFP4}" + export SRT_SLURM_MODEL_PREFIX="glm-5.2-fp4" else echo "Unsupported model prefix/precision for b200-nscale: $MODEL_PREFIX/$PRECISION" >&2 echo "Models staged under $NSCALE_MODEL_ROOT:" >&2 @@ -54,7 +58,8 @@ else fi if [[ $FRAMEWORK != "dynamo-vllm" ]] && - [[ $MODEL_PREFIX != "dsv4" || $PRECISION != "fp4" || $FRAMEWORK != "dynamo-sglang" || $SPEC_DECODING != "mtp" ]]; then + [[ $MODEL_PREFIX != "dsv4" || $PRECISION != "fp4" || $FRAMEWORK != "dynamo-sglang" || $SPEC_DECODING != "mtp" ]] && + [[ $MODEL_PREFIX != "glm5.2" || $PRECISION != "fp4" || $FRAMEWORK != "dynamo-sglang" || $SPEC_DECODING != "mtp" ]]; then echo "Unsupported framework/configuration for b200-nscale: $MODEL_PREFIX/$PRECISION/$FRAMEWORK/$SPEC_DECODING" >&2 exit 1 fi @@ -80,6 +85,16 @@ elif [[ $MODEL_PREFIX == "dsv4" && $FRAMEWORK == "dynamo-sglang" ]]; then git checkout 04e87fcc505d6d851451781a5499ca19a02ec2b4 || exit 1 mkdir -p recipes/sglang/deepseek-v4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4" recipes/sglang/deepseek-v4 +elif [[ $MODEL_PREFIX == "glm5.2" && $FRAMEWORK == "dynamo-sglang" ]]; then + # NVIDIA/srt-slurm v1.0.53 is the released schema validated against the + # GLM-5.2 B200 AgentX aggregate and disaggregated recipes. + git clone --branch v1.0.53 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + test "$(git rev-parse HEAD)" = "217f94387abeddfed7149a71955dc523e07cd765" || exit 1 + mkdir -p recipes/sglang/glm5.2/b200-fp4/agentic + cp -rT \ + "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic" \ + recipes/sglang/glm5.2/b200-fp4/agentic elif [[ $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 @@ -235,7 +250,8 @@ SRTCTL_PREFLIGHT_ARGS=() # These weights are staged on the Slurm compute nodes, not the login node. if [[ $MODEL_PREFIX == "kimik2.6" ]] || [[ $MODEL_PREFIX == "kimik3" ]] || - [[ $MODEL_PREFIX == "dsv4" && $FRAMEWORK == "dynamo-sglang" ]]; then + [[ $MODEL_PREFIX == "dsv4" && $FRAMEWORK == "dynamo-sglang" ]] || + [[ $MODEL_PREFIX == "glm5.2" ]]; then SRTCTL_PREFLIGHT_ARGS+=(--no-preflight) fi