From c0f60f3ea0b40574bbc153ec54e5092a4a8c1394 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Tue, 18 Aug 2026 15:54:15 -0700 Subject: [PATCH 1/4] feat(minimaxm3): add gb300 disaggregated agentx config --- .../gb300-fp4/1p1d-tep4-tp4-c1-agentic.yaml | 157 +++++++++++++++++ .../1p1d-tep4-tp4-c1-eval-agentic.yaml | 157 +++++++++++++++++ .../1p1d-tp2-tp4-c20-c24-agentic.yaml | 157 +++++++++++++++++ .../1p1d-tp2-tp4-c20-c24-eval-agentic.yaml | 157 +++++++++++++++++ .../gb300-fp4/1p3d-dep4-tp4-c24-agentic.yaml | 160 ++++++++++++++++++ .../1p3d-dep4-tp4-c24-eval-agentic.yaml | 160 ++++++++++++++++++ .../gb300-fp4/1p3d-tp2-tp2-c48-agentic.yaml | 157 +++++++++++++++++ .../1p3d-tp2-tp2-c48-eval-agentic.yaml | 157 +++++++++++++++++ .../gb300-fp4/2p5d-tp2-tp2-c120-agentic.yaml | 157 +++++++++++++++++ .../2p5d-tp2-tp2-c120-eval-agentic.yaml | 157 +++++++++++++++++ configs/nvidia-master.yaml | 81 +++++++++ perf-changelog.yaml | 9 + runners/launch_gb300-nv.sh | 12 +- 13 files changed, 1677 insertions(+), 1 deletion(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-eval-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-eval-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-eval-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-eval-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-eval-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-agentic.yaml new file mode 100644 index 0000000000..c69d2b79c2 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-agentic.yaml @@ -0,0 +1,157 @@ +name: "minimax-m3-vllm-disagg-gb300-1p1d-tep4-tp4-c1-fp4-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + version: "1.4.0.dev20260730" + install: true + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: false + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + env: + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + vllm_config: + prefill: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 4 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 4 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true + prefill_environment: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + decode_environment: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-eval-agentic.yaml new file mode 100644 index 0000000000..fee82caece --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-eval-agentic.yaml @@ -0,0 +1,157 @@ +name: "minimax-m3-vllm-disagg-gb300-1p1d-tep4-tp4-c1-fp4-eval-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + version: "1.4.0.dev20260730" + install: true + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: false + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + env: + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + vllm_config: + prefill: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 4 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 4 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true + prefill_environment: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + decode_environment: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-agentic.yaml new file mode 100644 index 0000000000..a2e4a09c24 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-agentic.yaml @@ -0,0 +1,157 @@ +name: "minimax-m3-vllm-disagg-gb300-1p1d-tp2-tp4-c20-c24-fp4-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + version: "1.4.0.dev20260730" + install: true + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 4 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + env: + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + vllm_config: + prefill: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 4 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true + prefill_environment: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + decode_environment: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-eval-agentic.yaml new file mode 100644 index 0000000000..7260d4bbd9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-eval-agentic.yaml @@ -0,0 +1,157 @@ +name: "minimax-m3-vllm-disagg-gb300-1p1d-tp2-tp4-c20-c24-fp4-eval-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + version: "1.4.0.dev20260730" + install: true + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 4 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + env: + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + vllm_config: + prefill: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 4 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true + prefill_environment: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + decode_environment: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-agentic.yaml new file mode 100644 index 0000000000..81835481ae --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-agentic.yaml @@ -0,0 +1,160 @@ +name: "minimax-m3-vllm-disagg-gb300-1p3d-dep4-tp4-c24-fp4-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + version: "1.4.0.dev20260730" + install: true + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 1 + decode_nodes: 3 + prefill_workers: 1 + decode_workers: 3 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + env: + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + vllm_config: + prefill: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 4 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true + prefill_environment: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + decode_environment: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-eval-agentic.yaml new file mode 100644 index 0000000000..b58237b346 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-eval-agentic.yaml @@ -0,0 +1,160 @@ +name: "minimax-m3-vllm-disagg-gb300-1p3d-dep4-tp4-c24-fp4-eval-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + version: "1.4.0.dev20260730" + install: true + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 1 + decode_nodes: 3 + prefill_workers: 1 + decode_workers: 3 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + env: + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + vllm_config: + prefill: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 4 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true + prefill_environment: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + decode_environment: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-agentic.yaml new file mode 100644 index 0000000000..ab743a38ae --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-agentic.yaml @@ -0,0 +1,157 @@ +name: "minimax-m3-vllm-disagg-gb300-1p3d-tp2-tp2-c48-fp4-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + version: "1.4.0.dev20260730" + install: true + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: true + prefill_nodes: 1 + decode_nodes: 3 + prefill_workers: 1 + decode_workers: 3 + gpus_per_prefill: 2 + gpus_per_decode: 2 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + env: + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + vllm_config: + prefill: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true + prefill_environment: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + decode_environment: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-eval-agentic.yaml new file mode 100644 index 0000000000..40ac18d2a7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-eval-agentic.yaml @@ -0,0 +1,157 @@ +name: "minimax-m3-vllm-disagg-gb300-1p3d-tp2-tp2-c48-fp4-eval-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + version: "1.4.0.dev20260730" + install: true + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: true + prefill_nodes: 1 + decode_nodes: 3 + prefill_workers: 1 + decode_workers: 3 + gpus_per_prefill: 2 + gpus_per_decode: 2 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + env: + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + vllm_config: + prefill: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true + prefill_environment: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + decode_environment: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-agentic.yaml new file mode 100644 index 0000000000..b24f72396b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-agentic.yaml @@ -0,0 +1,157 @@ +name: "minimax-m3-vllm-disagg-gb300-2p5d-tp2-tp2-c120-fp4-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + version: "1.4.0.dev20260730" + install: true + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: true + prefill_nodes: 2 + decode_nodes: 5 + prefill_workers: 2 + decode_workers: 5 + gpus_per_prefill: 2 + gpus_per_decode: 2 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + env: + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + vllm_config: + prefill: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true + prefill_environment: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + decode_environment: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-eval-agentic.yaml new file mode 100644 index 0000000000..90987f009b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-eval-agentic.yaml @@ -0,0 +1,157 @@ +name: "minimax-m3-vllm-disagg-gb300-2p5d-tp2-tp2-c120-fp4-eval-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + version: "1.4.0.dev20260730" + install: true + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: true + prefill_nodes: 2 + decode_nodes: 5 + prefill_workers: 2 + decode_workers: 5 + gpus_per_prefill: 2 + gpus_per_decode: 2 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + env: + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + vllm_config: + prefill: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true + prefill_environment: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + decode_environment: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 0af0ebfba8..b35b5c5b68 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7298,6 +7298,87 @@ minimaxm3-fp4-b300-vllm-agentic-mtp: search-space: - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [24] } +minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: + image: vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 + model: nvidia/MiniMax-M3-NVFP4 + model-prefix: minimaxm3 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.4.0.dev20260730" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.887 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [1] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-agentic.yaml" + - "EVAL_CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-eval-agentic.yaml" + decode: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [24] + prefill: + num-worker: 1 + tp: 1 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-agentic.yaml" + - "EVAL_CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-eval-agentic.yaml" + decode: { num-worker: 3, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [20, 24] + prefill: + num-worker: 1 + tp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-agentic.yaml" + - "EVAL_CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-eval-agentic.yaml" + decode: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [48] + prefill: + num-worker: 1 + tp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-agentic.yaml" + - "EVAL_CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-eval-agentic.yaml" + decode: { num-worker: 3, tp: 2, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [120] + prefill: + num-worker: 2 + tp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-agentic.yaml" + - "EVAL_CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-eval-agentic.yaml" + decode: { num-worker: 5, tp: 2, ep: 1, dp-attn: false } + # GB200 DeepSeek-V4 disaggregated AgentX frontier. The 3P/2D TEP8/TP8 curve # covers the middle/high-interactivity range omitted by the one-decode DEP # throughput curves below. Each engine start carries at most four concurrencies. diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 48012c8c5d..4e4fcbafe6 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6103,3 +6103,12 @@ - "Use a 270 GB/rank target HiCache pool for the GLM-5.2 B200 TP8 c12 and c16 AgentX points to avoid the host-cache capacity cliff." - "Keep the existing ratio-0.75 pool for c1, c4, and c8 while retaining the SGLang EAGLE MTP and golden synthetic acceptance setup." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2652 + +- config-keys: + - minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Add the MiniMax-M3 NVFP4 GB300 disaggregated AgentX config with EAGLE3-GQA MTP and Mooncake DRAM KV offload." + - "Use vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 with golden synthetic AL 2.78 for throughput and a real-verification eval twin." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 88b964a8ad..efbd53b857 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -80,6 +80,9 @@ elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp4" ]]; then elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp8" ]]; then export MODEL_PATH=/data/models/MiniMax-M2.5 export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-fp8" +elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp4" ]]; then + export MODEL_PATH=/scratch/models/MiniMax-M3-NVFP4 + export SRT_SLURM_MODEL_PREFIX="nvidia/MiniMax-M3-NVFP4" elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" ]]; then export MODEL_PATH=/data/models/MiniMax-M3-MXFP8 export SRT_SLURM_MODEL_PREFIX="minimax-m3-mxfp8" @@ -97,7 +100,7 @@ elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp8" ]]; then export MODEL_PATH=/scratch/models/Qwen3.5-397B-A17B-FP8 export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp8" else - echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4, glm5-fp4, glm5-fp8, glm5.2-fp4, minimaxm2.5-fp4, minimaxm2.5-fp8, kimik2.5-fp4, qwen3.5-fp4, qwen3.5-fp8" + echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4, glm5-fp4, glm5-fp8, glm5.2-fp4, minimaxm2.5-fp4, minimaxm2.5-fp8, minimaxm3-fp4, minimaxm3-fp8, kimik2.5-fp4, qwen3.5-fp4, qwen3.5-fp8" exit 1 fi @@ -180,6 +183,10 @@ if [[ "$USES_DCGM_POWER" == "1" ]]; then fi export EVAL_ONLY="${EVAL_ONLY:-false}" +if [[ "$EVAL_ONLY" == "true" && -n "${EVAL_CONFIG_FILE:-}" ]]; then + CONFIG_FILE="$EVAL_CONFIG_FILE" + echo "EVAL_ONLY=true: selecting real-verification recipe $CONFIG_FILE" +fi export ISL="$ISL" export OSL="$OSL" @@ -257,6 +264,9 @@ elif [[ "$IS_AGENTIC" == "1" ]]; then mkdir -p recipes/vllm/deepseek-v4/agentic || exit 1 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ recipes/vllm/deepseek-v4/agentic || exit 1 + mkdir -p recipes/vllm/minimax-m3/agentic || exit 1 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic" \ + recipes/vllm/minimax-m3/agentic || exit 1 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" From 1fbad05e2f76de76af247a2213c013dd7815a58d Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Tue, 18 Aug 2026 15:55:44 -0700 Subject: [PATCH 2/4] chore(changelog): link PR #2663 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4e4fcbafe6..47924c053c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6111,4 +6111,4 @@ description: - "Add the MiniMax-M3 NVFP4 GB300 disaggregated AgentX config with EAGLE3-GQA MTP and Mooncake DRAM KV offload." - "Use vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 with golden synthetic AL 2.78 for throughput and a real-verification eval twin." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2663 From e4b0828d6e6e72c31d508cdb05174ee2a488b2d3 Mon Sep 17 00:00:00 2001 From: Ankur-singh Date: Thu, 20 Aug 2026 21:04:55 -0700 Subject: [PATCH 3/4] =?UTF-8?q?fix(config):=20run=20MiniMax-M3=20GB300=20A?= =?UTF-8?q?gentX=20thinking-on=20to=20match=20golden=20AL=20/=20=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=20MiniMax-M3=20GB300=20AgentX=20=E6=80=9D=E8=80=83?= =?UTF-8?q?=E6=A8=A1=E5=BC=8F=E4=BB=A5=E5=8C=B9=E9=85=8D=E9=BB=84=E9=87=91?= =?UTF-8?q?=20AL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 10 GB300 disagg recipes pin synthetic_acceptance_length 2.78, which is golden_al_distribution/minimaxm3_eagle3_gqa.yaml minimax-m3.thinking_on[3], but nothing enabled thinking, so the sweep served thinking-off traffic (thinking_off[3] = 2.97). Eval logs from run 32194995479 confirm it: ~99.4% of gsm8k responses opened with a bare empty reasoning block. Adopt the GB200 MiniMax-M3 AgentX pattern, which is the established shape for this dynamo-vllm agentic stack: - AIPERF_EXTRA_INPUTS: thinking:true in benchmark.env - dyn-tool-call-parser / dyn-reasoning-parser alongside reasoning-parser in both the prefill and decode vLLM configs All 6 merged gb200-fp4/agentic recipes carry all three; these 10 carried none. Requires a fresh full sweep: acceptance target and token profile both change. --- .../agentic/gb300-fp4/1p1d-tep4-tp4-c1-agentic.yaml | 5 +++++ .../agentic/gb300-fp4/1p1d-tep4-tp4-c1-eval-agentic.yaml | 5 +++++ .../agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-agentic.yaml | 5 +++++ .../agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-eval-agentic.yaml | 5 +++++ .../agentic/gb300-fp4/1p3d-dep4-tp4-c24-agentic.yaml | 5 +++++ .../agentic/gb300-fp4/1p3d-dep4-tp4-c24-eval-agentic.yaml | 5 +++++ .../agentic/gb300-fp4/1p3d-tp2-tp2-c48-agentic.yaml | 5 +++++ .../agentic/gb300-fp4/1p3d-tp2-tp2-c48-eval-agentic.yaml | 5 +++++ .../agentic/gb300-fp4/2p5d-tp2-tp2-c120-agentic.yaml | 5 +++++ .../agentic/gb300-fp4/2p5d-tp2-tp2-c120-eval-agentic.yaml | 5 +++++ perf-changelog.yaml | 1 + 11 files changed, 51 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-agentic.yaml index c69d2b79c2..0d23566f87 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-agentic.yaml @@ -81,6 +81,8 @@ backend: language-model-only: true no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 stream-interval: 20 @@ -101,6 +103,8 @@ backend: language-model-only: true no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 max-num-seqs: 1024 @@ -151,6 +155,7 @@ benchmark: PORT: "8000" IS_MULTINODE: "true" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-eval-agentic.yaml index fee82caece..6d48b6bef5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-eval-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-eval-agentic.yaml @@ -81,6 +81,8 @@ backend: language-model-only: true no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 stream-interval: 20 @@ -101,6 +103,8 @@ backend: language-model-only: true no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 max-num-seqs: 1024 @@ -151,6 +155,7 @@ benchmark: PORT: "8000" IS_MULTINODE: "true" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-agentic.yaml index a2e4a09c24..9356135d0f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-agentic.yaml @@ -82,6 +82,8 @@ backend: language-model-only: true no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 stream-interval: 20 @@ -101,6 +103,8 @@ backend: language-model-only: true no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 max-num-seqs: 1024 @@ -151,6 +155,7 @@ benchmark: PORT: "8000" IS_MULTINODE: "true" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-eval-agentic.yaml index 7260d4bbd9..14183749f3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-eval-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-eval-agentic.yaml @@ -82,6 +82,8 @@ backend: language-model-only: true no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 stream-interval: 20 @@ -101,6 +103,8 @@ backend: language-model-only: true no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 max-num-seqs: 1024 @@ -151,6 +155,7 @@ benchmark: PORT: "8000" IS_MULTINODE: "true" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-agentic.yaml index 81835481ae..1e05c0bae7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-agentic.yaml @@ -82,6 +82,8 @@ backend: language-model-only: true no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 stream-interval: 20 @@ -104,6 +106,8 @@ backend: language-model-only: true no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 max-num-seqs: 1024 @@ -154,6 +158,7 @@ benchmark: PORT: "8000" IS_MULTINODE: "true" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-eval-agentic.yaml index b58237b346..e87af72a67 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-eval-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-eval-agentic.yaml @@ -82,6 +82,8 @@ backend: language-model-only: true no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 stream-interval: 20 @@ -104,6 +106,8 @@ backend: language-model-only: true no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 max-num-seqs: 1024 @@ -154,6 +158,7 @@ benchmark: PORT: "8000" IS_MULTINODE: "true" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-agentic.yaml index ab743a38ae..5694941ca5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-agentic.yaml @@ -82,6 +82,8 @@ backend: language-model-only: true no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 stream-interval: 20 @@ -101,6 +103,8 @@ backend: language-model-only: true no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 max-num-seqs: 1024 @@ -151,6 +155,7 @@ benchmark: PORT: "8000" IS_MULTINODE: "true" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-eval-agentic.yaml index 40ac18d2a7..bdac0c5178 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-eval-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-eval-agentic.yaml @@ -82,6 +82,8 @@ backend: language-model-only: true no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 stream-interval: 20 @@ -101,6 +103,8 @@ backend: language-model-only: true no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 max-num-seqs: 1024 @@ -151,6 +155,7 @@ benchmark: PORT: "8000" IS_MULTINODE: "true" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-agentic.yaml index b24f72396b..587b77595e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-agentic.yaml @@ -82,6 +82,8 @@ backend: language-model-only: true no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 stream-interval: 20 @@ -101,6 +103,8 @@ backend: language-model-only: true no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 max-num-seqs: 1024 @@ -151,6 +155,7 @@ benchmark: PORT: "8000" IS_MULTINODE: "true" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-eval-agentic.yaml index 90987f009b..4bf431139a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-eval-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-eval-agentic.yaml @@ -82,6 +82,8 @@ backend: language-model-only: true no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 stream-interval: 20 @@ -101,6 +103,8 @@ backend: language-model-only: true no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 max-num-seqs: 1024 @@ -151,6 +155,7 @@ benchmark: PORT: "8000" IS_MULTINODE: "true" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" HF_HUB_CACHE: "/hf_hub_cache" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ddf86ebe2c..be61fa74a3 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6271,4 +6271,5 @@ description: - "Add the MiniMax-M3 NVFP4 GB300 disaggregated AgentX config with EAGLE3-GQA MTP and Mooncake DRAM KV offload." - "Use vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 with golden synthetic AL 2.78 for throughput and a real-verification eval twin." + - "Drive the AgentX replay thinking-on via AIPERF_EXTRA_INPUTS thinking:true, and set dyn-reasoning-parser / dyn-tool-call-parser, matching the GB200 MiniMax-M3 AgentX recipes so the run matches the golden minimaxm3_eagle3_gqa thinking_on[3] = 2.78 acceptance target." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2663 From d30a3697a2d19ba7b46c153aaecc3ce4c9360678 Mon Sep 17 00:00:00 2001 From: Ankur-singh Date: Thu, 20 Aug 2026 21:08:17 -0700 Subject: [PATCH 4/4] =?UTF-8?q?fix(config):=20cover=20the=20eval=20twin=20?= =?UTF-8?q?with=20dyn-default-thinking-mode=20/=20=E7=94=A8=20dyn-default-?= =?UTF-8?q?thinking-mode=20=E8=A6=86=E7=9B=96=E8=AF=84=E6=B5=8B=E9=85=8D?= =?UTF-8?q?=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AIPERF_EXTRA_INPUTS only reaches the aiperf replay. The eval twin runs lm_eval, whose payload srt-slurm builds from a fixed env allow-list (cli/do_sweep.py), so benchmark.env can never reach it. Left alone, the accuracy gate would keep validating a different thinking mode than the throughput arm it is meant to certify. --dyn-default-thinking-mode is absorbed by dynamo.vllm's parse_known_args before the strict AsyncEngineArgs parse, published to the runtime config for non-prefill workers only (vllm/main.py:699-707), and applied by the frontend to any chat request that carries no explicit thinking control. Both arms hit the same frontend, so decode-side is enough to cover them. Set on the decode block only; prefill workers never publish it. The flag landed in dynamo 32c8091a8, before the 1.4.0.dev20260730 nightly cut this config pins, and is absent from the 1.3.1 the GB200 siblings use, which is why they had no server-side option available. --- .../minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-agentic.yaml | 1 + .../agentic/gb300-fp4/1p1d-tep4-tp4-c1-eval-agentic.yaml | 1 + .../agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-agentic.yaml | 1 + .../agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-eval-agentic.yaml | 1 + .../agentic/gb300-fp4/1p3d-dep4-tp4-c24-agentic.yaml | 1 + .../agentic/gb300-fp4/1p3d-dep4-tp4-c24-eval-agentic.yaml | 1 + .../minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-agentic.yaml | 1 + .../agentic/gb300-fp4/1p3d-tp2-tp2-c48-eval-agentic.yaml | 1 + .../agentic/gb300-fp4/2p5d-tp2-tp2-c120-agentic.yaml | 1 + .../agentic/gb300-fp4/2p5d-tp2-tp2-c120-eval-agentic.yaml | 1 + perf-changelog.yaml | 3 ++- 11 files changed, 12 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-agentic.yaml index 0d23566f87..421a096254 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-agentic.yaml @@ -105,6 +105,7 @@ backend: reasoning-parser: "minimax_m3" dyn-tool-call-parser: "minimax_m3" dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 max-num-seqs: 1024 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-eval-agentic.yaml index 6d48b6bef5..3bed52725e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-eval-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-eval-agentic.yaml @@ -105,6 +105,7 @@ backend: reasoning-parser: "minimax_m3" dyn-tool-call-parser: "minimax_m3" dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 max-num-seqs: 1024 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-agentic.yaml index 9356135d0f..fb4a6bf759 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-agentic.yaml @@ -105,6 +105,7 @@ backend: reasoning-parser: "minimax_m3" dyn-tool-call-parser: "minimax_m3" dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 max-num-seqs: 1024 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-eval-agentic.yaml index 14183749f3..755d2a8a12 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-eval-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-eval-agentic.yaml @@ -105,6 +105,7 @@ backend: reasoning-parser: "minimax_m3" dyn-tool-call-parser: "minimax_m3" dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 max-num-seqs: 1024 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-agentic.yaml index 1e05c0bae7..7bf7fbb74e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-agentic.yaml @@ -108,6 +108,7 @@ backend: reasoning-parser: "minimax_m3" dyn-tool-call-parser: "minimax_m3" dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 max-num-seqs: 1024 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-eval-agentic.yaml index e87af72a67..261f1fe6d2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-eval-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-eval-agentic.yaml @@ -108,6 +108,7 @@ backend: reasoning-parser: "minimax_m3" dyn-tool-call-parser: "minimax_m3" dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 max-num-seqs: 1024 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-agentic.yaml index 5694941ca5..8193a1f919 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-agentic.yaml @@ -105,6 +105,7 @@ backend: reasoning-parser: "minimax_m3" dyn-tool-call-parser: "minimax_m3" dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 max-num-seqs: 1024 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-eval-agentic.yaml index bdac0c5178..4a174c2c28 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-eval-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-eval-agentic.yaml @@ -105,6 +105,7 @@ backend: reasoning-parser: "minimax_m3" dyn-tool-call-parser: "minimax_m3" dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 max-num-seqs: 1024 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-agentic.yaml index 587b77595e..fe2d56097c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-agentic.yaml @@ -105,6 +105,7 @@ backend: reasoning-parser: "minimax_m3" dyn-tool-call-parser: "minimax_m3" dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 max-num-seqs: 1024 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-eval-agentic.yaml index 4bf431139a..bf01406d14 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-eval-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-eval-agentic.yaml @@ -105,6 +105,7 @@ backend: reasoning-parser: "minimax_m3" dyn-tool-call-parser: "minimax_m3" dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 max-num-seqs: 1024 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index be61fa74a3..8d12d517ec 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6271,5 +6271,6 @@ description: - "Add the MiniMax-M3 NVFP4 GB300 disaggregated AgentX config with EAGLE3-GQA MTP and Mooncake DRAM KV offload." - "Use vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 with golden synthetic AL 2.78 for throughput and a real-verification eval twin." - - "Drive the AgentX replay thinking-on via AIPERF_EXTRA_INPUTS thinking:true, and set dyn-reasoning-parser / dyn-tool-call-parser, matching the GB200 MiniMax-M3 AgentX recipes so the run matches the golden minimaxm3_eagle3_gqa thinking_on[3] = 2.78 acceptance target." + - "Pin thinking-on so the run matches the golden minimaxm3_eagle3_gqa thinking_on[3] = 2.78 acceptance target: dyn-default-thinking-mode enabled on the decode worker covers both the AgentX replay and the eval twin, with AIPERF_EXTRA_INPUTS thinking:true on the replay as the GB200-sibling belt-and-braces." + - "Set dyn-reasoning-parser and dyn-tool-call-parser alongside reasoning-parser, matching the GB200 MiniMax-M3 AgentX recipes." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2663