diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-agentic.yaml new file mode 100644 index 0000000000..421a096254 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-agentic.yaml @@ -0,0 +1,163 @@ +name: "minimax-m3-vllm-disagg-gb300-1p1d-tep4-tp4-c1-fp4-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + version: "1.4.0.dev20260730" + install: true + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: false + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + env: + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + vllm_config: + prefill: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 4 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 4 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true + prefill_environment: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + decode_environment: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-eval-agentic.yaml new file mode 100644 index 0000000000..3bed52725e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-eval-agentic.yaml @@ -0,0 +1,163 @@ +name: "minimax-m3-vllm-disagg-gb300-1p1d-tep4-tp4-c1-fp4-eval-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + version: "1.4.0.dev20260730" + install: true + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: false + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + env: + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + vllm_config: + prefill: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 4 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 4 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true + prefill_environment: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + decode_environment: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-agentic.yaml new file mode 100644 index 0000000000..fb4a6bf759 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-agentic.yaml @@ -0,0 +1,163 @@ +name: "minimax-m3-vllm-disagg-gb300-1p1d-tp2-tp4-c20-c24-fp4-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + version: "1.4.0.dev20260730" + install: true + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 4 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + env: + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + vllm_config: + prefill: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 4 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true + prefill_environment: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + decode_environment: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-eval-agentic.yaml new file mode 100644 index 0000000000..755d2a8a12 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-eval-agentic.yaml @@ -0,0 +1,163 @@ +name: "minimax-m3-vllm-disagg-gb300-1p1d-tp2-tp4-c20-c24-fp4-eval-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + version: "1.4.0.dev20260730" + install: true + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 4 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + env: + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + vllm_config: + prefill: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 4 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true + prefill_environment: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + decode_environment: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-agentic.yaml new file mode 100644 index 0000000000..7bf7fbb74e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-agentic.yaml @@ -0,0 +1,166 @@ +name: "minimax-m3-vllm-disagg-gb300-1p3d-dep4-tp4-c24-fp4-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + version: "1.4.0.dev20260730" + install: true + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 1 + decode_nodes: 3 + prefill_workers: 1 + decode_workers: 3 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + env: + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + vllm_config: + prefill: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 4 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true + prefill_environment: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + decode_environment: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-eval-agentic.yaml new file mode 100644 index 0000000000..261f1fe6d2 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-eval-agentic.yaml @@ -0,0 +1,166 @@ +name: "minimax-m3-vllm-disagg-gb300-1p3d-dep4-tp4-c24-fp4-eval-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + version: "1.4.0.dev20260730" + install: true + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 1 + decode_nodes: 3 + prefill_workers: 1 + decode_workers: 3 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + env: + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + vllm_config: + prefill: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 4 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true + prefill_environment: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + decode_environment: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-agentic.yaml new file mode 100644 index 0000000000..8193a1f919 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-agentic.yaml @@ -0,0 +1,163 @@ +name: "minimax-m3-vllm-disagg-gb300-1p3d-tp2-tp2-c48-fp4-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + version: "1.4.0.dev20260730" + install: true + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: true + prefill_nodes: 1 + decode_nodes: 3 + prefill_workers: 1 + decode_workers: 3 + gpus_per_prefill: 2 + gpus_per_decode: 2 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + env: + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + vllm_config: + prefill: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true + prefill_environment: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + decode_environment: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-eval-agentic.yaml new file mode 100644 index 0000000000..4a174c2c28 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-eval-agentic.yaml @@ -0,0 +1,163 @@ +name: "minimax-m3-vllm-disagg-gb300-1p3d-tp2-tp2-c48-fp4-eval-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + version: "1.4.0.dev20260730" + install: true + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: true + prefill_nodes: 1 + decode_nodes: 3 + prefill_workers: 1 + decode_workers: 3 + gpus_per_prefill: 2 + gpus_per_decode: 2 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + env: + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + vllm_config: + prefill: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true + prefill_environment: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + decode_environment: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-agentic.yaml new file mode 100644 index 0000000000..fe2d56097c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-agentic.yaml @@ -0,0 +1,163 @@ +name: "minimax-m3-vllm-disagg-gb300-2p5d-tp2-tp2-c120-fp4-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + version: "1.4.0.dev20260730" + install: true + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: true + prefill_nodes: 2 + decode_nodes: 5 + prefill_workers: 2 + decode_workers: 5 + gpus_per_prefill: 2 + gpus_per_decode: 2 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + env: + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + vllm_config: + prefill: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true + prefill_environment: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + decode_environment: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-eval-agentic.yaml new file mode 100644 index 0000000000..bf01406d14 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-eval-agentic.yaml @@ -0,0 +1,163 @@ +name: "minimax-m3-vllm-disagg-gb300-2p5d-tp2-tp2-c120-fp4-eval-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + version: "1.4.0.dev20260730" + install: true + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: true + prefill_nodes: 2 + decode_nodes: 5 + prefill_workers: 2 + decode_workers: 5 + gpus_per_prefill: 2 + gpus_per_decode: 2 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + env: + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + vllm_config: + prefill: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true + prefill_environment: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + decode_environment: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 50ed9b4713..5e5cee733e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7436,6 +7436,86 @@ minimaxm3-fp4-b300-vllm-agentic-mtp: search-space: - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [24] } +minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: + image: vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 + model: nvidia/MiniMax-M3-NVFP4 + model-prefix: minimaxm3 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.4.0.dev20260730" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.887 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [1] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-agentic.yaml" + - "EVAL_CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-eval-agentic.yaml" + decode: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [24] + prefill: + num-worker: 1 + tp: 1 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-agentic.yaml" + - "EVAL_CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-eval-agentic.yaml" + decode: { num-worker: 3, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [20, 24] + prefill: + num-worker: 1 + tp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-agentic.yaml" + - "EVAL_CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-eval-agentic.yaml" + decode: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [48] + prefill: + num-worker: 1 + tp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-agentic.yaml" + - "EVAL_CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-eval-agentic.yaml" + decode: { num-worker: 3, tp: 2, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [120] + prefill: + num-worker: 2 + tp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-agentic.yaml" + - "EVAL_CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-eval-agentic.yaml" + decode: { num-worker: 5, tp: 2, ep: 1, dp-attn: false } minimaxm3-fp4-b300-trtllm-agentic-mtp: image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 model: nvidia/MiniMax-M3-NVFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b2a5962b57..8d12d517ec 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6263,3 +6263,14 @@ description: - "Refresh with lower stream interval to collect correct client metrics" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2686 + +- config-keys: + - minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Add the MiniMax-M3 NVFP4 GB300 disaggregated AgentX config with EAGLE3-GQA MTP and Mooncake DRAM KV offload." + - "Use vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 with golden synthetic AL 2.78 for throughput and a real-verification eval twin." + - "Pin thinking-on so the run matches the golden minimaxm3_eagle3_gqa thinking_on[3] = 2.78 acceptance target: dyn-default-thinking-mode enabled on the decode worker covers both the AgentX replay and the eval twin, with AIPERF_EXTRA_INPUTS thinking:true on the replay as the GB200-sibling belt-and-braces." + - "Set dyn-reasoning-parser and dyn-tool-call-parser alongside reasoning-parser, matching the GB200 MiniMax-M3 AgentX recipes." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2663 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index ff6597d0bb..5a1d574769 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -80,6 +80,9 @@ elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp4" ]]; then elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp8" ]]; then export MODEL_PATH=/data/models/MiniMax-M2.5 export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-fp8" +elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp4" ]]; then + export MODEL_PATH=/scratch/models/MiniMax-M3-NVFP4 + export SRT_SLURM_MODEL_PREFIX="nvidia/MiniMax-M3-NVFP4" elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" ]]; then export MODEL_PATH=/data/models/MiniMax-M3-MXFP8 export SRT_SLURM_MODEL_PREFIX="minimax-m3-mxfp8" @@ -100,7 +103,7 @@ elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp8" ]]; then export MODEL_PATH=/scratch/models/Qwen3.5-397B-A17B-FP8 export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp8" else - echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4, glm5-fp4, glm5-fp8, glm5.2-fp4, minimaxm2.5-fp4, minimaxm2.5-fp8, kimik2.5-fp4, kimik3-fp4, qwen3.5-fp4, qwen3.5-fp8" + echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4, glm5-fp4, glm5-fp8, glm5.2-fp4, minimaxm2.5-fp4, minimaxm2.5-fp8, minimaxm3-fp4, minimaxm3-fp8, kimik2.5-fp4, kimik3-fp4, qwen3.5-fp4, qwen3.5-fp8" exit 1 fi @@ -184,6 +187,10 @@ if [[ "$USES_DCGM_POWER" == "1" ]]; then fi export EVAL_ONLY="${EVAL_ONLY:-false}" +if [[ "$EVAL_ONLY" == "true" && -n "${EVAL_CONFIG_FILE:-}" ]]; then + CONFIG_FILE="$EVAL_CONFIG_FILE" + echo "EVAL_ONLY=true: selecting real-verification recipe $CONFIG_FILE" +fi export ISL="$ISL" export OSL="$OSL" @@ -261,6 +268,9 @@ elif [[ "$IS_AGENTIC" == "1" ]]; then mkdir -p recipes/vllm/deepseek-v4/agentic || exit 1 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ recipes/vllm/deepseek-v4/agentic || exit 1 + mkdir -p recipes/vllm/minimax-m3/agentic || exit 1 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic" \ + recipes/vllm/minimax-m3/agentic || exit 1 mkdir -p recipes/vllm/kimi-k3/agentic || exit 1 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ recipes/vllm/kimi-k3/agentic || exit 1