diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index a4fe70803f..9cec822cca 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1996,7 +1996,7 @@ build_replay_cmd() { # rolling TTFT/ITL/throughput block and emit it every 30 seconds. export AIPERF_UI_REALTIME_METRICS_ENABLED=true REPLAY_CMD="$AIPERF_CLI profile --scenario inferencex-agentx-mvp" - REPLAY_CMD+=" --url http://localhost:$PORT" + REPLAY_CMD+=" --url ${AIPERF_SERVER_URL:-http://localhost:$PORT}" REPLAY_CMD+=" --endpoint /v1/chat/completions" REPLAY_CMD+=" --endpoint-type chat" REPLAY_CMD+=" --streaming" diff --git a/benchmarks/multi_node/agentic_srt.sh b/benchmarks/multi_node/agentic_srt.sh index 79a36da524..77c196c133 100644 --- a/benchmarks/multi_node/agentic_srt.sh +++ b/benchmarks/multi_node/agentic_srt.sh @@ -9,7 +9,12 @@ set -x INFMAX_CONTAINER_WORKSPACE="${INFMAX_CONTAINER_WORKSPACE:-/infmax-workspace}" source "$INFMAX_CONTAINER_WORKSPACE/benchmarks/benchmark_lib.sh" -check_env_vars MODEL MODEL_PREFIX FRAMEWORK PRECISION CONC RESULT_FILENAME DURATION +check_env_vars \ + MODEL MODEL_PREFIX FRAMEWORK PRECISION CONC RESULT_FILENAME DURATION \ + SRT_FRONTEND_HOST SRT_FRONTEND_PORT + +export AIPERF_SERVER_URL="http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" +echo "Using srt-slurm frontend endpoint: $AIPERF_SERVER_URL" BASE_RESULT_DIR="${RESULT_DIR:-/logs/agentic}" BASE_RESULT_FILENAME="$RESULT_FILENAME" @@ -32,7 +37,7 @@ install_agentic_deps wait_for_agentic_servers_idle() { local timeout_seconds="${AIPERF_DRAIN_TIMEOUT_SECONDS:-1800}" local poll_seconds="${AIPERF_DRAIN_POLL_SECONDS:-10}" - local frontend_metrics_url="http://localhost:${PORT}/metrics" + local frontend_metrics_url="${AIPERF_SERVER_URL%/}/metrics" "$AIPERF_PYTHON" - \ "$timeout_seconds" \ diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-c20-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-c20-b5-mtp5.yaml new file mode 100644 index 0000000000..54e5eda1e0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-c20-b5-mtp5.yaml @@ -0,0 +1,204 @@ +name: dynamo-disagg-b300-1p1d-tep8-c20-b5-mtp5 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 + +identity: + model: + repo: "nvidia/GLM-5.2-NVFP4" + revision: "aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa" + container: + image: "nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1" + frameworks: + dynamo: "1.4.0.dev20260807" + tensorrt_llm: "1.3.0rc22.post1" + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + request_plane: tcp + +slurm: + time_limit: "4:00:00" + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: b300 + gpus_per_node: 8 + spread_workers: true + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + +backend: + type: trtllm + prefill_environment: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + UCX_MAX_RNDV_RAILS: "2" + UCX_MAX_HCA_PER_GPU: inf + UCX_LOG_LEVEL: info + TRTLLM_CTX_LOCAL_HCA_PIN: "1" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "2.61" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + decode_environment: *server_environment + trtllm_config: + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 412316860416 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: EQUAL_PROGRESS + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 5 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1, 2, 4, 5] + enable_padding: true + enable_attention_dp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + host_cache_size: 0 + free_gpu_memory_fraction: 0.8 + tokens_per_block: 64 + max_batch_size: 5 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 8 + publish_events_and_metrics: false + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + orchestrator_placement: first_decode + +benchmark: + type: custom + client_placement: last_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: none + MODEL: nvidia/GLM-5.2-NVFP4 + MODEL_PREFIX: glm5.2 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '20' + DURATION: '3600' + RESULT_FILENAME: glm52_fp4_dynamo-trt_1p1d-tep8_c20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p3d-tep8-c1-b1-mtp7.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p3d-tep8-c1-b1-mtp7.yaml new file mode 100644 index 0000000000..7818fd3435 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p3d-tep8-c1-b1-mtp7.yaml @@ -0,0 +1,204 @@ +name: dynamo-disagg-b300-1p3d-tep8-c1-b1-mtp7 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 + +identity: + model: + repo: "nvidia/GLM-5.2-NVFP4" + revision: "aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa" + container: + image: "nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1" + frameworks: + dynamo: "1.4.0.dev20260807" + tensorrt_llm: "1.3.0rc22.post1" + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + request_plane: tcp + +slurm: + time_limit: "4:00:00" + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: b300 + gpus_per_node: 8 + spread_workers: true + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 3 + decode_workers: 3 + gpus_per_decode: 8 + +backend: + type: trtllm + prefill_environment: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + UCX_MAX_RNDV_RAILS: "2" + UCX_MAX_HCA_PER_GPU: inf + UCX_LOG_LEVEL: info + TRTLLM_CTX_LOCAL_HCA_PIN: "1" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "2.91" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + decode_environment: *server_environment + trtllm_config: + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 412316860416 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: EQUAL_PROGRESS + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 7 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1] + enable_padding: true + enable_attention_dp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + host_cache_size: 0 + free_gpu_memory_fraction: 0.8 + tokens_per_block: 64 + max_batch_size: 1 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + speculative_config: + decoding_type: MTP + max_draft_len: 7 + stream_interval: 20 + tensor_parallel_size: 8 + publish_events_and_metrics: false + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + orchestrator_placement: first_decode + +benchmark: + type: custom + client_placement: last_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: none + MODEL: nvidia/GLM-5.2-NVFP4 + MODEL_PREFIX: glm5.2 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '1' + DURATION: '3600' + RESULT_FILENAME: glm52_fp4_dynamo-trt_1p3d-tep8_c1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-c30-b2-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-c30-b2-mtp5.yaml new file mode 100644 index 0000000000..cffce82663 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-c30-b2-mtp5.yaml @@ -0,0 +1,204 @@ +name: dynamo-disagg-b300-1p4d-tep4-c30-b2-mtp5 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 + +identity: + model: + repo: "nvidia/GLM-5.2-NVFP4" + revision: "aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa" + container: + image: "nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1" + frameworks: + dynamo: "1.4.0.dev20260807" + tensorrt_llm: "1.3.0rc22.post1" + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + request_plane: tcp + +slurm: + time_limit: "4:00:00" + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: b300 + gpus_per_node: 8 + spread_workers: true + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 4 + decode_workers: 4 + gpus_per_decode: 4 + +backend: + type: trtllm + prefill_environment: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + UCX_MAX_RNDV_RAILS: "2" + UCX_MAX_HCA_PER_GPU: inf + UCX_LOG_LEVEL: info + TRTLLM_CTX_LOCAL_HCA_PIN: "1" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "2.61" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + decode_environment: *server_environment + trtllm_config: + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 274877906944 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: EQUAL_PROGRESS + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 5 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1, 2] + enable_padding: true + enable_attention_dp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + host_cache_size: 0 + free_gpu_memory_fraction: 0.8 + tokens_per_block: 64 + max_batch_size: 2 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 4 + publish_events_and_metrics: false + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + orchestrator_placement: first_decode + +benchmark: + type: custom + client_placement: last_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: none + MODEL: nvidia/GLM-5.2-NVFP4 + MODEL_PREFIX: glm5.2 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '30' + DURATION: '3600' + RESULT_FILENAME: glm52_fp4_dynamo-trt_1p4d-tep4_c30 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-c60-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-c60-b5-mtp5.yaml new file mode 100644 index 0000000000..70ed9ff108 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-c60-b5-mtp5.yaml @@ -0,0 +1,204 @@ +name: dynamo-disagg-b300-3p4d-tep4-c60-b5-mtp5 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 + +identity: + model: + repo: "nvidia/GLM-5.2-NVFP4" + revision: "aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa" + container: + image: "nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1" + frameworks: + dynamo: "1.4.0.dev20260807" + tensorrt_llm: "1.3.0rc22.post1" + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + request_plane: tcp + +slurm: + time_limit: "4:00:00" + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: b300 + gpus_per_node: 8 + spread_workers: true + prefill_nodes: 3 + prefill_workers: 3 + gpus_per_prefill: 4 + decode_nodes: 4 + decode_workers: 4 + gpus_per_decode: 4 + +backend: + type: trtllm + prefill_environment: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + UCX_MAX_RNDV_RAILS: "2" + UCX_MAX_HCA_PER_GPU: inf + UCX_LOG_LEVEL: info + TRTLLM_CTX_LOCAL_HCA_PIN: "1" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "2.61" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + decode_environment: *server_environment + trtllm_config: + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 197568495616 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: EQUAL_PROGRESS + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 5 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1, 2, 4, 5] + enable_padding: true + enable_attention_dp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + host_cache_size: 0 + free_gpu_memory_fraction: 0.8 + tokens_per_block: 64 + max_batch_size: 5 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 4 + publish_events_and_metrics: false + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + orchestrator_placement: first_decode + +benchmark: + type: custom + client_placement: last_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: none + MODEL: nvidia/GLM-5.2-NVFP4 + MODEL_PREFIX: glm5.2 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '60' + DURATION: '3600' + RESULT_FILENAME: glm52_fp4_dynamo-trt_3p4d-tep4_c60 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-c227-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-c227-b16-mtp3.yaml new file mode 100644 index 0000000000..2922fd5d37 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-c227-b16-mtp3.yaml @@ -0,0 +1,205 @@ +name: dynamo-disagg-b300-6p1d-dep8-c227-b16-mtp3 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 + +identity: + model: + repo: "nvidia/GLM-5.2-NVFP4" + revision: "aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa" + container: + image: "nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1" + frameworks: + dynamo: "1.4.0.dev20260807" + tensorrt_llm: "1.3.0rc22.post1" + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + request_plane: tcp + +slurm: + time_limit: "4:00:00" + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: b300 + gpus_per_node: 8 + spread_workers: true + prefill_nodes: 6 + prefill_workers: 6 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + +backend: + type: trtllm + prefill_environment: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + UCX_MAX_RNDV_RAILS: "2" + UCX_MAX_HCA_PER_GPU: inf + UCX_LOG_LEVEL: info + TRTLLM_CTX_LOCAL_HCA_PIN: "1" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "1.99" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + decode_environment: *server_environment + trtllm_config: + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 197568495616 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: EQUAL_PROGRESS + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1, 2, 4, 8, 16] + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + host_cache_size: 0 + free_gpu_memory_fraction: 0.9 + tokens_per_block: 64 + max_batch_size: 16 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 8 + publish_events_and_metrics: false + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + orchestrator_placement: first_decode + +benchmark: + type: custom + client_placement: last_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: none + MODEL: nvidia/GLM-5.2-NVFP4 + MODEL_PREFIX: glm5.2 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '227' + DURATION: '3600' + RESULT_FILENAME: glm52_fp4_dynamo-trt_6p1d-dep8_c227 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-c233-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-c233-b16-mtp3.yaml new file mode 100644 index 0000000000..1f9f2d9498 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-c233-b16-mtp3.yaml @@ -0,0 +1,204 @@ +name: dynamo-disagg-b300-8p2d-dep8-c233-b16-mtp3 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 + +identity: + model: + repo: "nvidia/GLM-5.2-NVFP4" + revision: "aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa" + container: + image: "nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1" + frameworks: + dynamo: "1.4.0.dev20260807" + tensorrt_llm: "1.3.0rc22.post1" + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + request_plane: tcp + +slurm: + time_limit: "4:00:00" + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: b300 + gpus_per_node: 8 + spread_workers: true + prefill_nodes: 8 + prefill_workers: 8 + gpus_per_prefill: 4 + decode_nodes: 2 + decode_workers: 2 + gpus_per_decode: 8 + +backend: + type: trtllm + prefill_environment: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + UCX_MAX_RNDV_RAILS: "2" + UCX_MAX_HCA_PER_GPU: inf + UCX_LOG_LEVEL: info + TRTLLM_CTX_LOCAL_HCA_PIN: "1" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "1.99" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + decode_environment: *server_environment + trtllm_config: + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 197568495616 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1, 2, 4, 8, 16] + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + host_cache_size: 0 + free_gpu_memory_fraction: 0.9 + tokens_per_block: 64 + max_batch_size: 16 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 8 + publish_events_and_metrics: false + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + orchestrator_placement: first_decode + +benchmark: + type: custom + client_placement: last_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: none + MODEL: nvidia/GLM-5.2-NVFP4 + MODEL_PREFIX: glm5.2 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '233' + DURATION: '3600' + RESULT_FILENAME: glm52_fp4_dynamo-trt_8p2d-dep8_c233 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 50ed9b4713..e541bee50b 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8471,6 +8471,113 @@ glm5.2-fp4-b300-sglang-agentic-mtp: search-space: - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16] } +# GLM-5.2 NVFP4 TensorRT-LLM AgentX on B300 with Dynamo disaggregated serving. +# Seven topology variants use NIXL KV transfer and MTP3/MTP5/MTP7 decoding. +glm5.2-fp4-b300-dynamo-trt-agentic-mtp: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + model: nvidia/GLM-5.2-NVFP4 + model-prefix: glm5.2 + runner: cluster:b300-nv + precision: fp4 + framework: dynamo-trt + router: { name: dynamo-router, version: "1.4.0.dev20260807" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + - spec-decoding: mtp + conc-list: [1] + kv-offloading: none + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p3d-tep8-c1-b1-mtp7.yaml" + decode: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: false + - spec-decoding: mtp + conc-list: [30] + kv-offloading: none + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-c30-b2-mtp5.yaml" + decode: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: mtp + conc-list: [20] + kv-offloading: none + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-c20-b5-mtp5.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: false + - spec-decoding: mtp + conc-list: [60] + kv-offloading: none + prefill: + num-worker: 3 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-c60-b5-mtp5.yaml" + decode: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: mtp + conc-list: [233] + kv-offloading: none + prefill: + num-worker: 8 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-c233-b16-mtp3.yaml" + decode: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + - spec-decoding: mtp + conc-list: [227] + kv-offloading: none + prefill: + num-worker: 6 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-c227-b16-mtp3.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + # First GLM-5.2 AgentX recipe on B200, shipped spec-decode-only per the AgentX # policy that agentic arms enable speculative decoding rather than running a # separate STP baseline (MODELS.md). SGLang EAGLE off GLM-5.2's built-in nextn diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7500c62923..a2584461c7 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6234,3 +6234,11 @@ description: - "Add GB200 Kimi K3 AgentX config." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2639 + +- config-keys: + - glm5.2-fp4-b300-dynamo-trt-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add GLM-5.2 NVFP4 B300 AgentX recipes for Dynamo and TensorRT-LLM disaggregated serving with NIXL KV transfer and MTP speculative decoding." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2666 diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index f7878a8ad8..a237d37495 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -44,6 +44,10 @@ elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo- elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-sglang" ]]; then export MODEL_PATH="${MODEL_PATH:-/scratch/models/DeepSeek-V4-Pro}" export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" +elif [[ $MODEL_PREFIX == "glm5.2" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-trt" ]]; then + export SERVED_MODEL_NAME="nvidia/GLM-5.2-NVFP4" + export MODEL_PATH="/scratch/models/GLM-5.2-NVFP4" + export SRT_SLURM_MODEL_PREFIX="nvidia/GLM-5.2-NVFP4" elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-vllm" ]]; then export MODEL_PATH="/data/models/MiniMax-M2.5-NVFP4" export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-nvfp4" @@ -57,7 +61,7 @@ elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" && $FRAMEWORK == "dy export MODEL_PATH="/data/models/MiniMax-M3-MXFP8" export SRT_SLURM_MODEL_PREFIX="MiniMaxAI/MiniMax-M3-MXFP8" else - echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4 with dynamo-vllm or dynamo-sglang, minimaxm2.5-fp4 with dynamo-vllm, minimaxm2.5-fp8 with dynamo-vllm, minimaxm3-fp4 with dynamo-vllm, minimaxm3-fp8 with dynamo-vllm" + echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4 with dynamo-vllm or dynamo-sglang, glm5.2-fp4 with dynamo-trt, minimaxm2.5-fp4 with dynamo-vllm, minimaxm2.5-fp8 with dynamo-vllm, minimaxm3-fp4 with dynamo-vllm, minimaxm3-fp8 with dynamo-vllm" exit 1 fi @@ -70,7 +74,20 @@ if [ -d "$SRT_REPO_DIR" ]; then fi # TODO(CJQ): make first class upon srt-slurm upstream refactor -if [[ "$IS_AGENTIC" == "1" ]]; then +if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "glm5.2" ]]; then + git clone --branch v1.0.36 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" || exit 1 + # This cluster requires root remapping to remain disabled for PMIx. + sed -i 's/CONTAINER_REMAP_ROOT_EXPORT = {"ENROOT_REMAP_ROOT": "yes"}/CONTAINER_REMAP_ROOT_EXPORT = {"ENROOT_REMAP_ROOT": "no"}/' src/srtctl/core/slurm.py + TRTLLM_RECIPES_DIR="benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2" + mkdir -p "$TRTLLM_RECIPES_DIR" + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2" \ + "$TRTLLM_RECIPES_DIR" + if [[ "${EVAL_ONLY:-false}" == "true" ]]; then + find "$TRTLLM_RECIPES_DIR" -name "*.yaml" \ + -exec sed -i '/TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS/d' {} + + fi +elif [[ "$IS_AGENTIC" == "1" ]]; then git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then @@ -143,6 +160,23 @@ export ISL="$ISL" export OSL="$OSL" export EVAL_ONLY="${EVAL_ONLY:-false}" +DEFAULT_MOUNTS_BLOCK='default_mounts:' +if [[ $FRAMEWORK != "dynamo-trt" || $MODEL_PREFIX != "glm5.2" ]]; then + DEFAULT_MOUNTS_BLOCK+=' + "/opt/ucx-no-ud": "/usr/local/ucx"' +else + echo "Using the container UCX for glm5.2 dynamo-trt NIXL transfers" +fi +if [[ "$IS_AGENTIC" == "1" ]]; then + AIPERF_MMAP_CACHE_HOST_PATH="/data/home/sa-shared/gharunners/ai-perf-cache" + HF_HUB_CACHE_HOST_PATH="/data/home/sa-shared/gharunners/hf-hub-cache" + mkdir -p "$AIPERF_MMAP_CACHE_HOST_PATH" "$HF_HUB_CACHE_HOST_PATH" + chmod 777 "$AIPERF_MMAP_CACHE_HOST_PATH" "$HF_HUB_CACHE_HOST_PATH" 2>/dev/null || true + DEFAULT_MOUNTS_BLOCK+=" + \"${AIPERF_MMAP_CACHE_HOST_PATH}\": \"/aiperf_mmap_cache\" + \"${HF_HUB_CACHE_HOST_PATH}\": \"/hf_hub_cache\"" +fi + # Create srtslurm.yaml for srtctl SRTCTL_ROOT="${GITHUB_WORKSPACE}/${SRT_REPO_DIR}" echo "Creating srtslurm.yaml configuration..." @@ -169,8 +203,7 @@ containers: "${IMAGE}": "${SQUASH_FILE}" nginx-sqsh: "${NGINX_SQUASH_FILE}" use_exclusive_sbatch_directive: true -default_mounts: - "/opt/ucx-no-ud": "/usr/local/ucx" +${DEFAULT_MOUNTS_BLOCK} EOF echo "Generated srtslurm.yaml:" @@ -198,6 +231,16 @@ if [[ ! -f "$CONFIG_PATH" ]]; then exit 1 fi +if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "glm5.2" ]]; then + sed -i "s|^ RESULT_FILENAME:.*| RESULT_FILENAME: ${RESULT_FILENAME}|" "$CONFIG_PATH" +fi + +# Eval runs execute lm-eval on the allocation head and connect over loopback. +# Throughput keeps the frontend on first_decode and the client on last_decode. +if [[ "${EVAL_ONLY:-false}" == "true" && "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "glm5.2" ]]; then + sed -i 's/^ orchestrator_placement: first_decode$/ orchestrator_placement: head/' "$CONFIG_PATH" +fi + # Override the job name in the recipe with the runner name. sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH" if [[ "$MODEL_PREFIX" == "minimaxm3" && -n "$MINIMAX_M3_SLURM_EXCLUDED_NODELIST" ]]; then @@ -218,6 +261,9 @@ fi if [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" && "$MODEL_PATH" == /scratch/models/* ]]; then SRTCTL_APPLY_ARGS+=(--no-preflight) fi +if [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "glm5.2" && "$MODEL_PATH" == /scratch/models/* ]]; then + SRTCTL_APPLY_ARGS+=(--no-preflight) +fi if [[ -n "$SRTCTL_SETUP_SCRIPT" ]]; then SRTCTL_APPLY_ARGS+=(--setup-script "$SRTCTL_SETUP_SCRIPT") fi @@ -353,6 +399,19 @@ if [[ "${RUN_EVAL:-false}" == "true" || "${EVAL_ONLY:-false}" == "true" ]]; then else echo "WARNING: RUN_EVAL=true but no eval results found at $EVAL_DIR" fi + + # srt-slurm stages eval artifacts but does not write the metadata file + # consumed by score validation. Reuse the canonical metadata writer so + # topology and recipe identity stay aligned with the workflow inputs. + eval_conc_value="${EVAL_CONC:-${CONC:-1}}" + ( + export IS_MULTINODE=true + # shellcheck source=benchmarks/benchmark_lib.sh + source "$GITHUB_WORKSPACE/benchmarks/benchmark_lib.sh" + _write_lm_eval_meta_json \ + "$GITHUB_WORKSPACE/meta_env.json" "" "$eval_conc_value" + ) + echo "Wrote meta_env.json (conc=${eval_conc_value}, prefix=${MODEL_PREFIX:-unknown})" fi # Clean up srt-slurm outputs to prevent NFS silly-rename lock files