From b77027b042e44d28f64e73d05a0d43f8e06f0a1c Mon Sep 17 00:00:00 2001 From: Charles Wu Date: Fri, 31 Jul 2026 10:56:32 +0800 Subject: [PATCH 1/8] feat(agentic): add GLM-5.2 MXFP4 MI355X sglang-disagg recipe and master config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce 1P1D TP8 disaggregated agentic bring-up for amd/GLM-5.2-MXFP4 on MI355X (models.yaml AgentX entry, launcher recipe, amd-master hicache sweep). Scope ionic MSN-safe MoRI IO QP tuning to GLM-5.2-MXFP4 with DISAGG=true in env.sh; keep HiCache on ratio for GLM-5.2 DSA pools. Pin sglang-rocm v0.5.16-rocm720-mi35x-20260729; improve SLURM_REUSE nodelist resolution. 中文：新增 GLM-5.2 MXFP4 MI355X 分离式 SGLang agentic 启动脚本与 amd-master 配置（1P1D TP8、HiCache）；在 env.sh 中仅对 GLM-5.2-MXFP4 且 DISAGG 开启时应用 MoRI IO ionic 调参；HiCache 对 GLM-5.2 使用 ratio sizing；镜像 v0.5.16；并改进 SLURM 复用作业的节点列表解析。 Co-authored-by: Cursor --- .../glm5.2_fp4_mi355x_sglang-disagg.sh | 155 ++++++++++++++++++ benchmarks/multi_node/amd_utils/env.sh | 15 +- benchmarks/multi_node/amd_utils/models.yaml | 27 +++ .../multi_node/amd_utils/server_sglang.sh | 6 +- benchmarks/multi_node/amd_utils/submit.sh | 22 ++- configs/amd-master.yaml | 40 +++++ 6 files changed, 260 insertions(+), 5 deletions(-) create mode 100755 benchmarks/multi_node/agentic/glm5.2_fp4_mi355x_sglang-disagg.sh diff --git a/benchmarks/multi_node/agentic/glm5.2_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/agentic/glm5.2_fp4_mi355x_sglang-disagg.sh new file mode 100755 index 0000000000..bd9452221c --- /dev/null +++ b/benchmarks/multi_node/agentic/glm5.2_fp4_mi355x_sglang-disagg.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash + +# Agentic trace-replay recipe for a disaggregated SGLang server on MI355X +# (GLM-5.2 MXFP4, 1P1D TP8, HiCache DRAM KV offload). +# +# CI-style sibling of dsv4_fp4_mi355x_sglang-disagg.sh: driven entirely by +# environment variables and submits a SLURM job via submit.sh. HiCache defaults: +# ratio 1.5 + page_first_direct + write_through_selective (L2; avoids Mori RDMA race). + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/../../benchmark_lib.sh" + +check_env_vars \ + CONC_LIST \ + ISL \ + OSL \ + IMAGE \ + SPEC_DECODING \ + MODEL_PATH \ + PREFILL_NUM_WORKERS \ + PREFILL_TP \ + PREFILL_EP \ + PREFILL_DP_ATTN \ + DECODE_NUM_WORKERS \ + DECODE_TP \ + DECODE_EP \ + DECODE_DP_ATTN \ + PREFILL_NODES \ + DECODE_NODES \ + RANDOM_RANGE_RATIO \ + DURATION \ + KV_OFFLOADING \ + IS_AGENTIC \ + FRAMEWORK + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +set -x + +cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 + +export TIME_LIMIT="${TIME_LIMIT:-08:00:00}" +export MODEL_PATH=$MODEL_PATH +export MODEL_NAME=$MODEL_NAME +export CONTAINER_IMAGE=$IMAGE +export CLIENT_IMAGE="${CLIENT_IMAGE:-$IMAGE}" + +export MODEL_PREFIX="${MODEL_PREFIX:-glm5.2}" +export PRECISION="${PRECISION:-fp4}" +export RESULT_FILENAME="${RESULT_FILENAME:-${RUNNER_NAME:-glm5.2-fp4-agentic}}" + +export DURATION="${DURATION:-1800}" +export MAX_MODEL_LEN="${MAX_MODEL_LEN:-1000000}" + +# GLM-5.2 is glm_moe_dsa (pure DSA/MLA); skip the hybrid-state mori_conn patch. +export MORI_CONN_PATCH="${MORI_CONN_PATCH:-skip}" + +export DISABLE_CUSTOM_ALL_REDUCE="${DISABLE_CUSTOM_ALL_REDUCE:-0}" + +export KV_OFFLOADING="${KV_OFFLOADING:-none}" +if [[ "$KV_OFFLOADING" != "none" ]]; then + export KV_OFFLOAD_BACKEND="${KV_OFFLOAD_BACKEND:-hicache}" +fi +if [[ "$KV_OFFLOADING" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" ]]; then + export HICACHE_TIER="${HICACHE_TIER:-L2}" + export HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-1}" + export HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-1}" + export HICACHE_RATIO="${HICACHE_RATIO:-1.5}" + export HICACHE_PREFETCH_POLICY="${HICACHE_PREFETCH_POLICY:-best_effort}" + + if [[ "${HICACHE_TIER^^}" == "L3" ]]; then + export HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first}" + export HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" + export HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" + export HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-mooncake}" + else + # write_through_selective evicts only under GPU memory pressure, avoiding the + # mori RDMA race that write_back / write_through can trigger under disagg load. + export HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" + export HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" + export HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through_selective}" + export HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-}" + fi + export MC_MASTER_PORT="${MC_MASTER_PORT:-58137}" + export MC_METADATA_PORT="${MC_METADATA_PORT:-8080}" + export MC_METRICS_PORT="${MC_METRICS_PORT:-19003}" + export MC_MASTER_THREADS="${MC_MASTER_THREADS:-64}" + export MC_EVICTION_HIGH_WATERMARK="${MC_EVICTION_HIGH_WATERMARK:-0.95}" + export MC_PATCH_HOSTPOOL="${MC_PATCH_HOSTPOOL:-1}" + export MC_PROTOCOL="${MC_PROTOCOL:-tcp}" + export MC_GLOBAL_SEG="${MC_GLOBAL_SEG:-64gb}" + export MC_DEVICE="${MC_DEVICE:-}" + export MC_MASTER_ADDR="${MC_MASTER_ADDR:-}" + export MC_METADATA_SERVER="${MC_METADATA_SERVER:-}" +fi + +# MoRI IO QP tuning for this recipe: amd_utils/env.sh when MODEL_NAME=GLM-5.2-MXFP4 +# and DISAGG=true (ionic MSN-safe overrides; other models/topologies unchanged). + +export PREFILL_ROUTER_POLICY="${PREFILL_ROUTER_POLICY:-cache_aware}" +export ENABLE_METRICS="${ENABLE_METRICS:-1}" + +export DECODE_MTP_SIZE="${DECODE_MTP_SIZE:-0}" + +# AIPerf reuses keep-alive sockets across agentic turns; outlast the default +# 5s SGLang keep-alive (same mitigation as glm5.2_fp4_b300_sglang.sh). +export SGLANG_TIMEOUT_KEEP_ALIVE="${SGLANG_TIMEOUT_KEEP_ALIVE:-900}" +export AIPERF_HTTP_TCP_USER_TIMEOUT="${AIPERF_HTTP_TCP_USER_TIMEOUT:-900000}" + +if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then +export PREFILL_ENABLE_EP=false +else +export PREFILL_ENABLE_EP=true +fi + +if [[ "$PREFILL_DP_ATTN" == "true" ]]; then +export PREFILL_ENABLE_DP=true +else +export PREFILL_ENABLE_DP=false +fi + +if [[ "${DECODE_EP:-1}" -eq 1 ]]; then +export DECODE_ENABLE_EP=false +else +export DECODE_ENABLE_EP=true +fi + +if [[ "$DECODE_DP_ATTN" == "true" ]]; then +export DECODE_ENABLE_DP=true +else +export DECODE_ENABLE_DP=false +fi + +SUBMIT_ARGS=( + "$PREFILL_NODES" "$PREFILL_NUM_WORKERS" "$DECODE_NODES" "$DECODE_NUM_WORKERS" + "$ISL" "$OSL" "${CONC_LIST// /x}" inf + "${PREFILL_ENABLE_EP}" "${PREFILL_ENABLE_DP}" + "${DECODE_ENABLE_EP}" "${DECODE_ENABLE_DP}" + "${PREFILL_TP}" "${DECODE_TP}" + "${RANDOM_RANGE_RATIO}" +) +if [[ -n "${NODE_LIST:-}" ]]; then + SUBMIT_ARGS+=("$NODE_LIST") +fi + +JOB_ID=$(bash ./submit.sh "${SUBMIT_ARGS[@]}") + +if [[ $? -ne 0 ]]; then + echo "Failed to submit job" >&2 + exit 1 +fi + +echo "$JOB_ID" diff --git a/benchmarks/multi_node/amd_utils/env.sh b/benchmarks/multi_node/amd_utils/env.sh index 0cf3978bbf..fb70b904c1 100755 --- a/benchmarks/multi_node/amd_utils/env.sh +++ b/benchmarks/multi_node/amd_utils/env.sh @@ -295,6 +295,19 @@ else export NCCL_BLOCKING_WAIT="${NCCL_BLOCKING_WAIT:-1}" # export NCCL_DEBUG="${NCCL_DEBUG:-INFO}" + # ========================================================================= + # GLM-5.2-MXFP4 disagg (benchmarks/multi_node/agentic/glm5.2_fp4_mi355x_sglang-disagg.sh) + # Ionic MSN-safe MoRI IO QP tuning; overrides global MoRI defaults only when + # this model runs in PD/disagg mode (DISAGG=true). Other models/topologies unchanged. + # ========================================================================= + if [[ "$MODEL_NAME" == "GLM-5.2-MXFP4" && "${DISAGG:-false}" == "true" ]]; then + export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-500000}" + export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-8192}" + export MORI_IO_QP_MAX_CQE="${MORI_IO_QP_MAX_CQE:-16384}" + export MORI_IO_QP_MAX_SGE="${MORI_IO_QP_MAX_SGE:-2}" + export MORI_IO_TC_DISABLE="${MORI_IO_TC_DISABLE:-0}" + fi + # ========================================================================= # DeepSeek-V4-Pro PD recipe overrides # Placed at the end of the SGLang env block so it wins over the global @@ -374,4 +387,4 @@ else export GPU_MAX_HW_QUEUES=5 fi -fi \ No newline at end of file +fi diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index 2231e6fe60..935cd2c46f 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -264,6 +264,33 @@ GLM-5-FP8: chunked_prefill_size: 262144 cuda_graph_bs_range: "1-128" +GLM-5.2-MXFP4-AgentX: + base_flags: "--watchdog-timeout 1200 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsa --dsa-prefill-backend tilelang --dsa-decode-backend tilelang --dsa-topk-backend sgl-kernel --disaggregation-transfer-backend mori --tool-call-parser glm47 --reasoning-parser glm45 --trust-remote-code --served-model-name amd/GLM-5.2-MXFP4 --log-level info --log-level-http error --model-loader-extra-config '{\\\"enable_multithread_load\\\": true, \\\"num_threads\\\": 8}'" + mtp_flags: "" + dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" + ep_flags: "--moe-a2a-backend mori" + prefill: + mem_fraction_static: 0.8 + disable_radix_cache: false + dp: + max_running_requests: 1024 + chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" + context_length: 1048576 + no_dp: + max_running_requests: 128 + chunked_prefill_size: 131072 + context_length: 1048576 + decode: + mem_fraction_static: 0.8 + prefill_round_robin_balance: true + disagg_decode_enable_radix_cache: false + dp: + max_running_requests: 1024 + cuda_graph_bs_range: "1-128" + no_dp: + max_running_requests: 128 + cuda_graph_bs_range: "1-128" + DeepSeek-R1-0528-MXFP4-Preview: base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index aaaca61ef5..3f2241b438 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -545,14 +545,14 @@ if [[ "$KV_OFFLOADING" != "none" && "$KV_OFFLOAD_BACKEND" == "hicache" ]]; then # ever sending a request. HICACHE_RATIO="${HICACHE_RATIO:-5}" HICACHE_SIZING_FLAGS="--hicache-ratio ${HICACHE_RATIO}" - # DeepSeek V4's hybrid HiCache pool rejects --hicache-size (requires - # --hicache-ratio), so the absolute per-node budget cannot be applied to it. + # DeepSeek V4 / GLM-5.2 hybrid or DSA HiCache pools reject --hicache-size + # (require --hicache-ratio). GB-based sizing also OOMs GLM-5.2 on MI355X. # See sglang _deepseek_v4_num_host_pages() (raises ValueError when # server_args.hicache_size > 0): # https://github.com/sgl-project/sglang/blob/9dd57ef8c48e2cd82292d849f01e2130c5203e67/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py#L262-L266 # FORCE_HICACHE_RATIO additionally lets a recipe opt into ratio-based sizing # for any other model without unsetting TOTAL_CPU_DRAM_GB (see comment above). - if [[ "${FORCE_HICACHE_RATIO:-0}" != "1" && -n "${TOTAL_CPU_DRAM_GB:-}" && "${TOTAL_CPU_DRAM_GB}" -gt 0 && "${MODEL_NAME}" != *DeepSeek-V4* ]]; then + if [[ "${FORCE_HICACHE_RATIO:-0}" != "1" && -n "${TOTAL_CPU_DRAM_GB:-}" && "${TOTAL_CPU_DRAM_GB}" -gt 0 && "${MODEL_NAME}" != *DeepSeek-V4* && "${MODEL_NAME}" != *GLM-5.2* ]]; then # TOTAL_CPU_DRAM_GB is the prefill worker's per-node budget (only prefill # offloads KV to CPU DRAM today); --hicache-size is per rank per host # pool. A prefill server may span nodes (PREFILL_TP_SIZE is its total diff --git a/benchmarks/multi_node/amd_utils/submit.sh b/benchmarks/multi_node/amd_utils/submit.sh index 6a1598d81d..e22fa722a1 100755 --- a/benchmarks/multi_node/amd_utils/submit.sh +++ b/benchmarks/multi_node/amd_utils/submit.sh @@ -151,6 +151,7 @@ export DRY_RUN="${DRY_RUN:-0}" # Eval-related env vars (threaded from workflow → runner → here → job.slurm → Docker) export RUN_EVAL="${RUN_EVAL:-false}" export EVAL_ONLY="${EVAL_ONLY:-false}" +export ROUTER_READINESS_CANARY="${ROUTER_READINESS_CANARY:-1}" export EVAL_CONC="${EVAL_CONC:-}" export FRAMEWORK="${FRAMEWORK:-}" export PRECISION="${PRECISION:-}" @@ -204,11 +205,30 @@ if [[ -n "${SLURM_REUSE_JOBID:-}" ]]; then # Resolve allocation's nodelist if not already provided. ALLOC_NODELIST="${SLURM_JOB_NODELIST:-$(squeue -h -j "$REUSE_JID" -o '%N' 2>/dev/null)}" + if [[ -z "$ALLOC_NODELIST" ]]; then + ALLOC_NODELIST="$(scontrol show job "$REUSE_JID" 2>/dev/null | awk -F= '/NodeList=/{print $2}' | awk '{print $1}')" + fi + # Expand Slurm bracket form: host-[01,57] -> host-01,host-57 + if [[ "$ALLOC_NODELIST" == *"["* ]]; then + EXPANDED="$(scontrol show hostnames "$ALLOC_NODELIST" 2>/dev/null | paste -sd, -)" + if [[ -n "$EXPANDED" ]]; then + ALLOC_NODELIST="$EXPANDED" + elif [[ -n "${NODE_LIST:-}" ]]; then + ALLOC_NODELIST="$NODE_LIST" + fi + fi if [[ -z "$ALLOC_NODELIST" ]]; then echo "Error: could not resolve nodelist for job ${REUSE_JID}" >&2 exit 1 fi - ALLOC_NNODES=$(scontrol show hostnames "$ALLOC_NODELIST" | wc -l) + if [[ "$ALLOC_NODELIST" == *","* ]]; then + ALLOC_NNODES=$(tr ',' '\n' <<< "$ALLOC_NODELIST" | grep -c .) + else + ALLOC_NNODES=$(scontrol show hostnames "$ALLOC_NODELIST" 2>/dev/null | wc -l) + if [[ "$ALLOC_NNODES" -lt 1 ]]; then + ALLOC_NNODES=1 + fi + fi if [[ "$ALLOC_NNODES" -lt "$NUM_NODES" ]]; then echo "Error: allocation ${REUSE_JID} has ${ALLOC_NNODES} nodes, need ${NUM_NODES}" >&2 exit 1 diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index cf5cdfeb6d..9b9afb1af4 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1568,6 +1568,46 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp: - "DECODE_NODES=1" - "DECODE_MTP_SIZE=3" +# GLM-5.2 MXFP4 MI355X SGLang disaggregated agentic (1P1D TP8, HiCache DRAM KV +# offload). Mirrors dsv4-fp4-mi355x-sglang-disagg-agentic-hicache; HiCache sizing +# uses ratio 0.75 (GLM-5.2 DSA/MLA-family replicated KV pool — see +# glm5.2_fp4_mi355x_sglang-disagg.sh). Weights: amd/GLM-5.2-MXFP4 on +# /it-share/hf_cache. +glm5.2-fp4-mi355x-sglang-disagg-agentic-hicache: + image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260729 + model: amd/GLM-5.2-MXFP4 + model-prefix: glm5.2 + runner: cluster:mi355x-amds + precision: fp4 + framework: sglang-disagg + kv-p2p-transfer: mori + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: "none" + conc-list: [ 2, 4, 8, 16, 32 ] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260729" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=0" + # DeepSeek-V4-Pro FP8 single-node on MI325X (gfx942) via vLLM. # EXTRAPOLATED bring-up. Same rationale as dsv4-fp8-mi300x-vllm: sglang has no # gfx942 build of the dsv4 nvfp4 MoE / TileLang-MLA kernels, so vLLM runs the From 81481ef1ed8b03d600f1cba8b50bdd5812a898d5 Mon Sep 17 00:00:00 2001 From: Charles Wu Date: Mon, 3 Aug 2026 15:03:48 +0800 Subject: [PATCH 2/8] feat(config): pin GLM-5.2 disagg HiCache L2 tunables in amd-master MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document validated 1P1D L2 settings (ratio 1.5, selective write-through, page_first_direct) in glm5.2-fp4-mi355x-sglang-disagg-agentic-hicache prefill additional-settings for sweep/CI parity with cluster bring-up. 中文：在 amd-master 的 GLM-5.2 分离式 agentic HiCache 配置中写入经 1P1D 验证的 L2 参数（ratio 1.5、write_through_selective、page_first_direct 等）， 便于扫描与 CI 和集群 bring-up 一致。 Co-authored-by: Cursor --- configs/amd-master.yaml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 9b9afb1af4..acbeeb837d 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1569,10 +1569,10 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp: - "DECODE_MTP_SIZE=3" # GLM-5.2 MXFP4 MI355X SGLang disaggregated agentic (1P1D TP8, HiCache DRAM KV -# offload). Mirrors dsv4-fp4-mi355x-sglang-disagg-agentic-hicache; HiCache sizing -# uses ratio 0.75 (GLM-5.2 DSA/MLA-family replicated KV pool — see -# glm5.2_fp4_mi355x_sglang-disagg.sh). Weights: amd/GLM-5.2-MXFP4 on -# /it-share/hf_cache. +# offload). Mirrors dsv4-fp4-mi355x-sglang-disagg-agentic-hicache; L2 HiCache +# tuning (ratio 1.5, page_first_direct, write_through_selective) validated on +# amd-aim 1P1D — see glm5.2_fp4_mi355x_sglang-disagg.sh. Weights: +# amd/GLM-5.2-MXFP4 on /it-share/hf_cache. glm5.2-fp4-mi355x-sglang-disagg-agentic-hicache: image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260729 model: amd/GLM-5.2-MXFP4 @@ -1599,6 +1599,11 @@ glm5.2-fp4-mi355x-sglang-disagg-agentic-hicache: additional-settings: - "PREFILL_NODES=1" - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260729" + - "HICACHE_RATIO=1.5" + - "HICACHE_WRITE_POLICY=write_through_selective" + - "HICACHE_MEM_LAYOUT=page_first_direct" + - "HICACHE_IO_BACKEND=direct" + - "HICACHE_PAGE_SIZE=1" decode: num-worker: 1 tp: 8 From d37d9cc97043a58774c9fa32dcdaae3f9d57be9c Mon Sep 17 00:00:00 2001 From: Charles Wu Date: Mon, 3 Aug 2026 15:49:56 +0800 Subject: [PATCH 3/8] fix(amd): default GLM-5.2 model paths and forward RDMA env via hicache config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Set MI355X amd-aim MODEL_PATH/MODEL_DIR to /it-share/hf_cache in the GLM sglang-disagg recipe so job.slurm does not fall back to /it-share/data. Write IBDEVICES and MORI_RDMA_TC into hicache_mc.env for sglang containers. 中文：在 GLM 分离式 recipe 中默认使用 /it-share/hf_cache，避免 job.slurm 误用 /it-share/data；并通过 hicache_mc.env 向容器传递 IBDEVICES 与 MORI_RDMA_TC。 Co-authored-by: Cursor --- .../agentic/glm5.2_fp4_mi355x_sglang-disagg.sh | 12 +++++++++--- benchmarks/multi_node/amd_utils/job.slurm | 2 ++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/benchmarks/multi_node/agentic/glm5.2_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/agentic/glm5.2_fp4_mi355x_sglang-disagg.sh index bd9452221c..4ee63e338b 100755 --- a/benchmarks/multi_node/agentic/glm5.2_fp4_mi355x_sglang-disagg.sh +++ b/benchmarks/multi_node/agentic/glm5.2_fp4_mi355x_sglang-disagg.sh @@ -10,13 +10,18 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../../benchmark_lib.sh" +# MI355X amd-aim bring-up: GLM-5.2 MXFP4 weights under /it-share/hf_cache (symlink +# GLM-5.2-MXFP4). Override MODEL_PATH/MODEL_DIR for other cluster mounts. +export MODEL_PATH="${MODEL_PATH:-/it-share/hf_cache}" +export MODEL_DIR="${MODEL_DIR:-$MODEL_PATH}" +export MODEL_NAME="${MODEL_NAME:-GLM-5.2-MXFP4}" + check_env_vars \ CONC_LIST \ ISL \ OSL \ IMAGE \ SPEC_DECODING \ - MODEL_PATH \ PREFILL_NUM_WORKERS \ PREFILL_TP \ PREFILL_EP \ @@ -42,8 +47,9 @@ set -x cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 export TIME_LIMIT="${TIME_LIMIT:-08:00:00}" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME +export MODEL_PATH="${MODEL_PATH}" +export MODEL_DIR="${MODEL_DIR:-$MODEL_PATH}" +export MODEL_NAME="${MODEL_NAME}" export CONTAINER_IMAGE=$IMAGE export CLIENT_IMAGE="${CLIENT_IMAGE:-$IMAGE}" diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index 3c8a6f16a8..35a0622337 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -557,6 +557,8 @@ MC_GLOBAL_SEG=${MC_GLOBAL_SEG:-} MC_DEVICE=${MC_DEVICE:-} MC_MASTER_ADDR=${MC_MASTER_ADDR:-} MC_METADATA_SERVER=${MC_METADATA_SERVER:-} +IBDEVICES=${IBDEVICES:-} +MORI_RDMA_TC=${MORI_RDMA_TC:-} EOF echo "[config] wrote HiCache/Mooncake settings -> $HICACHE_MC_CONFIG" From c63c1563f2904069fdf59d10241d9c5916a4a251 Mon Sep 17 00:00:00 2001 From: Charles Wu Date: Mon, 3 Aug 2026 18:54:08 +0800 Subject: [PATCH 4/8] fix(amd): force GLM-5.2 model root over CI launcher /it-share/data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use GLM52_MODEL_ROOT (default /it-share/hf_cache) for MODEL_PATH so launch_mi355x-amds.sh cannot override with /it-share/data via ${VAR:-}. 中文：GLM recipe 通过 GLM52_MODEL_ROOT 固定权重根目录（默认 /it-share/hf_cache）， 避免 CI launcher 已设置的 MODEL_PATH=/it-share/data 被 ${MODEL_PATH:-} 保留而导致找错路径。 Co-authored-by: Cursor --- .../agentic/glm5.2_fp4_mi355x_sglang-disagg.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/benchmarks/multi_node/agentic/glm5.2_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/agentic/glm5.2_fp4_mi355x_sglang-disagg.sh index 4ee63e338b..05660d76da 100755 --- a/benchmarks/multi_node/agentic/glm5.2_fp4_mi355x_sglang-disagg.sh +++ b/benchmarks/multi_node/agentic/glm5.2_fp4_mi355x_sglang-disagg.sh @@ -10,11 +10,12 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../../benchmark_lib.sh" -# MI355X amd-aim bring-up: GLM-5.2 MXFP4 weights under /it-share/hf_cache (symlink -# GLM-5.2-MXFP4). Override MODEL_PATH/MODEL_DIR for other cluster mounts. -export MODEL_PATH="${MODEL_PATH:-/it-share/hf_cache}" -export MODEL_DIR="${MODEL_DIR:-$MODEL_PATH}" +# MI355X amd-aim: GLM-5.2 MXFP4 lives under /it-share/hf_cache (symlink GLM-5.2-MXFP4). +# This file is GLM-only — do not inherit launch_mi355x-amds.sh MODEL_PATH=/it-share/data +# (DSv4/R1 staging). Override with GLM52_MODEL_ROOT if needed. export MODEL_NAME="${MODEL_NAME:-GLM-5.2-MXFP4}" +export MODEL_PATH="${GLM52_MODEL_ROOT:-/it-share/hf_cache}" +export MODEL_DIR="${MODEL_DIR:-$MODEL_PATH}" check_env_vars \ CONC_LIST \ From 57d17fffc0ffbb1c9beff6e623fb11539609faf4 Mon Sep 17 00:00:00 2001 From: Charles Wu Date: Tue, 4 Aug 2026 11:21:20 +0800 Subject: [PATCH 5/8] fix(amd): default GLM-5.2 model root to /it-share/data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Align GLM disagg recipe and amd-master comments with launch_mi355x-amds.sh and job.slurm mia1* MODEL_DIR; remove hf_cache / GLM52_MODEL_ROOT override. 中文：GLM 分离式 recipe 默认使用 /it-share/data，与 CI launcher 及 job.slurm 一致； 不再强制 /it-share/hf_cache。 Co-authored-by: Cursor --- .../multi_node/agentic/glm5.2_fp4_mi355x_sglang-disagg.sh | 8 ++++---- configs/amd-master.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/benchmarks/multi_node/agentic/glm5.2_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/agentic/glm5.2_fp4_mi355x_sglang-disagg.sh index 05660d76da..87179f3323 100755 --- a/benchmarks/multi_node/agentic/glm5.2_fp4_mi355x_sglang-disagg.sh +++ b/benchmarks/multi_node/agentic/glm5.2_fp4_mi355x_sglang-disagg.sh @@ -10,11 +10,11 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../../benchmark_lib.sh" -# MI355X amd-aim: GLM-5.2 MXFP4 lives under /it-share/hf_cache (symlink GLM-5.2-MXFP4). -# This file is GLM-only — do not inherit launch_mi355x-amds.sh MODEL_PATH=/it-share/data -# (DSv4/R1 staging). Override with GLM52_MODEL_ROOT if needed. +# MI355X amd-aim: default model root matches launch_mi355x-amds.sh and job.slurm +# (MODEL_DIR=/it-share/data on mia1*). Override MODEL_PATH/MODEL_DIR when weights +# live elsewhere (e.g. /it-share/hf_cache). export MODEL_NAME="${MODEL_NAME:-GLM-5.2-MXFP4}" -export MODEL_PATH="${GLM52_MODEL_ROOT:-/it-share/hf_cache}" +export MODEL_PATH="${MODEL_PATH:-/it-share/data}" export MODEL_DIR="${MODEL_DIR:-$MODEL_PATH}" check_env_vars \ diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index acbeeb837d..cabf58f314 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1572,7 +1572,7 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp: # offload). Mirrors dsv4-fp4-mi355x-sglang-disagg-agentic-hicache; L2 HiCache # tuning (ratio 1.5, page_first_direct, write_through_selective) validated on # amd-aim 1P1D — see glm5.2_fp4_mi355x_sglang-disagg.sh. Weights: -# amd/GLM-5.2-MXFP4 on /it-share/hf_cache. +# amd/GLM-5.2-MXFP4 under /it-share/data (default MODEL_DIR on mia1*). glm5.2-fp4-mi355x-sglang-disagg-agentic-hicache: image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260729 model: amd/GLM-5.2-MXFP4 From 236c1cf518333133c1ebbea456f57694e6e9f479 Mon Sep 17 00:00:00 2001 From: Charles Wu Date: Thu, 20 Aug 2026 00:50:26 +0800 Subject: [PATCH 6/8] feat(amd): GLM-5.2 AgentX disagg MTP bring-up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enable built-in MTP/EAGLE on the MI355X GLM-5.2 disagg HiCache agentic path: amd-master spec-decoding mtp with DECODE_MTP_SIZE=2, EAGLE mtp_flags and decode mem_fraction_static 0.85 in models.yaml, and agentic script MTP block with simulated acceptance for non-eval throughput runs. 中文：为 MI355X GLM-5.2 分离式 HiCache AgentX 配置启用内置 MTP/EAGLE 投机解码：在 amd-master 中改为 mtp 与 DECODE_MTP_SIZE=2，在 models.yaml 中补充 EAGLE mtp_flags 并将 decode 静态显存比例调至 0.85，在 agentic 脚本中增加 MTP 逻辑（非 eval 吞吐跑使用模拟接受长度）。 Co-authored-by: Cursor --- .../agentic/glm5.2_fp4_mi355x_sglang-disagg.sh | 18 ++++++++++++++++-- benchmarks/multi_node/amd_utils/models.yaml | 8 ++++++-- configs/amd-master.yaml | 10 +++++----- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/benchmarks/multi_node/agentic/glm5.2_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/agentic/glm5.2_fp4_mi355x_sglang-disagg.sh index 87179f3323..684a8dce7b 100755 --- a/benchmarks/multi_node/agentic/glm5.2_fp4_mi355x_sglang-disagg.sh +++ b/benchmarks/multi_node/agentic/glm5.2_fp4_mi355x_sglang-disagg.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Agentic trace-replay recipe for a disaggregated SGLang server on MI355X -# (GLM-5.2 MXFP4, 1P1D TP8, HiCache DRAM KV offload). +# (GLM-5.2 MXFP4, 1P1D TP8, HiCache DRAM KV offload, built-in MTP/EAGLE). # # CI-style sibling of dsv4_fp4_mi355x_sglang-disagg.sh: driven entirely by # environment variables and submits a SLURM job via submit.sh. HiCache defaults: @@ -109,7 +109,21 @@ fi export PREFILL_ROUTER_POLICY="${PREFILL_ROUTER_POLICY:-cache_aware}" export ENABLE_METRICS="${ENABLE_METRICS:-1}" -export DECODE_MTP_SIZE="${DECODE_MTP_SIZE:-0}" +if [[ "${SPEC_DECODING:-none}" == "mtp" || "${DECODE_MTP_SIZE:-0}" -gt 0 ]]; then + # Do not use ${DECODE_MTP_SIZE:-2}: wrappers may pre-set 0 and block :- defaulting. + if [[ "${DECODE_MTP_SIZE:-0}" -le 0 ]]; then + export DECODE_MTP_SIZE=2 + fi + # Throughput: synthetic acceptance from golden_al_distribution/glm5.2_mtp.yaml + # (thinking_on, num_speculative_tokens=2 -> AL 2.50). EVAL_ONLY runs use real MTP. + if [[ "${EVAL_ONLY:-false}" != "true" ]]; then + export SGLANG_SIMULATE_ACC_LEN="${SGLANG_SIMULATE_ACC_LEN:-2.50}" + export SGLANG_SIMULATE_ACC_METHOD="${SGLANG_SIMULATE_ACC_METHOD:-match-expected}" + export SGLANG_SIMULATE_ACC_TOKEN_MODE="${SGLANG_SIMULATE_ACC_TOKEN_MODE:-real-draft-token}" + fi +else + export DECODE_MTP_SIZE="${DECODE_MTP_SIZE:-0}" +fi # AIPerf reuses keep-alive sockets across agentic turns; outlast the default # 5s SGLang keep-alive (same mitigation as glm5.2_fp4_b300_sglang.sh). diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index 935cd2c46f..3bea8777f0 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -266,7 +266,11 @@ GLM-5-FP8: GLM-5.2-MXFP4-AgentX: base_flags: "--watchdog-timeout 1200 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsa --dsa-prefill-backend tilelang --dsa-decode-backend tilelang --dsa-topk-backend sgl-kernel --disaggregation-transfer-backend mori --tool-call-parser glm47 --reasoning-parser glm45 --trust-remote-code --served-model-name amd/GLM-5.2-MXFP4 --log-level info --log-level-http error --model-loader-extra-config '{\\\"enable_multithread_load\\\": true, \\\"num_threads\\\": 8}'" - mtp_flags: "" + # GLM-5.2 built-in MTP via the EAGLE spec path (eagle-topk 1), matching + # srt-slurm-recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml and DeepSeek-V4 DI. + # build_server_config appends --speculative-num-steps/--speculative-num-draft-tokens + # from DECODE_MTP_SIZE when spec-decoding: mtp. + mtp_flags: "--speculative-algorithm EAGLE --speculative-eagle-topk 1" dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" ep_flags: "--moe-a2a-backend mori" prefill: @@ -281,7 +285,7 @@ GLM-5.2-MXFP4-AgentX: chunked_prefill_size: 131072 context_length: 1048576 decode: - mem_fraction_static: 0.8 + mem_fraction_static: 0.85 prefill_round_robin_balance: true disagg_decode_enable_radix_cache: false dp: diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index cabf58f314..935048e40e 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1569,9 +1569,9 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp: - "DECODE_MTP_SIZE=3" # GLM-5.2 MXFP4 MI355X SGLang disaggregated agentic (1P1D TP8, HiCache DRAM KV -# offload). Mirrors dsv4-fp4-mi355x-sglang-disagg-agentic-hicache; L2 HiCache -# tuning (ratio 1.5, page_first_direct, write_through_selective) validated on -# amd-aim 1P1D — see glm5.2_fp4_mi355x_sglang-disagg.sh. Weights: +# offload, built-in MTP). Mirrors dsv4-fp4-mi355x-sglang-disagg-agentic-hicache; +# L2 HiCache tuning (ratio 1.5, page_first_direct, write_through_selective) +# validated on amd-aim 1P1D — see glm5.2_fp4_mi355x_sglang-disagg.sh. Weights: # amd/GLM-5.2-MXFP4 under /it-share/data (default MODEL_DIR on mia1*). glm5.2-fp4-mi355x-sglang-disagg-agentic-hicache: image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260729 @@ -1587,7 +1587,7 @@ glm5.2-fp4-mi355x-sglang-disagg-agentic-hicache: agentic-coding: - dram-utilization: 0.80 search-space: - - spec-decoding: "none" + - spec-decoding: "mtp" conc-list: [ 2, 4, 8, 16, 32 ] kv-offloading: dram kv-offload-backend: { name: hicache } @@ -1611,7 +1611,7 @@ glm5.2-fp4-mi355x-sglang-disagg-agentic-hicache: dp-attn: false additional-settings: - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" + - "DECODE_MTP_SIZE=2" # DeepSeek-V4-Pro FP8 single-node on MI325X (gfx942) via vLLM. # EXTRAPOLATED bring-up. Same rationale as dsv4-fp8-mi300x-vllm: sglang has no From 7782ddfa10cc8519f0e7fc79000a0703f7a62abd Mon Sep 17 00:00:00 2001 From: Charles Wu Date: Thu, 20 Aug 2026 00:50:26 +0800 Subject: [PATCH 7/8] chore(changelog): append GLM-5.2 MI355X sglang-disagg HiCache MTP entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：在 perf-changelog.yaml 末尾追加 glm5.2-fp4-mi355x-sglang-disagg-agentic-hicache 条目。 Co-authored-by: Cursor --- perf-changelog.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 52c74bf168..84c2c6a30d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6195,3 +6195,13 @@ - "Use AITER INT4 quick-reduce, ptpc_fp8 online quantization excluding embeddings, lm_head, gates, and experts, an FP8 KV cache, and three-token MTP with golden synthetic acceptance length 2.99 for benchmark runs." - "Set max-num-seqs to twice the concurrency, select CUDA graph capture sizes by concurrency, and cap max-num-batched-tokens at 16384." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2576 + +- config-keys: + - glm5.2-fp4-mi355x-sglang-disagg-agentic-hicache + scenario-type: + - agentic-coding + description: + - "Add GLM-5.2 MXFP4 MI355X 1P1D TP8 disaggregated SGLang AgentX with DRAM HiCache KV offload and built-in MTP (spec-decoding: mtp, DECODE_MTP_SIZE=2, EAGLE topk 1)." + - "HiCache L2: ratio 1.5, write_through_selective, page_first_direct, direct IO. Throughput uses synthetic AL 2.50 from golden_al_distribution/glm5.2_mtp.yaml; EVAL_ONLY uses real draft-model MTP." + - "Image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260729; model amd/GLM-5.2-MXFP4; conc-list [2, 4, 8, 16, 32]." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/TBD From 69b967908d337bf4709894a596087e1573ad7e41 Mon Sep 17 00:00:00 2001 From: Charles Wu Date: Thu, 20 Aug 2026 00:51:39 +0800 Subject: [PATCH 8/8] chore(changelog): fill glm5.2-fp4-mi355x-sglang-disagg-agentic-hicache pr-link MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：将 perf-changelog.yaml 中该条目的 pr-link 填为 #2679。 Co-authored-by: Cursor --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 84c2c6a30d..27ad392b61 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6204,4 +6204,4 @@ - "Add GLM-5.2 MXFP4 MI355X 1P1D TP8 disaggregated SGLang AgentX with DRAM HiCache KV offload and built-in MTP (spec-decoding: mtp, DECODE_MTP_SIZE=2, EAGLE topk 1)." - "HiCache L2: ratio 1.5, write_through_selective, page_first_direct, direct IO. Throughput uses synthetic AL 2.50 from golden_al_distribution/glm5.2_mtp.yaml; EVAL_ONLY uses real draft-model MTP." - "Image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260729; model amd/GLM-5.2-MXFP4; conc-list [2, 4, 8, 16, 32]." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/TBD + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2679