[Power] Require GLM-5.2 and add DSV4 H200 telemetry / 强制 GLM-5.2 并新增 DSV4 H200 功耗采集

.github/workflows/e2e-tests.yml

@@ -523,6 +523,8 @@ jobs:
             duration: ${{ inputs.agentx-fast && '1200' || (inputs.duration-override != '' && inputs.duration-override || matrix.config.duration) }}
             agentx-fast: ${{ inputs.agentx-fast }}
             run-eval: false
+            require-power: ${{ inputs.require-power }}
+            power-producer-sha: ${{ inputs.power-producer-sha }}
             scenario-type: agentic-coding
             ref: ${{ inputs.ref }}
 

.github/workflows/test-process-result.yml

@@ -8,10 +8,14 @@ on:
       - '.github/workflows/e2e-tests.yml'
       - '.github/workflows/test-process-result.yml'
       - 'benchmarks/benchmark_lib.sh'
+      - 'benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-2p2d-pcp8-tp8-dp8-mtp.yaml'
+      - 'benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-h200-tp8-mtp-kvoffload.yaml'
       - 'benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/1p1d-tp4-tp4.yaml'
       - 'benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/1p1d-tp4-tp4.yaml'
       - 'runners/launch_gb200-nv.sh'
       - 'runners/launch_gb300-nv.sh'
+      - 'runners/launch_h200-dgxc-slurm.sh'
+      - 'runners/inject_srt_power_concurrencies.py'
       - 'utils/aggregate_power.py'
       - 'utils/aggregate_power_multinode.py'
       - 'utils/agentic/aggregation/power_adapter.py'
@@ -26,6 +30,8 @@ on:
       - 'utils/test_aggregate_power_multinode.py'
       - 'utils/test_gb200_power_official_contract.py'
       - 'utils/test_gb300_power_official_contract.py'
+      - 'utils/test_h200_power_official_contract.py'
+      - 'utils/test_inject_srt_power_concurrencies.py'
       - 'utils/test_process_result.py'
 
 permissions:
@@ -54,4 +60,4 @@ jobs:
       - name: Run pytest
         run: |
           cd utils
-          python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/test_power_adapter.py agentic/aggregation/test_power_lifecycle.py agentic/aggregation/test_process_agentic_result.py test_gb200_power_official_contract.py test_gb300_power_official_contract.py test_process_result.py -v
+          python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/test_power_adapter.py agentic/aggregation/test_power_lifecycle.py agentic/aggregation/test_process_agentic_result.py test_gb200_power_official_contract.py test_gb300_power_official_contract.py test_h200_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py -v

benchmarks/benchmark_lib.sh

@@ -2190,11 +2190,16 @@ run_agentic_replay_and_write_outputs() (
     local validation_rc
     local power_rc=0
     local agentx_power_enabled=0
+    local agentx_multinode_power_enabled=0
     local agentx_monitor_stopped=1
 
     case "${ENABLE_AGENTX_POWER:-1}" in
         1|true|TRUE|yes|YES)
-            if [ "${IS_MULTINODE:-false}" != "true" ]; then
+            if [ "${IS_MULTINODE:-false}" = "true" ]; then
+                if [ -n "${SRT_MEASUREMENT_WINDOW_DIR:-}" ]; then
+                    agentx_multinode_power_enabled=1
+                fi
+            else
                 agentx_power_enabled=1
             fi
             ;;
@@ -2207,11 +2212,42 @@ run_agentic_replay_and_write_outputs() (
         fi
     }
 
-    if [ "$agentx_power_enabled" = "1" ]; then
+    _write_agentx_multinode_window() {
+        local state="$1"
+        local -a power_args
+        power_args=(
+            --result-dir "$result_dir"
+            --concurrency "${CONC:?CONC must be set for multinode AgentX power}"
+            --write-multinode-window "$state"
+        )
+        case "${REQUIRE_POWER:-0}" in
+            1|true|TRUE|yes|YES) power_args+=(--require-power) ;;
+        esac
+        (
+            cd "$INFMAX_CONTAINER_WORKSPACE"
+            "$AIPERF_PYTHON" -m utils.agentic.aggregation.power_adapter "${power_args[@]}"
+        )
+    }
+
+    if [ "$agentx_power_enabled" = "1" ] || [ "$agentx_multinode_power_enabled" = "1" ]; then
         # AIPerf currently exports naive local datetimes while SMI emits the
         # same host wall clock. Capture the launch-time offset so the adapter
         # can attach it explicitly before normalizing the profiling window.
         date +%z > "$result_dir/agentic_power_timezone_offset.txt"
+    fi
+
+    if [ "$agentx_multinode_power_enabled" = "1" ]; then
+        set +e
+        _write_agentx_multinode_window running
+        power_rc=$?
+        set -e
+        if [ "$power_rc" -ne 0 ]; then
+            echo "ERROR: failed to publish the AgentX formal running power window" >&2
+            return "$power_rc"
+        fi
+    fi
+
+    if [ "$agentx_power_enabled" = "1" ]; then
         start_gpu_monitor --output "$result_dir/gpu_metrics.csv"
         agentx_monitor_stopped=0
         # This function runs in a subshell, so these handlers cannot replace
@@ -2238,6 +2274,13 @@ run_agentic_replay_and_write_outputs() (
 
     write_agentic_result_json "$result_dir"
 
+    if [ "$agentx_multinode_power_enabled" = "1" ] && [ "$replay_rc" -eq 0 ]; then
+        set +e
+        _write_agentx_multinode_window completed
+        power_rc=$?
+        set -e
+    fi
+
     if [ "$agentx_power_enabled" = "1" ]; then
         local expected_num_gpus
         local -a power_args

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-h200-tp8-mtp-kvoffload.yaml

@@ -101,6 +101,19 @@ srun_options:
   # The custom benchmark installs its isolated AIPerf environment at runtime.
   container-remap-root: ""
 
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: false
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 10
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 9401
+
 benchmark:
   type: custom
   command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
@@ -109,7 +122,7 @@ benchmark:
     RESULT_DIR: /logs/agentic
     PORT: "8000"
     # Aggregated serving uses one TP8 worker for both prefill and decode.
-    IS_MULTINODE: "false"
+    IS_MULTINODE: "true"
     TP: "8"
     AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true"
     AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"

benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-2p2d-pcp8-tp8-dp8-mtp.yaml

@@ -137,6 +137,19 @@ srun_options:
   mem: "0"
   container-remap-root: ""
 
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: true
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 10
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 9401
+
 benchmark:
   type: custom
   command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh

runners/inject_srt_power_concurrencies.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""Inject exact matrix concurrencies into a runtime srt-slurm recipe copy."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import tempfile
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+def _validate_concurrencies(concurrencies: list[Any]) -> list[int]:
+    if (
+        not concurrencies
+        or any(isinstance(value, bool) or not isinstance(value, int) for value in concurrencies)
+        or any(value <= 0 for value in concurrencies)
+        or len(set(concurrencies)) != len(concurrencies)
+    ):
+        raise ValueError("concurrencies must be positive unique integers")
+    return concurrencies
+
+
+def inject_concurrencies(recipe_path: Path, concurrencies: list[Any]) -> None:
+    """Atomically set benchmark.concurrencies on a disposable recipe copy."""
+    values = _validate_concurrencies(concurrencies)
+    try:
+        recipe = yaml.safe_load(recipe_path.read_text(encoding="utf-8"))
+    except (OSError, yaml.YAMLError) as exc:
+        raise ValueError(f"failed to load recipe: {exc}") from exc
+    if not isinstance(recipe, dict) or not isinstance(recipe.get("benchmark"), dict):
+        raise ValueError("recipe must contain a benchmark mapping")
+
+    recipe["benchmark"]["concurrencies"] = values
+    fd, temporary_name = tempfile.mkstemp(
+        dir=recipe_path.parent,
+        prefix=f".{recipe_path.name}.",
+        text=True,
+    )
+    temporary_path = Path(temporary_name)
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as handle:
+            yaml.safe_dump(recipe, handle, sort_keys=False)
+            handle.flush()
+            os.fsync(handle.fileno())
+        os.replace(temporary_path, recipe_path)
+    except BaseException:
+        temporary_path.unlink(missing_ok=True)
+        raise
+
+
+def _positive_integer(raw: str) -> int:
+    try:
+        value = int(raw)
+    except ValueError as exc:
+        raise argparse.ArgumentTypeError("must be a positive integer") from exc
+    if value <= 0 or str(value) != raw:
+        raise argparse.ArgumentTypeError("must be a canonical positive integer")
+    return value
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("recipe", type=Path)
+    parser.add_argument("concurrencies", nargs="+", type=_positive_integer)
+    args = parser.parse_args()
+    try:
+        inject_concurrencies(args.recipe, args.concurrencies)
+    except ValueError as exc:
+        parser.error(str(exc))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())