[Quality] Balanced Coding + Agentic Coding | GLM-5.2 | VNG Cloud MaaS HCM | CCU4

.github/workflows/benchmark-tmpl.yml

@@ -182,6 +182,39 @@ on:
         required: false
         type: string
         default: ""
+      # Quality-eval inputs. Only consumed by quality-eval scenario types
+      # (runner: cluster:quality-eval); every other recipe leaves these
+      # empty and is unaffected.
+      benchmark-name:
+        description: "Quality benchmark to run (gpqa, mmlu_pro, hle, livecodebench, bfcl, scicode, swebench_pro, deepswe)"
+        required: false
+        type: string
+        default: ""
+      quality-endpoint:
+        description: "OpenAI-compatible endpoint URL for quality evals"
+        required: false
+        type: string
+        default: ""
+      quality-model-name:
+        description: "Model name for quality evals (e.g. openai/z-ai/glm-5.2)"
+        required: false
+        type: string
+        default: ""
+      smoke:
+        description: "Smoke-test mode: skip threshold validation, only verify artifact exists"
+        required: false
+        type: boolean
+        default: false
+      num-concurrent:
+        description: "API request concurrency for quality evals (passed as NUM_CONCURRENT env var)"
+        required: false
+        type: string
+        default: ""
+      max-gen-tokens:
+        description: "Per-request output-token ceiling; empty uses the benchmark-specific default"
+        required: false
+        type: string
+        default: ""
 env:
   RANDOM_RANGE_RATIO: 0.8
   HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }}
@@ -208,7 +241,7 @@ env:
   EVAL_ONLY: ${{ inputs.eval-only }}
   # Agentic-coding env. Fixed-seq-len jobs leave these empty.
   SCENARIO_TYPE: ${{ inputs.scenario-type }}
-  SCENARIO_SUBDIR: ${{ inputs.scenario-type == 'agentic-coding' && 'agentic/' || 'fixed_seq_len/' }}
+  SCENARIO_SUBDIR: ${{ (inputs.scenario-type == 'agentic-coding' && 'agentic/') || (startsWith(inputs.scenario-type, 'quality-') && 'quality/') || 'fixed_seq_len/' }}
   IS_AGENTIC: ${{ inputs.scenario-type == 'agentic-coding' && '1' || '0' }}
   KV_OFFLOADING: ${{ inputs.kv-offloading }}
   KV_OFFLOAD_BACKEND: ${{ inputs.kv-offload-backend }}
@@ -228,6 +261,14 @@ env:
   REMOTE_RESET_URL: ${{ inputs.remote-reset-url }}
   REMOTE_RUNNER_TYPE: ${{ inputs.remote-runner-type }}
   REMOTE_MAX_CONTEXT_LENGTH: ${{ inputs.remote-max-context-length }}
+  # Quality-eval env. Only consumed by quality-eval scenario types.
+  QUALITY_BENCHMARK_NAME: ${{ inputs.benchmark-name }}
+  QUALITY_ENDPOINT: ${{ inputs.quality-endpoint }}
+  QUALITY_API_KEY: ${{ secrets.GREENNODE_API_KEY }}
+  QUALITY_MODEL_NAME: ${{ inputs.quality-model-name }}
+  SMOKE: ${{ inputs.smoke }}
+  NUM_CONCURRENT: ${{ inputs.num-concurrent }}
+  MAX_GEN_TOKENS: ${{ inputs.max-gen-tokens }}
   AIPERF_FAILED_REQUEST_THRESHOLD: '0.10'
   RESULT_DIR: /workspace/results
   PYTHONDONTWRITEBYTECODE: '1'
@@ -268,6 +309,7 @@ jobs:
     timeout-minutes: 500
     name: >-
       p${{ inputs.priority }} | ${{ inputs.model-prefix }} ${{ inputs.precision }} ${{ inputs.runner }} ${{ inputs.framework == 'sglang' && 'sgl' || inputs.framework == 'dynamo-sglang' && 'dyn-sgl' || inputs.framework == 'sglang-disagg' && 'sgl-disagg' || inputs.framework }}
+      ${{ inputs.benchmark-name != '' && inputs.benchmark-name || '' }}
       TP${{ inputs.tp }}${{ inputs.pp != '' && inputs.pp != '1' && format('/PP{0}', inputs.pp) || '' }}${{ inputs.dcp-size != '' && inputs.dcp-size != '1' && format('/DCP{0}', inputs.dcp-size) || '' }}${{ inputs.pcp-size != '' && inputs.pcp-size != '1' && format('/PCP{0}', inputs.pcp-size) || '' }}${{ inputs.ep != '' && inputs.ep != '1' && format('/EP{0}', inputs.ep) || '' }}${{ inputs.dp-attn && '/DPA' || '' }}
       ${{ inputs.spec-decoding != 'none' && inputs.spec-decoding || '' }}
       ${{ inputs.kv-offloading != '' && inputs.kv-offloading != 'none' && format('{0} KV offload', inputs.kv-offloading) || '' }}
@@ -321,7 +363,13 @@ jobs:
           # Export RESULT_FILENAME early so it's available for artifact uploads even if cancelled
           echo "RESULT_FILENAME=${RESULT_FILENAME}" >> $GITHUB_ENV
 
-          bash ./runners/launch_${RUNNER_NAME%%_*}.sh
+          # Quality-eval scenarios run on any runner with the quality-eval label;
+          # dispatch directly to launch_quality-eval.sh regardless of runner name.
+          if [ "${SCENARIO_SUBDIR}" = "quality/" ]; then
+            bash ./runners/launch_quality-eval.sh
+          else
+            bash ./runners/launch_${RUNNER_NAME%%_*}.sh
+          fi
 
           if [ "${{ inputs.eval-only }}" = "true" ]; then
             echo "Eval-only mode: skipping benchmark result file check"
@@ -436,12 +484,24 @@ jobs:
             agent_preds.json
             predictions.jsonl
             swebench_report_*.json
+            eval_results*.json
             *.traj*
+            *.csv
+            lcb_results*.json
+            *.jsonl
+            bfcl_inference_audit.json
+            bfcl_result_*.json
+            scicode_debug.tar.gz
           if-no-files-found: ${{ inputs.eval-only && 'error' || 'ignore' }}
 
       - name: Verify eval scores
         if: ${{ (success() || failure()) && inputs.eval-only }}
-        run: python3 utils/evals/validate_scores.py
+        run: |
+          SMOKE_FLAG=""
+          if [ "${{ inputs.smoke }}" = "true" ]; then
+            SMOKE_FLAG="--smoke"
+          fi
+          python3 utils/evals/validate_scores.py $SMOKE_FLAG
 
       - name: Cleanup eval outputs (post-upload)
         if: ${{ always() && (env.RUN_EVAL == 'true' || inputs.eval-only) }}
@@ -450,7 +510,9 @@ jobs:
           # Remove any eval results JSONs that were moved into workspace
           rm -f results*.json || true
           rm -f sample*.jsonl || true
-          rm -f agent_preds.json predictions.jsonl swebench_report_*.json *.traj* || true
+          rm -f agent_preds.json predictions.jsonl swebench_report_*.json eval_results*.json *.traj* || true
+          rm -f *.csv lcb_results*.json || true
+          rm -f bfcl_inference_audit.json bfcl_result_*.json scicode_debug.tar.gz || true
 
       - name: Resource cleanup (post-run)
         if: always()

.github/workflows/e2e-tests.yml

@@ -178,6 +178,7 @@ jobs:
             agentic-eval-config: ${{ steps.get-jobs.outputs.agentic-eval-config }}
             multi-node-agentic-config: ${{ steps.get-jobs.outputs.multi-node-agentic-config }}
             multi-node-agentic-eval-config: ${{ steps.get-jobs.outputs.multi-node-agentic-eval-config }}
+            quality-eval-config: ${{ steps.get-jobs.outputs.quality-eval-config }}
         steps:
             - name: Checkout code (ref)
               if: ${{ inputs.ref && inputs.ref != '' }}
@@ -259,20 +260,67 @@ jobs:
                   AGENTIC_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x and x.get('run-eval', False)]))" | score_matrix agentic-eval)
                   MULTI_AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x and not x.get('run-eval', False)]))" | score_matrix multi-agentic)
                   MULTI_AGENTIC_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x and x.get('run-eval', False)]))" | score_matrix multi-agentic-eval)
-                  SINGLE=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix single)
-                  MULTI=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix multi)
+                  QUALITY=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type', '').startswith('quality-')]))" | score_matrix quality-eval)
+                  SINGLE=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and not x.get('scenario-type', '').startswith('quality-') and not x.get('eval-only', False)]))" | score_matrix single)
+                  MULTI=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and not x.get('scenario-type', '').startswith('quality-') and not x.get('eval-only', False)]))" | score_matrix multi)
                   MULTI_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))" | score_matrix multi-eval)
                   {
                     echo "agentic-config=$AGENTIC"
                     echo "agentic-eval-config=$AGENTIC_EVAL"
                     echo "multi-node-agentic-config=$MULTI_AGENTIC"
                     echo "multi-node-agentic-eval-config=$MULTI_AGENTIC_EVAL"
+                    echo "quality-eval-config=$QUALITY"
                     echo "single-node-config=$SINGLE"
                     echo "multi-node-config=$MULTI"
                     echo "eval-config=$EVALS"
                     echo "multi-node-eval-config=$MULTI_EVAL"
                   } >> "$GITHUB_OUTPUT"
 
+    test-sweep-quality-evals:
+        needs: get-jobs
+        if: ${{ needs.get-jobs.outputs.quality-eval-config != '[]' }}
+        uses: ./.github/workflows/benchmark-tmpl.yml
+        name: quality eval /
+        strategy:
+            fail-fast: ${{ inputs.fail-fast }}
+            matrix:
+                config: ${{ fromJson(needs.get-jobs.outputs.quality-eval-config) }}
+        secrets: inherit
+        with:
+            exp-name: ${{ matrix.config.exp-name }}
+            runner: ${{ matrix.config.runner }}
+            priority: ${{ matrix.config.priority }}
+            queue-token: ${{ matrix.config['queue-token'] }}
+            skip-queue-pr: ${{ matrix.config['skip-queue-pr'] || '' }}
+            image: ${{ matrix.config.image }}
+            model: ${{ matrix.config.model }}
+            model-prefix: ${{ matrix.config.model-prefix }}
+            framework: ${{ matrix.config.framework }}
+            precision: ${{ matrix.config.precision }}
+            tp: '1'
+            pp: '1'
+            dcp-size: '1'
+            pcp-size: '1'
+            ep: '1'
+            dp-attn: false
+            conc: '1'
+            spec-decoding: 'none'
+            disagg: 'false'
+            isl: '0'
+            osl: '0'
+            max-model-len: '0'
+            run-eval: true
+            eval-only: true
+            scenario-type: ${{ matrix.config.scenario-type }}
+            benchmark-name: ${{ matrix.config.benchmark-name }}
+            quality-endpoint: ${{ matrix.config.quality-endpoint }}
+            quality-model-name: ${{ matrix.config.quality-model-name }}
+            smoke: ${{ matrix.config.smoke || false }}
+            num-concurrent: ${{ matrix.config.num-concurrent || '' }}
+            max-gen-tokens: ${{ matrix.config.max-gen-tokens || '' }}
+            eval-limit: ${{ matrix.config.eval-limit || inputs.eval-limit }}
+            ref: ${{ inputs.ref }}
+
     test-sweep-multi-node:
         needs: get-jobs
         if: ${{ needs.get-jobs.outputs.multi-node-config != '[]' }}
@@ -665,8 +713,8 @@ jobs:
             result-prefix: "bmk"
 
     collect-evals:
-        needs: [test-sweep-evals, test-sweep-multi-node-evals, test-sweep-agentic-evals, test-sweep-multi-node-agentic-evals]
-        if: ${{ always() && (needs.test-sweep-evals.result != 'skipped' || needs.test-sweep-multi-node-evals.result != 'skipped' || needs.test-sweep-agentic-evals.result != 'skipped' || needs.test-sweep-multi-node-agentic-evals.result != 'skipped') }}
+        needs: [test-sweep-evals, test-sweep-multi-node-evals, test-sweep-agentic-evals, test-sweep-multi-node-agentic-evals, test-sweep-quality-evals]
+        if: ${{ always() && (needs.test-sweep-evals.result != 'skipped' || needs.test-sweep-multi-node-evals.result != 'skipped' || needs.test-sweep-agentic-evals.result != 'skipped' || needs.test-sweep-multi-node-agentic-evals.result != 'skipped' || needs.test-sweep-quality-evals.result != 'skipped') }}
         uses: ./.github/workflows/collect-evals.yml
         secrets: inherit
 

.github/workflows/run-sweep.yml

@@ -925,6 +925,61 @@ jobs:
             eval-conc: ${{ matrix.config['eval-conc'] }}
             scenario-type: agentic-coding
 
+    # Quality-eval rows carry the quality-eval input shape (benchmark-name,
+    # quality-endpoint, quality-model-name, smoke), so they are dispatched
+    # with their own job rather than sweep-evals' fixed-seq-len inputs.
+    sweep-quality-evals:
+        needs: [setup, canary-select, canary-sweep]
+        if: >-
+            ${{
+              !cancelled() &&
+              needs.setup.result == 'success' &&
+              needs.setup.outputs.reuse-enabled != 'true' &&
+              (needs.canary-sweep.result == 'success' || needs.canary-sweep.result == 'skipped') &&
+              toJson(fromJson(needs.setup.outputs.search-space-config).quality_evals) != '[]' &&
+              toJson(fromJson(needs.setup.outputs.search-space-config).quality_evals) != 'null'
+            }}
+        uses: ./.github/workflows/benchmark-tmpl.yml
+        name: quality eval /
+        strategy:
+            fail-fast: ${{ contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast') || contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast-no-canary') }}
+            matrix:
+                config: ${{ fromJson(needs.setup.outputs.search-space-config).quality_evals }}
+        secrets: inherit
+        with:
+            exp-name: ${{ matrix.config.exp-name }}
+            runner: ${{ matrix.config.runner }}
+            priority: ${{ matrix.config.priority }}
+            queue-token: ${{ matrix.config['queue-token'] }}
+            skip-queue-pr: ${{ matrix.config['skip-queue-pr'] || '' }}
+            image: ${{ matrix.config.image }}
+            model: ${{ matrix.config.model }}
+            model-prefix: ${{ matrix.config.model-prefix }}
+            framework: ${{ matrix.config.framework }}
+            precision: ${{ matrix.config.precision }}
+            tp: '1'
+            pp: '1'
+            dcp-size: '1'
+            pcp-size: '1'
+            ep: '1'
+            dp-attn: false
+            conc: '1'
+            spec-decoding: 'none'
+            disagg: 'false'
+            isl: '0'
+            osl: '0'
+            max-model-len: '0'
+            run-eval: true
+            eval-only: true
+            scenario-type: ${{ matrix.config.scenario-type }}
+            benchmark-name: ${{ matrix.config.benchmark-name }}
+            quality-endpoint: ${{ matrix.config.quality-endpoint }}
+            quality-model-name: ${{ matrix.config.quality-model-name }}
+            smoke: ${{ matrix.config.smoke || false }}
+            num-concurrent: ${{ matrix.config.num-concurrent || '' }}
+            max-gen-tokens: ${{ matrix.config.max-gen-tokens || '' }}
+            eval-limit: ${{ matrix.config.eval-limit || (contains(github.event.pull_request.labels.*.name, 'sweep-enabled') && '10' || '') }}
+
     collect-results:
         needs:
             [
@@ -955,8 +1010,8 @@ jobs:
             result-prefix: "bmk"
 
     collect-evals:
-        needs: [sweep-evals, sweep-agentic-evals, sweep-multi-node-evals, sweep-multi-node-agentic-evals, setup]
-        if: ${{ always() && needs.setup.result != 'skipped' && (needs.sweep-evals.result != 'skipped' || needs.sweep-agentic-evals.result != 'skipped' || needs.sweep-multi-node-evals.result != 'skipped' || needs.sweep-multi-node-agentic-evals.result != 'skipped') }}
+        needs: [sweep-evals, sweep-agentic-evals, sweep-multi-node-evals, sweep-multi-node-agentic-evals, sweep-quality-evals, setup]
+        if: ${{ always() && needs.setup.result != 'skipped' && (needs.sweep-evals.result != 'skipped' || needs.sweep-agentic-evals.result != 'skipped' || needs.sweep-multi-node-evals.result != 'skipped' || needs.sweep-multi-node-agentic-evals.result != 'skipped' || needs.sweep-quality-evals.result != 'skipped') }}
         uses: ./.github/workflows/collect-evals.yml
         secrets: inherit
 

BENCHMARK_TIMES.md

@@ -0,0 +1,23 @@
+# Quality Benchmark Time Estimates
+
+Estimates are for GLM-5.2 at concurrency 4. API latency, reasoning length,
+rate limits, image setup, and sandbox execution can change wall time.
+
+| Benchmark | Full size | Full estimate | 20-sample estimate | Default max generation |
+|---|---:|---:|---:|---:|
+| GPQA-Diamond | 198 | ~20 min | ~2 min | 8K |
+| MMLU-Pro | 12,032 | ~22 h | ~2 min* | 8K |
+| HLE | 2,158 | ~4.5 h | ~5 min* | 16K |
+| LiveCodeBench | 1,055 | ~8 h | ~15 min | 16K |
+| BFCL v4 | ~2,000 | ~1.5 h | <1 min | 8K |
+| SciCode | 288 subproblems | ~5.5 h | ~23 min | 16K |
+| SWE-bench Pro | 731 | ~45 h | ~30 min | 32K per agent turn |
+| DeepSWE | 113 | ~29 h | ~5 h | 32K per agent turn |
+
+\* `LIMIT` applies per subtask for MMLU-Pro and HLE, so `LIMIT=20` evaluates
+more than 20 total questions.
+
+These are output ceilings, not target lengths. Natural stop conditions still
+end generation early. Override any default with `MAX_GEN_TOKENS`; CI matrix
+rows may use `max-gen-tokens`. Streaming avoids idle connection timeouts but
+does not remove total request deadlines or the cost of long generations.