ci: add agent guide and benchmark comparison

.github/workflows/benchmark_comment.yml

@@ -0,0 +1,123 @@
+name: CI / Benchmark Comment
+
+permissions:
+  actions: read
+  contents: read
+  issues: write
+  pull-requests: write
+
+defaults:
+  run:
+    shell: bash
+
+on:
+  workflow_run:
+    workflows: ["CI / Benchmark Compare"]
+    types: [completed]
+
+jobs:
+  benchmark-comment:
+    if: github.event.workflow_run.event == 'pull_request'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+
+    steps:
+      - name: Find benchmark artifact
+        id: artifact
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const run = context.payload.workflow_run;
+            const {owner, repo} = context.repo;
+            let pr = run.pull_requests && run.pull_requests[0];
+            if (!pr) {
+              const prs = await github.paginate(
+                github.rest.repos.listPullRequestsAssociatedWithCommit,
+                {owner, repo, commit_sha: run.head_sha});
+              pr = prs.find((item) => item.state === 'open') || prs[0];
+            }
+            if (!pr) {
+              core.setOutput('skip', 'true');
+              return;
+            }
+
+            const artifacts = await github.paginate(
+              github.rest.actions.listWorkflowRunArtifacts,
+              {owner, repo, run_id: run.id});
+            const artifact = artifacts.find((item) =>
+              item.name === 'benchmark-compare' && !item.expired);
+            if (!artifact) {
+              core.setOutput('skip', 'true');
+              return;
+            }
+
+            core.setOutput('skip', 'false');
+            core.setOutput('pr_number', String(pr.number));
+            core.setOutput('artifact_id', String(artifact.id));
+
+      - name: Download benchmark artifact
+        if: steps.artifact.outputs.skip != 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          ARTIFACT_ID: ${{ steps.artifact.outputs.artifact_id }}
+          REPOSITORY: ${{ github.repository }}
+        run: |
+          gh api "repos/${REPOSITORY}/actions/artifacts/${ARTIFACT_ID}/zip" > "$RUNNER_TEMP/benchmark-compare.zip"
+          python3 - <<'PY'
+          import os
+          import zipfile
+
+          zip_path = os.path.join(os.environ["RUNNER_TEMP"], "benchmark-compare.zip")
+          out_path = os.path.join(os.environ["RUNNER_TEMP"], "benchmark_comment.md")
+
+          with zipfile.ZipFile(zip_path) as archive:
+              names = [
+                  name for name in archive.namelist()
+                  if name.endswith("benchmark_comment.md")
+                  or name.endswith("benchmark_summary.md")
+              ]
+              if not names:
+                  raise SystemExit("benchmark markdown result not found in artifact")
+              with archive.open(names[0]) as src, open(out_path, "wb") as dst:
+                  dst.write(src.read())
+          PY
+
+      - name: Comment benchmark result
+        if: steps.artifact.outputs.skip != 'true'
+        uses: actions/github-script@v7
+        env:
+          PR_NUMBER: ${{ steps.artifact.outputs.pr_number }}
+        with:
+          script: |
+            const fs = require('fs');
+            const path = require('path');
+            const marker = '<!-- sonic-benchmark-compare -->';
+            const commentPath = path.join(process.env.RUNNER_TEMP, 'benchmark_comment.md');
+            const result = fs.existsSync(commentPath)
+              ? fs.readFileSync(commentPath, 'utf8')
+              : '## Benchmark\n\nBenchmark comparison did not produce a result. Check the workflow logs and artifacts.';
+            const body = `${marker}\n${result}`;
+            const {owner, repo} = context.repo;
+            const issue_number = Number(process.env.PR_NUMBER);
+            const comments = await github.paginate(github.rest.issues.listComments, {
+              owner,
+              repo,
+              issue_number,
+            });
+            const existing = comments.find((comment) =>
+              comment.body && comment.body.includes(marker));
+            if (existing) {
+              await github.rest.issues.updateComment({
+                owner,
+                repo,
+                comment_id: existing.id,
+                body,
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner,
+                repo,
+                issue_number,
+                body,
+              });
+            }

.github/workflows/benchmark_compare.yml

@@ -0,0 +1,117 @@
+name: CI / Benchmark Compare
+
+permissions:
+  contents: read
+  pull-requests: read
+
+defaults:
+  run:
+    shell: bash
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  benchmark-compare:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+
+    steps:
+      - name: Initialize benchmark result
+        run: |
+          cat > "$RUNNER_TEMP/benchmark_comment.md" <<'EOF'
+          ## Benchmark
+
+          Benchmark comparison did not complete. Check the workflow logs.
+
+          Overall severity: `critical`
+
+          Conclusion: informational only. Benchmark issues are reported in this summary/artifact and do not block the workflow.
+          EOF
+
+      - name: Checkout base
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ github.event.pull_request.base.repo.full_name || github.repository }}
+          ref: ${{ github.event.pull_request.base.sha || github.sha }}
+          path: base
+          fetch-depth: 1
+
+      - name: Checkout head
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          path: head
+          fetch-depth: 1
+
+      - name: Cache Bazel
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/bazel
+            ~/.cache/bazelisk
+          key: ${{ runner.os }}-benchmark-bazel-${{ hashFiles('base/.bazelversion', 'head/.bazelversion', 'base/MODULE.bazel.lock', 'head/MODULE.bazel.lock', 'base/WORKSPACE.bzlmod', 'head/WORKSPACE.bzlmod', 'base/BUILD.bazel', 'head/BUILD.bazel') }}
+          restore-keys: |
+            ${{ runner.os }}-benchmark-bazel-
+
+      - name: Benchmark base
+        id: benchmark_base
+        continue-on-error: true
+        working-directory: base
+        run: |
+          source scripts/bazel_common.sh
+          BAZEL_BIN="$(sonic_pick_bazel "$PWD")"
+          "$BAZEL_BIN" run --compilation_mode=opt --//:sonic_arch=haswell --//:sonic_dispatch=static :benchmark -- --benchmark_filter=Sonic --benchmark_repetitions=5 --benchmark_report_aggregates_only=true --benchmark_out="$RUNNER_TEMP/sonic_bench_base.json" --benchmark_out_format=json
+
+      - name: Benchmark head
+        id: benchmark_head
+        continue-on-error: true
+        working-directory: head
+        run: |
+          source scripts/bazel_common.sh
+          BAZEL_BIN="$(sonic_pick_bazel "$PWD")"
+          "$BAZEL_BIN" run --compilation_mode=opt --//:sonic_arch=haswell --//:sonic_dispatch=static :benchmark -- --benchmark_filter=Sonic --benchmark_repetitions=5 --benchmark_report_aggregates_only=true --benchmark_out="$RUNNER_TEMP/sonic_bench_head.json" --benchmark_out_format=json
+
+      - name: Write benchmark execution failure
+        if: steps.benchmark_base.outcome == 'failure' || steps.benchmark_head.outcome == 'failure'
+        run: |
+          cat > "$RUNNER_TEMP/benchmark_comment.md" <<EOF
+          ## Benchmark
+
+          Benchmark execution did not complete before comparison.
+
+          | Severity | Step | Outcome |
+          | --- | --- | --- |
+          | critical | base | ${{ steps.benchmark_base.outcome }} |
+          | critical | head | ${{ steps.benchmark_head.outcome }} |
+
+          Overall severity: `critical`
+
+          Conclusion: informational only. Benchmark issues are reported in this summary/artifact and do not block the workflow.
+          EOF
+          cat "$RUNNER_TEMP/benchmark_comment.md" >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Compare benchmark
+        if: steps.benchmark_base.outcome == 'success' && steps.benchmark_head.outcome == 'success'
+        id: compare
+        working-directory: head
+        run: |
+          python3 scripts/tools/compare-benchmark.py "$RUNNER_TEMP/sonic_bench_base.json" "$RUNNER_TEMP/sonic_bench_head.json" --include-regex=Sonic --warn-threshold=3 --fail-threshold=10 --output="$RUNNER_TEMP/benchmark_comment.md"
+          cat "$RUNNER_TEMP/benchmark_comment.md" >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload benchmark artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-compare
+          if-no-files-found: warn
+          path: |
+            ${{ runner.temp }}/sonic_bench_base.json
+            ${{ runner.temp }}/sonic_bench_head.json
+            ${{ runner.temp }}/benchmark_comment.md

.github/workflows/test_coverage.yml

@@ -72,8 +72,19 @@ jobs:
             bazel-coverage-html
             coverage.dat
 
-      - name: Upload coverage to CodeCov
+      - name: Upload pull request coverage to CodeCov
+        if: github.event_name == 'pull_request'
         uses: codecov/codecov-action@v4
         with:
           files: coverage.dat
           verbose: true
+          fail_ci_if_error: true
+
+      - name: Upload push coverage to CodeCov
+        if: github.event_name == 'push'
+        uses: codecov/codecov-action@v4
+        with:
+          use_oidc: true
+          files: coverage.dat
+          verbose: true
+          fail_ci_if_error: true