ci(verify): nightly diff against the cap-v0.6.0 baseline

.github/workflows/baseline-diff.yml

@@ -0,0 +1,161 @@
+name: Baseline diff
+
+# Report artifacts expire after 90 days and the baseline branch never
+# changes, so the baseline is re-run weekly.
+
+on:
+  workflow_run:
+    workflows: ["Engine verify", "Verify recording (real capture)"]
+    types: [completed]
+    branches: [main]
+  schedule:
+    - cron: "17 2 * * 1"
+  workflow_dispatch:
+    inputs:
+      main-run:
+        description: "Run id on main to compare (Engine verify or real capture)"
+        required: true
+      baseline-run:
+        description: "Baseline run id (default: the latest of the same workflow on scratch/prod-baseline)"
+        default: ""
+
+permissions:
+  contents: read
+
+concurrency:
+  group: baseline-diff-${{ github.event.workflow_run.id || inputs.main-run || 'refresh' }}
+  cancel-in-progress: false
+
+env:
+  BASELINE_BRANCH: scratch/prod-baseline
+
+jobs:
+  refresh:
+    if: github.event_name == 'schedule'
+    runs-on: blacksmith-2vcpu-ubuntu-2404
+    permissions:
+      actions: write
+    steps:
+      - name: Re-run the baseline
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh workflow run engine-verify.yml --repo "$GITHUB_REPOSITORY" --ref "$BASELINE_BRANCH" --raw-field tier=full
+          gh workflow run verify-recording.yml --repo "$GITHUB_REPOSITORY" --ref "$BASELINE_BRANCH" \
+            --raw-field tier=full --raw-field repeat=3 --raw-field runners=both
+
+  diff:
+    permissions:
+      contents: read
+      actions: read
+    if: >-
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'workflow_run' && github.event.workflow_run.event == 'schedule')
+    runs-on: blacksmith-2vcpu-ubuntu-2404
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          persist-credentials: false
+
+      - name: Runs
+        id: runs
+        env:
+          GH_TOKEN: ${{ github.token }}
+          MAIN_RUN: ${{ github.event.workflow_run.id || inputs.main-run }}
+          BASELINE_INPUT: ${{ inputs.baseline-run }}
+        run: |
+          set -euo pipefail
+          workflow=$(gh run view "$MAIN_RUN" --repo "$GITHUB_REPOSITORY" --json workflowDatabaseId --jq .workflowDatabaseId)
+          baseline="$BASELINE_INPUT"
+          if [[ -z "$baseline" ]]; then
+            # Baseline runs fail (0.6.0 has known bugs), but a run with a
+            # cancelled or timed-out job stopped before reporting every case.
+            for candidate in $(gh run list --repo "$GITHUB_REPOSITORY" --workflow "$workflow" --branch "$BASELINE_BRANCH" \
+              --limit 20 --json databaseId,conclusion \
+              --jq '.[] | select(.conclusion == "success" or .conclusion == "failure") | .databaseId'); do
+              stopped=$(gh run view "$candidate" --repo "$GITHUB_REPOSITORY" --json conclusion,jobs \
+                --jq '[.conclusion, .jobs[].conclusion] | map(select(. == "cancelled" or . == "timed_out")) | length')
+              if [[ "$stopped" == 0 ]]; then
+                baseline="$candidate"
+                break
+              fi
+            done
+          fi
+          if [[ -z "$baseline" ]]; then
+            echo "::error::no completed baseline run of workflow $workflow on $BASELINE_BRANCH"
+            exit 1
+          fi
+          echo "main=$MAIN_RUN" >> "$GITHUB_OUTPUT"
+          echo "baseline=$baseline" >> "$GITHUB_OUTPUT"
+          echo "Comparing main run $MAIN_RUN with baseline run $baseline" | tee -a "$GITHUB_STEP_SUMMARY"
+
+      - name: Download reports
+        env:
+          GH_TOKEN: ${{ github.token }}
+          MAIN_RUN: ${{ steps.runs.outputs.main }}
+          BASELINE_RUN: ${{ steps.runs.outputs.baseline }}
+        run: |
+          set -euo pipefail
+          for side in main baseline; do
+            run=$([[ $side == main ]] && echo "$MAIN_RUN" || echo "$BASELINE_RUN")
+            dest="$RUNNER_TEMP/reports/$side"
+            mkdir -p "$dest"
+            # Real capture also uploads its reports alone; runs from before
+            # that only have the artifact with the recordings.
+            names=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$run/artifacts" --paginate --jq '.artifacts[].name')
+            if grep -q '^verify-report-' <<< "$names"; then
+              pattern='verify-report-*'
+            elif grep -q '^verify-recording-report-' <<< "$names"; then
+              pattern='verify-recording-report-*'
+            elif grep -q '^verify-recording-' <<< "$names"; then
+              pattern='verify-recording-*'
+            else
+              echo "::error::run $run has no verification reports"
+              exit 1
+            fi
+            gh run download "$run" --repo "$GITHUB_REPOSITORY" --dir "$dest" --pattern "$pattern"
+            find "$dest" -type d -name verify-scratch -prune -exec rm -r {} +
+            echo "$side: $(find "$dest" -name '*.jsonl' | wc -l) report files"
+          done
+
+      - name: Baseline covers main's runners
+        shell: bash
+        run: |
+          set -euo pipefail
+          # Lanes as diff.py names them: shard numbers and tiers dropped.
+          lanes() {
+            local dir
+            for dir in "$1"/verify-*/; do
+              if [[ -d "$dir" ]]; then basename "$dir"; fi
+            done | sed -E 's/^verify-recording-report-/verify-recording-/; s/-(fast|full)(-[0-9]+)?$//' | sort -u
+          }
+          main_lanes=$(lanes "$RUNNER_TEMP/reports/main")
+          baseline_lanes=$(lanes "$RUNNER_TEMP/reports/baseline")
+          if [[ -z "$main_lanes" ]]; then
+            echo "::error::main's run has no report directories"
+            exit 1
+          fi
+          missing=$(comm -23 <(echo "$main_lanes") <(echo "$baseline_lanes"))
+          if [[ -n "$missing" ]]; then
+            echo "::error::the baseline run has no reports for: $(echo $missing)"
+            exit 1
+          fi
+
+      - name: Diff
+        run: |
+          python3 scripts/verify/diff.py \
+            "$RUNNER_TEMP/reports/baseline" "$RUNNER_TEMP/reports/main" \
+            --na scripts/verify/baseline-na.tsv \
+            --title "cap-v0.6.0 baseline vs main" \
+            --fail-on-worse > "$RUNNER_TEMP/baseline-diff.md"
+          status=$?
+          cat "$RUNNER_TEMP/baseline-diff.md" >> "$GITHUB_STEP_SUMMARY"
+          exit "$status"
+        shell: bash {0}
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: baseline-diff
+          path: ${{ runner.temp }}/baseline-diff.md
+          if-no-files-found: ignore

.github/workflows/verify-recording.yml

@@ -317,6 +317,14 @@ jobs:
         run: |
           python3 scripts/verify/summary.py --title "real capture ${{ matrix.runner }} ${{ needs.scope.outputs.tier }}" "$CAP_VERIFY_REPORT_DIR" >> "$GITHUB_STEP_SUMMARY" || true
 
+      - name: Upload reports
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: verify-recording-report-${{ matrix.runner }}
+          path: ${{ github.workspace }}/verify-report
+          if-no-files-found: ignore
+
       - name: Upload artifacts
         if: always()
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2

scripts/verify/baseline-na.tsv

@@ -0,0 +1,2 @@
+# glob<TAB>reason (cases n/a on the cap-v0.6.0 production baseline)
+captions/*	caption projection model (synchronize_captions) and transcription audio assembly did not exist in cap-v0.6.0; captions suite not run

scripts/verify/diff.py

@@ -34,7 +34,7 @@ def lane_of(artifact):
     m = re.match(r"verify-report-(.+?)-(fast|full)(-\d+)?$", artifact)
     if m:
         return m.group(1)
-    m = re.match(r"verify-recording-(.+)$", artifact)
+    m = re.match(r"verify-recording-(?:report-)?(.+)$", artifact)
     if m:
         return "recording-" + m.group(1)
     return artifact
@@ -326,6 +326,7 @@ def main():
     ap.add_argument("--json")
     ap.add_argument("--title", default="baseline vs main")
     ap.add_argument("--na", help="file of 'case-glob<TAB>reason' lines: cases n/a on the baseline")
+    ap.add_argument("--fail-on-worse", action="store_true", help="exit 1 when any lane has WORSE or junit WORSE rows")
     args = ap.parse_args()
 
     na = load_na(args.na)
@@ -400,7 +401,31 @@ def main():
     if args.json:
         with open(args.json, "w") as fh:
             json.dump(out, fh, indent=1, default=list)
+    if args.fail_on_worse:
+        # A unit test that main no longer has was renamed or removed with the
+        # code it tested; the case-level rows cover the product.
+        def counted(items):
+            return [item for item in items or [] if not any(r.startswith("missing on main (baseline") for r in item[1])]
+
+        worse = sum(len(counted(lane.get("WORSE"))) + len(counted(lane.get("JUNIT-WORSE"))) for lane in out.values())
+        # A baseline run that stopped early, or a baseline suite that fell
+        # behind main's, leaves main's cases unmatched. Cases new on main are
+        # expected, a tenth of a lane is not.
+        for lane in lanes:
+            main_cases = len(mrec.get(lane) or {})
+            only_main = len(out.get(lane, {}).get("ONLY-MAIN") or [])
+            if main_cases and (not brec.get(lane) or only_main * 10 > main_cases):
+                print(
+                    f"lane {lane}: {only_main} of {main_cases} main cases have no baseline record "
+                    "(re-run the baseline, or refresh its suite)",
+                    file=sys.stderr,
+                )
+                worse += 1
+        if worse:
+            print(f"\n{worse} row(s) worse than the baseline", file=sys.stderr)
+            return 1
+    return 0
 
 
 if __name__ == "__main__":
-    main()
+    sys.exit(main())