diff --git a/.github/workflows/baseline-diff.yml b/.github/workflows/baseline-diff.yml new file mode 100644 index 0000000000..2a71f59ec4 --- /dev/null +++ b/.github/workflows/baseline-diff.yml @@ -0,0 +1,161 @@ +name: Baseline diff + +# Report artifacts expire after 90 days and the baseline branch never +# changes, so the baseline is re-run weekly. + +on: + workflow_run: + workflows: ["Engine verify", "Verify recording (real capture)"] + types: [completed] + branches: [main] + schedule: + - cron: "17 2 * * 1" + workflow_dispatch: + inputs: + main-run: + description: "Run id on main to compare (Engine verify or real capture)" + required: true + baseline-run: + description: "Baseline run id (default: the latest of the same workflow on scratch/prod-baseline)" + default: "" + +permissions: + contents: read + +concurrency: + group: baseline-diff-${{ github.event.workflow_run.id || inputs.main-run || 'refresh' }} + cancel-in-progress: false + +env: + BASELINE_BRANCH: scratch/prod-baseline + +jobs: + refresh: + if: github.event_name == 'schedule' + runs-on: blacksmith-2vcpu-ubuntu-2404 + permissions: + actions: write + steps: + - name: Re-run the baseline + env: + GH_TOKEN: ${{ github.token }} + run: | + gh workflow run engine-verify.yml --repo "$GITHUB_REPOSITORY" --ref "$BASELINE_BRANCH" --raw-field tier=full + gh workflow run verify-recording.yml --repo "$GITHUB_REPOSITORY" --ref "$BASELINE_BRANCH" \ + --raw-field tier=full --raw-field repeat=3 --raw-field runners=both + + diff: + permissions: + contents: read + actions: read + if: >- + github.event_name == 'workflow_dispatch' || + (github.event_name == 'workflow_run' && github.event.workflow_run.event == 'schedule') + runs-on: blacksmith-2vcpu-ubuntu-2404 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + + - name: Runs + id: runs + env: + GH_TOKEN: ${{ github.token }} + MAIN_RUN: ${{ github.event.workflow_run.id || inputs.main-run }} + BASELINE_INPUT: ${{ inputs.baseline-run }} + run: | + set -euo pipefail + workflow=$(gh run view "$MAIN_RUN" --repo "$GITHUB_REPOSITORY" --json workflowDatabaseId --jq .workflowDatabaseId) + baseline="$BASELINE_INPUT" + if [[ -z "$baseline" ]]; then + # Baseline runs fail (0.6.0 has known bugs), but a run with a + # cancelled or timed-out job stopped before reporting every case. + for candidate in $(gh run list --repo "$GITHUB_REPOSITORY" --workflow "$workflow" --branch "$BASELINE_BRANCH" \ + --limit 20 --json databaseId,conclusion \ + --jq '.[] | select(.conclusion == "success" or .conclusion == "failure") | .databaseId'); do + stopped=$(gh run view "$candidate" --repo "$GITHUB_REPOSITORY" --json conclusion,jobs \ + --jq '[.conclusion, .jobs[].conclusion] | map(select(. == "cancelled" or . == "timed_out")) | length') + if [[ "$stopped" == 0 ]]; then + baseline="$candidate" + break + fi + done + fi + if [[ -z "$baseline" ]]; then + echo "::error::no completed baseline run of workflow $workflow on $BASELINE_BRANCH" + exit 1 + fi + echo "main=$MAIN_RUN" >> "$GITHUB_OUTPUT" + echo "baseline=$baseline" >> "$GITHUB_OUTPUT" + echo "Comparing main run $MAIN_RUN with baseline run $baseline" | tee -a "$GITHUB_STEP_SUMMARY" + + - name: Download reports + env: + GH_TOKEN: ${{ github.token }} + MAIN_RUN: ${{ steps.runs.outputs.main }} + BASELINE_RUN: ${{ steps.runs.outputs.baseline }} + run: | + set -euo pipefail + for side in main baseline; do + run=$([[ $side == main ]] && echo "$MAIN_RUN" || echo "$BASELINE_RUN") + dest="$RUNNER_TEMP/reports/$side" + mkdir -p "$dest" + # Real capture also uploads its reports alone; runs from before + # that only have the artifact with the recordings. + names=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$run/artifacts" --paginate --jq '.artifacts[].name') + if grep -q '^verify-report-' <<< "$names"; then + pattern='verify-report-*' + elif grep -q '^verify-recording-report-' <<< "$names"; then + pattern='verify-recording-report-*' + elif grep -q '^verify-recording-' <<< "$names"; then + pattern='verify-recording-*' + else + echo "::error::run $run has no verification reports" + exit 1 + fi + gh run download "$run" --repo "$GITHUB_REPOSITORY" --dir "$dest" --pattern "$pattern" + find "$dest" -type d -name verify-scratch -prune -exec rm -r {} + + echo "$side: $(find "$dest" -name '*.jsonl' | wc -l) report files" + done + + - name: Baseline covers main's runners + shell: bash + run: | + set -euo pipefail + # Lanes as diff.py names them: shard numbers and tiers dropped. + lanes() { + local dir + for dir in "$1"/verify-*/; do + if [[ -d "$dir" ]]; then basename "$dir"; fi + done | sed -E 's/^verify-recording-report-/verify-recording-/; s/-(fast|full)(-[0-9]+)?$//' | sort -u + } + main_lanes=$(lanes "$RUNNER_TEMP/reports/main") + baseline_lanes=$(lanes "$RUNNER_TEMP/reports/baseline") + if [[ -z "$main_lanes" ]]; then + echo "::error::main's run has no report directories" + exit 1 + fi + missing=$(comm -23 <(echo "$main_lanes") <(echo "$baseline_lanes")) + if [[ -n "$missing" ]]; then + echo "::error::the baseline run has no reports for: $(echo $missing)" + exit 1 + fi + + - name: Diff + run: | + python3 scripts/verify/diff.py \ + "$RUNNER_TEMP/reports/baseline" "$RUNNER_TEMP/reports/main" \ + --na scripts/verify/baseline-na.tsv \ + --title "cap-v0.6.0 baseline vs main" \ + --fail-on-worse > "$RUNNER_TEMP/baseline-diff.md" + status=$? + cat "$RUNNER_TEMP/baseline-diff.md" >> "$GITHUB_STEP_SUMMARY" + exit "$status" + shell: bash {0} + + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: always() + with: + name: baseline-diff + path: ${{ runner.temp }}/baseline-diff.md + if-no-files-found: ignore diff --git a/.github/workflows/verify-recording.yml b/.github/workflows/verify-recording.yml index 28a0af8872..41ef09139c 100644 --- a/.github/workflows/verify-recording.yml +++ b/.github/workflows/verify-recording.yml @@ -317,6 +317,14 @@ jobs: run: | python3 scripts/verify/summary.py --title "real capture ${{ matrix.runner }} ${{ needs.scope.outputs.tier }}" "$CAP_VERIFY_REPORT_DIR" >> "$GITHUB_STEP_SUMMARY" || true + - name: Upload reports + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: verify-recording-report-${{ matrix.runner }} + path: ${{ github.workspace }}/verify-report + if-no-files-found: ignore + - name: Upload artifacts if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 diff --git a/scripts/verify/baseline-na.tsv b/scripts/verify/baseline-na.tsv new file mode 100644 index 0000000000..877252ee91 --- /dev/null +++ b/scripts/verify/baseline-na.tsv @@ -0,0 +1,2 @@ +# globreason (cases n/a on the cap-v0.6.0 production baseline) +captions/* caption projection model (synchronize_captions) and transcription audio assembly did not exist in cap-v0.6.0; captions suite not run diff --git a/scripts/verify/diff.py b/scripts/verify/diff.py index d9332ad199..3beb84d7cf 100644 --- a/scripts/verify/diff.py +++ b/scripts/verify/diff.py @@ -34,7 +34,7 @@ def lane_of(artifact): m = re.match(r"verify-report-(.+?)-(fast|full)(-\d+)?$", artifact) if m: return m.group(1) - m = re.match(r"verify-recording-(.+)$", artifact) + m = re.match(r"verify-recording-(?:report-)?(.+)$", artifact) if m: return "recording-" + m.group(1) return artifact @@ -326,6 +326,7 @@ def main(): ap.add_argument("--json") ap.add_argument("--title", default="baseline vs main") ap.add_argument("--na", help="file of 'case-globreason' lines: cases n/a on the baseline") + ap.add_argument("--fail-on-worse", action="store_true", help="exit 1 when any lane has WORSE or junit WORSE rows") args = ap.parse_args() na = load_na(args.na) @@ -400,7 +401,31 @@ def main(): if args.json: with open(args.json, "w") as fh: json.dump(out, fh, indent=1, default=list) + if args.fail_on_worse: + # A unit test that main no longer has was renamed or removed with the + # code it tested; the case-level rows cover the product. + def counted(items): + return [item for item in items or [] if not any(r.startswith("missing on main (baseline") for r in item[1])] + + worse = sum(len(counted(lane.get("WORSE"))) + len(counted(lane.get("JUNIT-WORSE"))) for lane in out.values()) + # A baseline run that stopped early, or a baseline suite that fell + # behind main's, leaves main's cases unmatched. Cases new on main are + # expected, a tenth of a lane is not. + for lane in lanes: + main_cases = len(mrec.get(lane) or {}) + only_main = len(out.get(lane, {}).get("ONLY-MAIN") or []) + if main_cases and (not brec.get(lane) or only_main * 10 > main_cases): + print( + f"lane {lane}: {only_main} of {main_cases} main cases have no baseline record " + "(re-run the baseline, or refresh its suite)", + file=sys.stderr, + ) + worse += 1 + if worse: + print(f"\n{worse} row(s) worse than the baseline", file=sys.stderr) + return 1 + return 0 if __name__ == "__main__": - main() + sys.exit(main())