diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8f999c66..645fecb4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -230,7 +230,22 @@ jobs: # spurious "NEW FAILURES: - " line and fail this gate. jq -r '(.failures.parity // []) + (.failures.compile // []) | .[] | select(. != "")' "$REPORT" | sort -u > /tmp/all_fails.txt if [[ -f "$KNOWN" ]]; then - jq -r 'keys[]' "$KNOWN" | sort -u > /tmp/known.txt + # Issue #797 — known_failures.json moved from flat strings to + # structured records. Keep the audit metadata (the `_schema` + # key at the top of the file) out of the test-name set so + # CI doesn't try to match a real test against it. + jq -r 'keys[] | select(. != "_schema")' "$KNOWN" | sort -u > /tmp/known.txt + + # Schema sanity check — every non-_schema entry must be an + # object with non-empty `category` and `reason`. Fails the + # build on malformed entries so the format doesn't silently + # drift back to the legacy flat-string shape. + bad="$(jq -r 'to_entries | map(select(.key != "_schema")) | .[] | select((.value | type) != "object" or (.value.category // "") == "" or (.value.reason // "") == "") | .key' "$KNOWN")" + if [[ -n "$bad" ]]; then + echo "Malformed known_failures.json entries (missing category/reason or not an object):" + echo "$bad" | sed 's/^/ - /' + exit 1 + fi else : > /tmp/known.txt fi diff --git a/test-parity/known_failures.json b/test-parity/known_failures.json index d14e5841..6e6b25fe 100644 --- a/test-parity/known_failures.json +++ b/test-parity/known_failures.json @@ -1,60 +1,359 @@ { - "issue_655_repro": "issue #655 repro \u2014 kept as the canonical regression-catcher; will flip to PASS when #655 lands.", - "test_gap_array_methods": "Array method coverage probe \u2014 small divergences (e.g. flat/flatMap/finally specific edge cases). Targeted bisect needed; not a recent regression.", - "test_issue_233_async_array_param_push": "Async array-param push edge case (#233) regresses on Linux CI but passes locally on macOS. Bisect to pin Linux-only divergence.", - "test_issue_561_sigv4_chain": "SigV4 chain test (#561) regresses on Linux CI but passes locally on macOS. Likely related to crypto/HMAC ordering. Bisect needed.", - "test_issue_654_typed_array_dispatch": "Typed array dispatch test (#654) regresses on Linux CI but passes locally on macOS. Bisect needed for Linux-specific dispatch divergence.", - "test_edge_regression": "Broad edge-case bundle covering multiple historical regressions; bisect needed to pin which sub-case still fails. Not a recent regression.", - "test_edge_type_narrowing": "Several TS type-narrowing patterns (control-flow narrowing, discriminated unions through generics, `in` operator narrowing) Perry's HIR doesn't track yet. Surfaced multiple narrow-fix candidates.", - "test_gap_regexp_advanced": "Lookbehind regex assertions \u2014 Rust's `regex` crate doesn't support `(?<=)` / `(?: ` + `at `). Matching Node byte-for-byte requires source-text retention through the runtime.", - "test_issue_510_primitive_method_typeerror": "Same root cause as test_issue_462 \u2014 see #616. Behavior correct; framing differs. Regression vector: v0.5.702 (#596) routed throw helpers through js_throw which exposed the format gap.", - "test_issue_685_nested_class_static_param": "#685 nested-class static-field init closed in v0.5.814 on macOS; Linux CI still surfaces an adjacent shape (different sub-case). Needs targeted Linux-side bisect.", - "test_net_min": "Net test passes locally with echo server running, fails on Linux CI (no echo fixture server). Environmental.", - "test_net_socket": "Net test passes locally with echo server running, fails on Linux CI (no echo fixture server). Environmental.", - "test_net_upgrade_tls": "Net test needs a TLS-capable echo server fixture; not available on CI. Environmental.", - "test_parity_assert": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:assert` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_async_hooks": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:async_hooks` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_buffer": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:buffer` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_child_process": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:child_process` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_cluster": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:cluster` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_console": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:console` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_crypto": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:crypto` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_dgram": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:dgram` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_diagnostics_channel": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:diagnostics_channel` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_dns": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:dns` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_dns_promises": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:dns_promises` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_events": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:events` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_fs": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:fs` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_fs_promises": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:fs_promises` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_http": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:http` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_http2": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:http2` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_https": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:https` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_module": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:module` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_net": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:net` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_os": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:os` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_path": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:path` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_perf_hooks": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:perf_hooks` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_process": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:process` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_querystring": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:querystring` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_readline": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:readline` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_readline_promises": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:readline_promises` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_sqlite": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:sqlite` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_stream": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:stream` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_stream_consumers": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:stream_consumers` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_stream_promises": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:stream_promises` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_stream_web": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:stream_web` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_string_decoder": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:string_decoder` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_sys": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:sys` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_test": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:test` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_timers": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:timers` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_timers_promises": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:timers_promises` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_tls": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:tls` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_tty": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:tty` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_url": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:url` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_util": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:util` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_worker_threads": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:worker_threads` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_parity_zlib": "Node.js module inventory \u2014 Perry hasn't implemented the full `node:zlib` surface yet. Tracker for surface coverage; flips to PASS as each module's API lands. Not a regression.", - "test_sock_write_map": "Net test passes locally with echo server running, fails on Linux CI (no echo fixture server). Environmental \u2014 issue #91 dispatch fix landed in v0.5.581." + "_schema": { + "description": "Each entry maps a parity-suite test name to a structured record. The CI gate in .github/workflows/test.yml reads `keys[]` from this file — a test that fails AND is not a key here breaks CI. Add entries here only with full provenance (issue + added date) so we can revalidate.", + "fields": { + "issue": "Open GitHub issue number tracking this failure (e.g. \"793\"), or null when the failure is environmental / pending triage. Closed issues must be re-evaluated — flag the entry as `category: bug-stale` until a new tracking issue is filed.", + "added": "ISO date (YYYY-MM-DD) when the test was first skip-listed. Use 2026-05-15 for entries inherited from the pre-audit format where the historical date is unknown.", + "category": "ci-env | module-inventory | bug-open | bug-stale | gap-categorical | gap-bisect — see test-parity/README.md for definitions.", + "reason": "Free-text explanation. Keep the most-actionable signal first — what changes the verdict (a fixture, a Perry fix, a Node spec change)." + } + }, + "issue_655_repro": { + "issue": "655", + "added": "2026-05-15", + "category": "bug-open", + "reason": "Canonical regression-catcher for issue #655. Flips to PASS when the underlying fix lands. Keep regardless of CI verdict — it's a deliberate failing repro." + }, + "test_gap_array_methods": { + "issue": null, + "added": "2026-05-15", + "category": "gap-bisect", + "reason": "Array method coverage probe — small divergences (flat/flatMap/finally specific edge cases). Targeted bisect needed; not a recent regression. File a tracking issue once a specific sub-case is pinned." + }, + "test_issue_233_async_array_param_push": { + "issue": "793", + "added": "2026-05-15", + "category": "bug-stale", + "reason": "#233 (16-element cap on async array-param push) closed on macOS. Linux CI still surfaces a divergence — bisect needed to pin the Linux-only sub-case before filing a new tracking issue. Rolled under the parity roadmap (#793) in the meantime." + }, + "test_issue_561_sigv4_chain": { + "issue": "793", + "added": "2026-05-15", + "category": "bug-stale", + "reason": "#561 (Web Crypto subtle.digest/sign for HMAC-SHA-256) closed but the SigV4 chain test still regresses on Linux CI. Bisect needed for crypto/HMAC ordering on Linux; file a Linux-specific tracking issue once the divergence is identified." + }, + "test_issue_654_typed_array_dispatch": { + "issue": "793", + "added": "2026-05-15", + "category": "bug-stale", + "reason": "#654 (Float64Array typeof + .sort/.at dispatch) closed but the typed-array dispatch test still regresses on Linux CI. Linux-specific dispatch divergence; bisect needed before re-filing." + }, + "test_edge_regression": { + "issue": null, + "added": "2026-05-15", + "category": "gap-bisect", + "reason": "Broad edge-case bundle covering multiple historical regressions. Bisect needed to pin which sub-case still fails so a tracking issue can be filed against that specific behavior. Not a recent regression." + }, + "test_edge_type_narrowing": { + "issue": null, + "added": "2026-05-15", + "category": "gap-bisect", + "reason": "Several TS type-narrowing patterns (control-flow narrowing, discriminated unions through generics, `in` operator narrowing) Perry's HIR doesn't track yet. Surfaced multiple narrow-fix candidates; split into per-pattern issues before tackling." + }, + "test_gap_regexp_advanced": { + "issue": null, + "added": "2026-05-15", + "category": "gap-categorical", + "reason": "Lookbehind regex assertions — Rust's `regex` crate doesn't support `(?<=)` / `(?