From 23529b069138a8036d809e34df52912293185b84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20R=C3=B8ed?= Date: Thu, 7 May 2026 11:57:13 +0200 Subject: [PATCH 01/65] ecosystem CI scaffolding + super-rentals triage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pinned-snapshot validation against 10 public Ember repos (super-rentals, ember-website, ember-power-select, ember-modifier, ember-simple-auth, aria-voyager, ember-a11y-testing, hashicorp/design-system, ember-primitives, NullVoxPopuli/limber). Each target lives in `ecosystem/targets.json` with a pinned SHA, glob, and per-target Glint flag. The runner clones, installs deps (best-effort, lockfile-driven), validates with the same `:gts-recommended` config that `validate-gts` uses, and diffs against committed baselines. A non-empty diff fails the workflow. `ecosystem/run.ts` is single-file, runs via `tsx`, no heavy deps. Glint extraction is gated by `glint` flag per target (default true) and falls back to no-Glint when install fails so the run continues. Per-target triage docs live under `ecosystem/triage/`: - `/triage.md` — every finding classified - `/issue.md` — draft to file upstream when warranted - `FP-FIX-REPORT.md` — aggregate FPs across targets - `STYLISTIC-FINDINGS.md` — input for default-policy review super-rentals triaged. 11 TP, 2 FP-plugin (img splat → required-attr missing), 0 FP-rule. The 2 FPs are the first entry in FP-FIX-REPORT. `vitest.config.ts` scopes test discovery to `test/**` so cloned repos' test files don't get picked up. `tsconfig.test.json` typechecks `ecosystem/**`. --- .github/workflows/ecosystem.yml | 60 + .gitignore | 1 + ecosystem/README.md | 65 + ecosystem/baselines/aria-voyager.json | 97 + ecosystem/baselines/ember-a11y-testing.json | 62 + ecosystem/baselines/ember-modifier.json | 5 + ecosystem/baselines/ember-power-select.json | 244 +++ ecosystem/baselines/ember-primitives.json | 97 + ecosystem/baselines/ember-simple-auth.json | 48 + ecosystem/baselines/ember-website.json | 1959 +++++++++++++++++ ecosystem/baselines/hds-design-system.json | 2127 +++++++++++++++++++ ecosystem/baselines/limber.json | 279 +++ ecosystem/baselines/super-rentals.json | 97 + ecosystem/run.ts | 441 ++++ ecosystem/targets.json | 93 + ecosystem/triage/FP-FIX-REPORT.md | 24 + ecosystem/triage/STYLISTIC-FINDINGS.md | 29 + ecosystem/triage/super-rentals/issue.md | 55 + ecosystem/triage/super-rentals/triage.md | 38 + package-lock.json | 532 ++++- package.json | 5 +- tsconfig.test.json | 4 +- vitest.config.ts | 12 + 23 files changed, 6369 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/ecosystem.yml create mode 100644 ecosystem/README.md create mode 100644 ecosystem/baselines/aria-voyager.json create mode 100644 ecosystem/baselines/ember-a11y-testing.json create mode 100644 ecosystem/baselines/ember-modifier.json create mode 100644 ecosystem/baselines/ember-power-select.json create mode 100644 ecosystem/baselines/ember-primitives.json create mode 100644 ecosystem/baselines/ember-simple-auth.json create mode 100644 ecosystem/baselines/ember-website.json create mode 100644 ecosystem/baselines/hds-design-system.json create mode 100644 ecosystem/baselines/limber.json create mode 100644 ecosystem/baselines/super-rentals.json create mode 100644 ecosystem/run.ts create mode 100644 ecosystem/targets.json create mode 100644 ecosystem/triage/FP-FIX-REPORT.md create mode 100644 ecosystem/triage/STYLISTIC-FINDINGS.md create mode 100644 ecosystem/triage/super-rentals/issue.md create mode 100644 ecosystem/triage/super-rentals/triage.md create mode 100644 vitest.config.ts diff --git a/.github/workflows/ecosystem.yml b/.github/workflows/ecosystem.yml new file mode 100644 index 0000000..a09983e --- /dev/null +++ b/.github/workflows/ecosystem.yml @@ -0,0 +1,60 @@ +name: ecosystem + +on: + pull_request: + branches: [main, master] + paths: + - '*.ts' + - 'lib/**' + - 'ecosystem/**' + - 'package.json' + - 'package-lock.json' + - 'tsconfig.json' + - '.github/workflows/ecosystem.yml' + workflow_dispatch: + +# Cancel in-flight ecosystem runs when a PR force-pushes — clones are +# expensive and the only result we care about is the latest one. +concurrency: + group: ecosystem-${{ github.ref }} + cancel-in-progress: true + +jobs: + ecosystem: + runs-on: ubuntu-latest + # Generous because targets get cloned + their deps installed for Glint + # type-aware extraction. First run is the slow path (cold caches); + # subsequent runs hit the pnpm/npm store cache and are much faster. + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + + - run: corepack enable + - run: npm ci + + # Cache target repo clones AND their installed node_modules across runs. + # Keyed on targets.json so any SHA bump invalidates the cache and forces + # a fresh fetch + reinstall (we want the baseline-vs-actual diff to + # reflect the pinned SHA, not whatever was cached from a previous run). + - name: Cache target clones + node_modules + uses: actions/cache@v4 + with: + path: ecosystem/.cache + key: ecosystem-clones-${{ hashFiles('ecosystem/targets.json') }} + + # The pnpm store is content-addressed; caching it across runs makes + # repeat installs near-instant when the lockfile barely changes. + - name: Cache pnpm store + uses: actions/cache@v4 + with: + path: ~/.pnpm-store + key: ecosystem-pnpm-${{ hashFiles('ecosystem/targets.json') }} + restore-keys: ecosystem-pnpm- + + - name: Run ecosystem checks + run: npm run ecosystem:check diff --git a/.gitignore b/.gitignore index dd6e803..5996dc1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ node_modules/ dist/ *.log .DS_Store +ecosystem/.cache/ diff --git a/ecosystem/README.md b/ecosystem/README.md new file mode 100644 index 0000000..cec123c --- /dev/null +++ b/ecosystem/README.md @@ -0,0 +1,65 @@ +# Ecosystem CI + +Runs `html-validate-ember` against pinned snapshots of public Ember repos and diffs the findings against committed baselines. A non-empty diff fails the workflow. + +## What's here + +- `targets.json` — the list of pinned repos (owner/repo + SHA + globs + per-target options). +- `run.ts` — the runner. Clones, installs deps, validates with Glint, diffs. +- `baselines/.json` — committed expected output per target. The runner refuses to diff if the baseline's pinned SHA disagrees with `targets.json`. + +Cached clones (with their installed `node_modules`) live in `.cache/` (gitignored). + +## Glint + +By default each target is validated with Glint on (`HVE_GLINT=1`). That requires the target's deps installed locally — Glint resolves `@glint/ember-tsc` and the target's TypeScript types from the target's own `node_modules`. The runner detects the package manager from the lockfile and installs with `--ignore-scripts`. If install fails the runner falls back to no-Glint validation for that target and prints a warning rather than failing the run. + +Per-target opt-out: set `"glint": false` on a target in `targets.json` (e.g., for repos where install doesn't work cleanly or isn't worth the time). + +## Local commands + +```bash +npm run ecosystem:check # all targets, fail on diff +npm run ecosystem:check -- --target=limber # just one +npm run ecosystem:check -- --target=a,b,c # subset +npm run ecosystem:check -- --no-clone # skip clone/fetch (faster, uses .cache) +npm run ecosystem:update # rewrite all baselines +npm run ecosystem:update -- --target=foo # rewrite one +``` + +## When to update a baseline + +You're working on the plugin and a real-world target's findings changed. That's the signal — the diff is the value statement of the change. Inspect the `added:` and `removed:` blocks in the runner output: + +- **Removed findings** (fewer reports) — usually a fix landed; a FP got eliminated. Good. Update the baseline in the same PR; reviewer sees what disappeared. +- **Added findings** (new reports) — either the plugin now catches something it didn't before (good — explain in the PR), or it's a regression (don't update the baseline; fix it). + +The runner's diff output is the input to the PR description. Don't paraphrase it; paste the relevant lines. + +## Bumping target SHAs + +Manual for now. Re-fetch the default branch's HEAD, update `ref` in `targets.json`, run `npm run ecosystem:update -- --target=`, commit baseline + targets.json together. The diff in the PR shows what changed in the target's templates between the old and new SHA — those are upstream-side changes, not plugin-side, but they're still worth reading: they're candidates for issue drafts to file with the upstream repo. + +If a baseline is committed against a different SHA than `targets.json`, the runner refuses to diff (apples-to-oranges) and asks you to re-baseline. This guards against forgetting the second commit. + +## Triage flow (separate from CI) + +The committed baselines contain a mix of real bugs, plugin FPs, and html-validate rules that fire on legitimate Ember patterns. Triaging them is *not* what CI does — CI only catches regressions. Triage is a manual, batched activity that produces: + +1. **Plugin fixes** for the FPs (which then show up as `removed:` in the next baseline update). +2. **Issue drafts** for the real bugs in the target repos (one per repo, batched, you file). + +The expected workflow is to triage one target at a time, file an issue with the upstream repo if findings warrant it, and let the natural flow of plugin fixes update baselines over time. + +## Adding a target + +1. Pick a repo with non-trivial templates (`.gts`/`.gjs`/`.hbs`). +2. Probe its layout: `git clone --depth=1 --filter=blob:none /tmp/probe && find /tmp/probe -name '*.gts' -o -name '*.gjs' -o -name '*.hbs' | head`. +3. Pin the current default-branch SHA: `git ls-remote HEAD`. +4. Add a target entry. Globs should cover *real* templates (addon source + docs/test apps), not test fixtures (those often contain intentionally broken HTML for integration tests). +5. `npm run ecosystem:update -- --target=` to seed the baseline. +6. Commit `targets.json` and `baselines/.json` together. + +## Why bake findings into the baseline rather than gate on zero findings + +These targets have hundreds of real-world findings — some genuine, some plugin FPs, some debatable a11y choices. Refusing to merge any plugin change that doesn't drive findings to zero is impractical (the targets are out of our control). Refusing changes that *introduce* new findings is tractable. So the baseline is "current state", and the CI signal is "did this PR change real-world output, and is the change intentional?". diff --git a/ecosystem/baselines/aria-voyager.json b/ecosystem/baselines/aria-voyager.json new file mode 100644 index 0000000..2a3d15f --- /dev/null +++ b/ecosystem/baselines/aria-voyager.json @@ -0,0 +1,97 @@ +{ + "ref": "d17b494399871171e416fc943e0fb043516dd504", + "fileCount": 3, + "findings": [ + { + "file": "packages/ember-aria-voyager/tests/rendering/listbox-test.gts", + "line": 33, + "column": 14, + "ruleId": "prefer-native-element", + "message": "Prefer to use the native element" + }, + { + "file": "packages/ember-aria-voyager/tests/rendering/listbox-test.gts", + "line": 91, + "column": 13, + "ruleId": "prefer-native-element", + "message": "Prefer to use the native element" + }, + { + "file": "packages/ember-aria-voyager/tests/rendering/listbox-test.gts", + "line": 136, + "column": 16, + "ruleId": "prefer-native-element", + "message": "Prefer to use the native element" + }, + { + "file": "packages/ember-aria-voyager/tests/rendering/listbox-test.gts", + "line": 180, + "column": 16, + "ruleId": "prefer-native-element", + "message": "Prefer to use the native element" + }, + { + "file": "packages/ember-aria-voyager/tests/rendering/listbox-test.gts", + "line": 214, + "column": 16, + "ruleId": "prefer-native-element", + "message": "Prefer to use the native element" + }, + { + "file": "packages/ember-aria-voyager/tests/rendering/listbox-test.gts", + "line": 266, + "column": 13, + "ruleId": "prefer-native-element", + "message": "Prefer to use the native element" + }, + { + "file": "packages/ember-aria-voyager/tests/rendering/listbox-test.gts", + "line": 309, + "column": 16, + "ruleId": "prefer-native-element", + "message": "Prefer to use the native element is not permitted as content under
    " + }, + { + "file": "demo-app/templates/violations.gts", + "line": 103, + "column": 14, + "ruleId": "element-permitted-content", + "message": " element is not permitted as content under
      " + }, + { + "file": "demo-app/templates/violations.gts", + "line": 114, + "column": 12, + "ruleId": "element-permitted-content", + "message": " element is not permitted as content under
        " + }, + { + "file": "demo-app/templates/violations.gts", + "line": 114, + "column": 12, + "ruleId": "wcag/h37", + "message": " is missing required \"alt\" attribute" + } + ] +} diff --git a/ecosystem/baselines/ember-modifier.json b/ecosystem/baselines/ember-modifier.json new file mode 100644 index 0000000..2b86458 --- /dev/null +++ b/ecosystem/baselines/ember-modifier.json @@ -0,0 +1,5 @@ +{ + "ref": "deb8359e2e1c35b0a1166969b25110de3a473eb5", + "fileCount": 1, + "findings": [] +} diff --git a/ecosystem/baselines/ember-power-select.json b/ecosystem/baselines/ember-power-select.json new file mode 100644 index 0000000..4ee88c4 --- /dev/null +++ b/ecosystem/baselines/ember-power-select.json @@ -0,0 +1,244 @@ +{ + "ref": "bf21b6c08ff87c1921f9fb19d200f994fdfea28c", + "fileCount": 114, + "findings": [ + { + "file": "demo-app/components/custom-multiple-search-placeholder.gts", + "line": 14, + "column": 15, + "ruleId": "no-inline-style", + "message": "Inline style is not allowed" + }, + { + "file": "demo-app/components/custom-placeholder.gts", + "line": 12, + "column": 13, + "ruleId": "no-inline-style", + "message": "Inline style is not allowed" + }, + { + "file": "docs/app/components/code-example.gts", + "line": 53, + "column": 13, + "ruleId": "prefer-native-element", + "message": "Prefer to use the native